Create cassette submodule (#4420)

* gfeat: specify directory of cassettes and automatically load them depending on module

fix: formatting for linter

test: commit newly generated cassettes to their respective folder

tests: update latest fixtures with master

fix: update .gitattributes with updated path to cassettes

fix: use cassettes from master instead of generating them myself

fix: update path in .gitattributes

fix: make sure to match default functionality by using test name for cassette directory

fix: actually add git submodule

ci: checkout git submodules in CI

ci: update git submodules separately to ensure it gets called

feat: add a hooks directory so we can update git submodules on post-checkout

feat: make sure we push the tests/cassettes submodule on merge into master

ci: remove unused code now that we are using git submodules to keep cassettes in sync

fix: simplify how we load the submodule and fix updating cassettes on merge to master

chore: remove echo of checkout hook, it's unneeded

ci: remove unneccesary step

* cassettes submodule

* cassettes submodule

* cassettes submodule

* cassettes submodule

* cassettes submodule

---------

Co-authored-by: Stefan Ayala <stefanayala3266@gmail.com>

.github/workflows/ci.yml

@@ -2,9 +2,11 @@ name: Python CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, ci-test*]
+    paths-ignore:
+      - 'tests/Auto-GPT-test-cassettes'
   pull_request_target:
-    branches: [ master, stable ]
+    branches: [ master, stable , ci-test*]
 
 concurrency:
   group: ${{ format('ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }}
@@ -71,6 +73,33 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: true
+          token: ${{ secrets.PAT_REVIEW }}
+
+      - name: Use cassettes previously sent
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
+          git config --global user.name "Github Actions"
+          git config --global user.email "github-actions@github.com"
+          cd tests/Auto-GPT-test-cassettes
+
+          if git ls-remote --exit-code --heads origin $new_branch ; then
+            git fetch origin $new_branch:$new_branch
+            git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
+            
+            git checkout $new_branch
+
+            if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then
+              echo "No merge conflict detected. We can use the cassettes previously sent."
+            else
+              echo "Merge conflict detected. This means we cannot use the cassettes previously sent, so we will take the cassettes of ${{ github.event.pull_request.base.ref }}."
+              git merge --abort
+              git checkout ${{ github.event.pull_request.base.ref }}
+            fi
+          else
+            echo "Branch $new_branch does not exist. We will use the cassettes of ${{ github.event.pull_request.base.ref }}"
+          fi
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -82,7 +111,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      - name: Run unittest tests with coverage
+      - name: Run pytest tests with coverage
         run: |
           pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term
         env:
@@ -93,3 +122,82 @@ jobs:
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
+
+      - name: Update cassette submodule to push target if push event
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          cd tests/Auto-GPT-test-cassettes
+          current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g")
+          git fetch origin $current_branch
+          git config --global user.name "Auto-GPT-Bot"
+          git config --global user.email "github-bot@agpt.co"
+          git add .
+
+          # Check if there are any changes
+          if ! git diff-index --quiet HEAD; then
+              git commit -m "Auto-update cassettes after Push event"
+              git pull --rebase origin $current_branch
+
+              git push origin HEAD:refs/heads/$current_branch
+
+              cd ../..
+              git add tests/Auto-GPT-test-cassettes
+              git commit -m "Update submodule reference"
+              git push origin HEAD:refs/heads/$current_branch
+          else
+              echo "No changes to commit"
+              exit 0
+          fi
+
+      - name: Update cassette submodule to submodule branch if PR event
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}"
+
+          cd tests/Auto-GPT-test-cassettes
+          git config --global user.name "Auto-GPT-Bot"
+          git config --global user.email "github-bot@agpt.co"
+          git add .
+
+          # Check if there are any changes
+          if ! git diff-index --quiet HEAD; then
+              git commit -m "Auto-update cassettes after merging PR #$pr_number"
+              git push -f origin HEAD:refs/heads/$new_branch
+              
+          else
+              echo "No changes to commit"
+              exit 0
+          fi
+          git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} || echo "DIFF_EXISTS=false" >> $GITHUB_ENV
+          differences=$(git diff origin/$new_branch origin/${{ github.event.pull_request.base.ref }})
+    
+          if [ -n "$differences" ]; then
+            echo "DIFF_EXISTS=true" >> $GITHUB_ENV
+          else
+            echo "DIFF_EXISTS=false" >> $GITHUB_ENV
+          fi
+
+      - name: Apply or remove prompt change label and comment
+        if: ${{ github.event_name == 'pull_request_target' }}
+        run: |
+          PR_NUMBER=${{ github.event.pull_request.number }}
+          TOKEN=${{ secrets.PAT_REVIEW }}
+          REPO=${{ github.repository }}
+
+          if [[ "${{ env.DIFF_EXISTS }}" == "true" ]]; then
+            echo "Adding label and comment..."
+            curl -X POST \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \
+            -d '{"labels":["prompt change"]}'
+
+            echo $TOKEN | gh auth login --with-token
+            gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's prompt. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged."
+          else
+            echo "Removing label..."
+            curl -X DELETE \
+            -H "Authorization: Bearer $TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/prompt%20change
+          fi

.github/workflows/docker-ci.yml

@@ -76,41 +76,44 @@ jobs:
     timeout-minutes: 30
     needs: build
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: true
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
 
-    - id: build
-      name: Build image
-      uses: docker/build-push-action@v3
-      with:
-        build-args: BUILD_TYPE=dev  # include pytest
-        tags: ${{ env.IMAGE_NAME }}
-        load: true                  # save to docker images
-        # cache layers in GitHub Actions cache to speed up builds
-        cache-from: type=gha,scope=docker-dev
-        cache-to: type=gha,scope=docker-dev,mode=max
+      - id: build
+        name: Build image
+        uses: docker/build-push-action@v3
+        with:
+          build-args: BUILD_TYPE=dev  # include pytest
+          tags: ${{ env.IMAGE_NAME }}
+          load: true                  # save to docker images
+          # cache layers in GitHub Actions cache to speed up builds
+          cache-from: type=gha,scope=docker-dev
+          cache-to: type=gha,scope=docker-dev,mode=max
 
-    - id: test
-      name: Run tests
-      env:
-        CI: true
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        set +e
-        test_output=$(
-          docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
-          pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1
-        )
-        test_failure=$?
-
-        echo "$test_output"
-
-        cat << $EOF >> $GITHUB_STEP_SUMMARY
-        # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌')
-        \`\`\`
-        $test_output
-        \`\`\`
-        $EOF
+      - id: test
+        name: Run tests
+        env:
+          CI: true
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          set +e
+          test_output=$(
+            docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
+            pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term 2>&1
+          )
+          test_failure=$?
+  
+          echo "$test_output"
+  
+          cat << $EOF >> $GITHUB_STEP_SUMMARY
+          # Tests $([ $test_failure = 0 ] && echo '✅' || echo '❌')
+          \`\`\`
+          $test_output
+          \`\`\`
+          $EOF