diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 90d8c116..c05a326e 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -4,16 +4,9 @@ version: '3.9' services: auto-gpt: - depends_on: - - redis build: dockerfile: .devcontainer/Dockerfile context: ../ tty: true - environment: - MEMORY_BACKEND: ${MEMORY_BACKEND:-redis} - REDIS_HOST: ${REDIS_HOST:-redis} volumes: - ../:/workspace/Auto-GPT - redis: - image: 'redis/redis-stack-server:latest' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8651931c..df9148c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,19 +2,24 @@ name: Python CI on: push: - branches: [ master, ci-test*] + branches: [ master, ci-test* ] paths-ignore: - 'tests/Auto-GPT-test-cassettes' - 'tests/integration/challenges/current_score.json' + pull_request: + branches: [ stable, master ] pull_request_target: - branches: [ master, stable , ci-test*] + branches: [ master, ci-test* ] concurrency: group: ${{ format('ci-{0}', github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }} - cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} + cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') && github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target') }} jobs: lint: + # eliminate duplicate runs on master + if: github.event_name == 'push' || github.ref_name != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) + runs-on: ubuntu-latest env: min-python-version: "3.10" @@ -67,6 +72,9 @@ jobs: $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1) test: + # eliminate duplicate runs on master + if: github.event_name == 'push' || github.ref_name != 'master' || (github.event.pull_request.head.repo.fork == (github.event_name == 'pull_request_target')) + permissions: # Gives the action the necessary permissions for publishing new # comments in pull requests. @@ -89,31 +97,35 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} submodules: true - token: ${{ secrets.PAT_REVIEW }} - - name: Use cassettes previously sent - if: ${{ github.event_name == 'pull_request_target' }} + - id: checkout_cassettes + name: Check out cassettes + if: ${{ startsWith(github.event_name, 'pull_request') }} run: | - new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" - git config --global user.name "Github Actions" - git config --global user.email "github-actions@github.com" + cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" cd tests/Auto-GPT-test-cassettes - if git ls-remote --exit-code --heads origin $new_branch ; then - git fetch origin $new_branch:$new_branch - git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} - - git checkout $new_branch + if git ls-remote --exit-code --heads origin $cassette_branch ; then + git fetch origin $cassette_branch + git fetch origin ${{ github.event.pull_request.base.ref }} + + git checkout $cassette_branch if git merge --no-commit --no-ff ${{ github.event.pull_request.base.ref }}; then - echo "No merge conflict detected. We can use the cassettes previously sent." + echo "Using cassettes from mirror branch, synced to upstream branch '${{ github.event.pull_request.base.ref }}'" else - echo "Merge conflict detected. This means we cannot use the cassettes previously sent, so we will take the cassettes of ${{ github.event.pull_request.base.ref }}." + echo "Could not merge upstream changes to cassettes. Using cassettes from ${{ github.event.pull_request.base.ref }}." git merge --abort git checkout ${{ github.event.pull_request.base.ref }} + + # Delete branch to prevent conflict when re-creating it + git branch -D $cassette_branch fi + echo "cassette_branch=$(git branch --show-current)" >> $GITHUB_OUTPUT else - echo "Branch $new_branch does not exist. We will use the cassettes of ${{ github.event.pull_request.base.ref }}" + echo "Branch '$cassette_branch' does not exist in cassette submodule."\ + "Using cassettes from ${{ github.event.pull_request.base.ref }}." + echo "cassette_branch=${{ github.event.pull_request.base.ref }}" >> $GITHUB_OUTPUT fi - name: Set up Python ${{ matrix.python-version }} @@ -148,77 +160,100 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 - - name: Update cassette submodule to push target if push event - if: ${{ github.event_name == 'push' }} + - id: setup_git_auth + name: Set up git token authentication run: | - current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g") git config --global user.name "Auto-GPT-Bot" git config --global user.email "github-bot@agpt.co" - git add tests/integration/challenges/current_score.json - if ! git diff-index --quiet HEAD; then - git commit -m "Update current score" - git push origin HEAD:refs/heads/$current_branch - else - echo "The current score didn't change." - fi - + config_key="http.${{ github.server_url }}/.extraheader" + base64_pat=$(echo -n "pat:${{ secrets.PAT_REVIEW }}" | base64 -w0) + + git config "$config_key" \ + "Authorization: Basic $base64_pat" + cd tests/Auto-GPT-test-cassettes - git fetch origin $current_branch - git add . + git config "$config_key" \ + "Authorization: Basic $base64_pat" - # Check if there are any changes - if ! git diff-index --quiet HEAD; then - git commit -m "Auto-update cassettes after Push event" - git pull --rebase origin $current_branch + echo "config_key=$config_key" >> $GITHUB_OUTPUT - git push origin HEAD:refs/heads/$current_branch + - name: Push updated challenge scores + if: github.event_name == 'push' + run: | + score_file="tests/integration/challenges/current_score.json" - cd ../.. + if ! git diff --quiet $score_file; then + git add $score_file + git commit -m "Update challenge scores" + git push origin HEAD:${{ github.ref }} + else + echo "The challenge scores didn't change." + fi + + - id: push_cassettes + name: Push updated cassettes + run: | + if [ "${{ startsWith(github.event_name, 'pull_request') }}" = "true" ]; then + is_pull_request=true + cassette_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" + cassette_source_branch="${{ steps.checkout_cassettes.outputs.cassette_branch }}" + base_branch="${{ github.event.pull_request.base.ref }}" + else + current_branch=$(echo ${{ github.ref }} | sed -e "s/refs\/heads\///g") + cassette_branch=$current_branch + fi + + cd tests/Auto-GPT-test-cassettes + git fetch origin $cassette_source_branch:$cassette_source_branch + + # Commit & push changes to cassettes if any + if ! git diff --quiet $cassette_source_branch --; then + if [ "$cassette_branch" != "$cassette_source_branch" ]; then + git checkout -b $cassette_branch + fi + git add . + git commit -m "Auto-update cassettes" + + if [ $is_pull_request ]; then + git push --force origin HEAD:$cassette_branch + else + git push origin HEAD:$cassette_branch + fi + + cd ../.. + if [ $is_pull_request ]; then + git fetch origin $base_branch + cassette_diff=$(git diff origin/$base_branch) + else git add tests/Auto-GPT-test-cassettes - git commit -m "Update submodule reference" - git push origin HEAD:refs/heads/$current_branch + git commit -m "Update cassette submodule" + git push origin HEAD:$current_branch + fi else - echo "No cassettes changes to commit" - exit 0 + echo "No cassette changes to commit" fi - - name: Update cassette submodule to submodule branch if PR event - if: ${{ github.event_name == 'pull_request_target' }} + if [ -n "$cassette_diff" ]; then + echo "updated=true" >> $GITHUB_OUTPUT + else + echo "updated=false" >> $GITHUB_OUTPUT + fi + + - name: Post Set up git token auth + if: steps.setup_git_auth.outcome == 'success' run: | - new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" + git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' + git submodule foreach git config --unset-all '${{ steps.setup_git_auth.outputs.config_key }}' - cd tests/Auto-GPT-test-cassettes - git config --global user.name "Auto-GPT-Bot" - git config --global user.email "github-bot@agpt.co" - git add . - - # Check if there are any changes - if ! git diff-index --quiet HEAD; then - git commit -m "Auto-update cassettes after merging PR #$pr_number" - git push -f origin HEAD:refs/heads/$new_branch - - else - echo "No changes to commit" - exit 0 - fi - git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} || echo "DIFF_EXISTS=false" >> $GITHUB_ENV - differences=$(git diff origin/$new_branch origin/${{ github.event.pull_request.base.ref }}) - - if [ -n "$differences" ]; then - echo "DIFF_EXISTS=true" >> $GITHUB_ENV - else - echo "DIFF_EXISTS=false" >> $GITHUB_ENV - fi - - - name: Apply or remove behaviour change label and comment - if: ${{ github.event_name == 'pull_request_target' }} + - name: Apply or remove behaviour change label and comment on PR + if: ${{ startsWith(github.event_name, 'pull_request') }} run: | PR_NUMBER=${{ github.event.pull_request.number }} TOKEN=${{ secrets.PAT_REVIEW }} REPO=${{ github.repository }} - if [[ "${{ env.DIFF_EXISTS }}" == "true" ]]; then + if [[ "${{ steps.push_cassettes.outputs.updated }}" == "true" ]]; then echo "Adding label and comment..." curl -X POST \ -H "Authorization: Bearer $TOKEN" \ diff --git a/BULLETIN.md b/BULLETIN.md index 70be3c3e..4c858b73 100644 --- a/BULLETIN.md +++ b/BULLETIN.md @@ -14,46 +14,34 @@ We have to be somewhat selective in order to keep making progress, but this does mean you can't contribute. Check out the contribution guide on our wiki: https://github.com/Significant-Gravitas/Auto-GPT/wiki/Contributing -# ๐Ÿš€ v0.3.1 Release ๐Ÿš€ -Over a week and 47 pull requests have passed since v0.3.0, and we are happy to announce -the release of v0.3.1! +# ๐Ÿš€ v0.4.0 Release ๐Ÿš€ +Two weeks and 76 pull requests have passed since v0.3.1, and we are happy to announce +the release of v0.4.0! -Highlights and notable changes since v0.2.2: +Highlights and notable changes since v0.3.0: -## Changes to Docker configuration ๐Ÿ‹ - * The workdir has been changed from */home/appuser* to */app*. - Be sure to update any volume mounts accordingly! - * Docker-compose 1.29.0 is now required. +## โš ๏ธ Command `send_tweet` is REMOVED +Twitter functionality (and more) is now covered by plugins. -## Logging ๐Ÿงพ - * Log functionality has been improved for better understanding - and easier summarization. - * All LLM interactions are now logged to logs/DEBUG, to help with - debugging and development. - -## Other - * Edge browser is now supported by the `browse_website` command. - * Sets of commands can now be disabled using DISABLED_COMMAND_CATEGORIES in .env. - -# โš ๏ธ Command `send_tweet` is REMOVED -Twitter functionality (and more) is now covered by plugins, see [Plugin support ๐Ÿ”Œ] - -## Plugin support ๐Ÿ”Œ -Auto-GPT now has support for plugins! With plugins, you can extend Auto-GPT's abilities, -adding support for third-party services and more. -See https://github.com/Significant-Gravitas/Auto-GPT-Plugins for instructions and -available plugins. Specific plugins can be allowlisted/denylisted in .env. - -## Memory backend deprecation โš ๏ธ +## โš ๏ธ Memory backend deprecation ๐Ÿ’พ The Milvus, Pinecone and Weaviate memory backends were rendered incompatible by work on the memory system, and have been removed in `master`. The Redis -memory store was also temporarily removed but we aim to merge a new implementation -before the next release. +memory store was also temporarily removed; we will merge a new implementation ASAP. Whether built-in support for the others will be added back in the future is subject to discussion, feel free to pitch in: https://github.com/Significant-Gravitas/Auto-GPT/discussions/4280 -# Challenge Workflow ๐Ÿ† -If you have been working on challenges... Thank You! -But to run the debugger challenge or other challenges using cassettes and VCR in docker, You will now need to `pip uninstall vcrpy` and `pip install -r requirements.txt` again. -This will install a new version of vcrpy that is compatible with running vcr in docker. -This workflow will be fixed as soon as the maintainer from VCRpy merges our changes. +## Document support in `read_file` ๐Ÿ“„ +Auto-GPT can now read text from document files, with support added for PDF, DOCX, CSV, +HTML, TeX and more! + +## Managing Auto-GPT's access to commands โŒ๐Ÿ”ง +You can now disable set of built-in commands through the *DISABLED_COMMAND_CATEGORIES* +variable in .env. Specific shell commands can also be disabled using *DENY_COMMANDS*, +or selectively enabled using *ALLOW_COMMANDS*. + +## Further fixes and changes ๐Ÿ› ๏ธ +Other highlights include improvements to self-feedback mode and continuous mode, +documentation, docker and devcontainer setups, and much more. Most of the improvements +that were made are not yet visible to users, but will pay off in the long term. +Take a look at the Release Notes on Github for the full changelog! +https://github.com/Significant-Gravitas/Auto-GPT/releases diff --git a/docker-compose.yml b/docker-compose.yml index a23aa431..d6878e45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,17 +5,9 @@ version: "3.9" services: auto-gpt: - depends_on: - - redis build: ./ env_file: - .env - environment: - MEMORY_BACKEND: ${MEMORY_BACKEND:-redis} - REDIS_HOST: ${REDIS_HOST:-redis} volumes: - ./:/app profiles: ["exclude-from-up"] - - redis: - image: "redis/redis-stack-server:latest" diff --git a/docs/setup.md b/docs/setup.md index c4755a8d..257e07c1 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -47,13 +47,8 @@ Get your OpenAI API key from: [https://platform.openai.com/account/api-keys](htt services: auto-gpt: image: significantgravitas/auto-gpt - depends_on: - - redis env_file: - .env - environment: - MEMORY_BACKEND: ${MEMORY_BACKEND:-redis} - REDIS_HOST: ${REDIS_HOST:-redis} profiles: ["exclude-from-up"] volumes: - ./auto_gpt_workspace:/app/autogpt/auto_gpt_workspace @@ -68,8 +63,6 @@ Get your OpenAI API key from: [https://platform.openai.com/account/api-keys](htt #- type: bind # source: ./ai_settings.yaml # target: /app/ai_settings.yaml - redis: - image: "redis/redis-stack-server:latest" 4. Create the necessary [configuration](#configuration) files. If needed, you can find templates in the [repository]. diff --git a/pyproject.toml b/pyproject.toml index bf71c70c..d695ac08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agpt" -version = "0.3.0" +version = "0.4.0" authors = [ { name="Torantulino", email="support@agpt.co" }, ] diff --git a/tests/unit/test_web_selenium.py b/tests/integration/test_web_selenium.py similarity index 67% rename from tests/unit/test_web_selenium.py rename to tests/integration/test_web_selenium.py index 0415007d..2a03a3c0 100644 --- a/tests/unit/test_web_selenium.py +++ b/tests/integration/test_web_selenium.py @@ -1,7 +1,10 @@ +from pytest_mock import MockerFixture + from autogpt.commands.web_selenium import browse_website +from autogpt.config import Config -def test_browse_website(config): +def test_browse_website(config: Config, patched_api_requestor: MockerFixture): url = "https://barrel-roll.com" question = "How to execute a barrel roll"