From e2accab87e3b5d339cb5695eb55908ff52b235c3 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Wed, 19 Apr 2023 01:27:29 +0200 Subject: [PATCH 1/5] Move to Python 3.10 & improve CI workflow (#2369) * Use Python 3.10 in CI, benchmark, devcontainer, docker config, .sourcery.yaml * Improve Python CI workflow --- .devcontainer/Dockerfile | 8 +++--- .github/workflows/benchmark.yml | 4 +-- .github/workflows/ci.yml | 46 +++++++++++++++++++++++---------- .sourcery.yaml | 4 +-- Dockerfile | 2 +- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 379f6310..02f580a0 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,10 +1,10 @@ -# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster ARG VARIANT=3-bullseye -FROM --platform=linux/amd64 python:3.8 +FROM --platform=linux/amd64 python:3.10 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 - && apt-get purge -y imagemagick imagemagick-6-common + && apt-get purge -y imagemagick imagemagick-6-common # Temporary: Upgrade python packages due to https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-40897 # They are installed by the base image (python) which does not have the patch. @@ -25,4 +25,4 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # && apt-get -y install --no-install-recommends # [Optional] Uncomment this line to install global node packages. -# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5a42b2c..750366d8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,11 +9,11 @@ jobs: environment: benchmark strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb34b9d..f437a71c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,26 +2,24 @@ name: Python CI on: push: - branches: - - master + branches: [master] pull_request: - branches: - - '**' - pull_request_target: - branches: - - '**' + branches: [master] + +concurrency: + group: ${{ format('ci-{0}', format('pr-{0}', github.event.pull_request.number) || github.sha) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: + lint: runs-on: ubuntu-latest - strategy: matrix: - python-version: [3.8] + python-version: ['3.10'] steps: - name: Check out repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 @@ -34,16 +32,35 @@ jobs: pip install -r requirements.txt - name: Lint with flake8 - continue-on-error: false run: flake8 - name: Check black formatting - continue-on-error: false run: black . --check + if: success() || failure() - name: Check isort formatting - continue-on-error: false run: isort . --check + if: success() || failure() + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + + steps: + - name: Check out repository + uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Run unittest tests with coverage run: | @@ -53,3 +70,4 @@ jobs: run: | coverage report coverage xml + if: success() || failure() diff --git a/.sourcery.yaml b/.sourcery.yaml index a7f5b9d7..da171e75 100644 --- a/.sourcery.yaml +++ b/.sourcery.yaml @@ -30,7 +30,7 @@ rule_settings: - refactoring - suggestion - comment - python_version: '3.9' # A string specifying the lowest Python version your project supports. Sourcery will not suggest refactorings requiring a higher Python version. + python_version: '3.10' # A string specifying the lowest Python version your project supports. Sourcery will not suggest refactorings requiring a higher Python version. # rules: # A list of custom rules Sourcery will include in its analysis. # - id: no-print-statements @@ -68,4 +68,4 @@ rule_settings: # proxy: # url: # ssl_certs_file: -# no_ssl_verify: false \ No newline at end of file +# no_ssl_verify: false diff --git a/Dockerfile b/Dockerfile index 09b5303d..83961549 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python base image from the Docker Hub -FROM python:3.11-slim +FROM python:3.10-slim # Install git RUN apt-get -y update From 74aa4add1b066bcab5d1749a6f50898a576c9499 Mon Sep 17 00:00:00 2001 From: Will Callender <29266431+willcallender@users.noreply.github.com> Date: Tue, 18 Apr 2023 19:37:31 -0400 Subject: [PATCH 2/5] fix(python-run): prompt users to install Docker when execute_python_file encounters a Docker error (#2231) fix(python-run): make error message more explicit --------- Co-authored-by: Reinier van der Leer --- autogpt/commands/execute_code.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/autogpt/commands/execute_code.py b/autogpt/commands/execute_code.py index 95ba6122..11266f85 100644 --- a/autogpt/commands/execute_code.py +++ b/autogpt/commands/execute_code.py @@ -84,6 +84,12 @@ def execute_python_file(file: str) -> str: return logs + except docker.errors.DockerException as e: + print( + "Could not run the script in a container. If you haven't already, please install Docker https://docs.docker.com/get-docker/" + ) + return f"Error: {str(e)}" + except Exception as e: return f"Error: {str(e)}" From 24d5e1fc8aa5787c504294cc2280be45a6590a0a Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Wed, 19 Apr 2023 01:38:42 +0200 Subject: [PATCH 3/5] Ensure Python 3.10 & 3.11 compatability (#1815) CI: Ensure compatability with Python 3.10 & 3.11 --------- Co-authored-by: Reinier van der Leer --- .github/workflows/benchmark.yml | 2 +- .github/workflows/ci.yml | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 750366d8..53b6e7a0 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,7 +9,7 @@ jobs: environment: benchmark strategy: matrix: - python-version: ['3.10'] + python-version: ['3.10', '3.11'] steps: - name: Check out repository diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f437a71c..b37e7f00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,18 +13,17 @@ concurrency: jobs: lint: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.10'] + env: + min-python-version: '3.10' steps: - name: Check out repository uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ env.min-python-version }} uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.min-python-version }} - name: Install dependencies run: | @@ -46,7 +45,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10'] + python-version: ['3.10', '3.11'] steps: - name: Check out repository From 9514919d376c9aca9ff678cac3a774bf807b2d30 Mon Sep 17 00:00:00 2001 From: Josh XT <102809327+Josh-XT@users.noreply.github.com> Date: Tue, 18 Apr 2023 19:54:38 -0400 Subject: [PATCH 4/5] Option to disable working directory restrictions (#1875) Remove restriction on working directory if RESTRICT_TO_WORKSPACE != True --------- Co-authored-by: Reinier van der Leer --- .env.template | 2 ++ autogpt/commands/file_operations.py | 3 +-- autogpt/config/config.py | 3 +++ autogpt/workspace.py | 8 ++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.env.template b/.env.template index 855cb91f..4fb09cf2 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,8 @@ ################################################################################ # EXECUTE_LOCAL_COMMANDS - Allow local command execution (Example: False) EXECUTE_LOCAL_COMMANDS=False +# RESTRICT_TO_WORKSPACE - Restrict file operations to workspace ./auto_gpt_workspace (Default: True) +RESTRICT_TO_WORKSPACE=True # BROWSE_CHUNK_MAX_LENGTH - When browsing website, define the length of chunk stored in memory BROWSE_CHUNK_MAX_LENGTH=8192 # USER_AGENT - Define the user-agent used by the requests library to browse website (string) diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 72b02b5d..ad145ec9 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -3,8 +3,7 @@ from __future__ import annotations import os import os.path -from pathlib import Path -from typing import Generator, List +from typing import Generator import requests from colorama import Back, Fore diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 2fd300dd..1fe44edf 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -39,6 +39,9 @@ class Config(metaclass=Singleton): self.execute_local_commands = ( os.getenv("EXECUTE_LOCAL_COMMANDS", "False") == "True" ) + self.restrict_to_workspace = ( + os.getenv("RESTRICT_TO_WORKSPACE", "True") == "True" + ) if self.use_azure: self.load_azure_config() diff --git a/autogpt/workspace.py b/autogpt/workspace.py index 964a94d1..6fb0e311 100644 --- a/autogpt/workspace.py +++ b/autogpt/workspace.py @@ -3,6 +3,10 @@ from __future__ import annotations import os from pathlib import Path +from autogpt.config import Config + +CFG = Config() + # Set a dedicated folder for file I/O WORKSPACE_PATH = Path(os.getcwd()) / "auto_gpt_workspace" @@ -35,9 +39,9 @@ def safe_path_join(base: Path, *paths: str | Path) -> Path: """ joined_path = base.joinpath(*paths).resolve() - if not joined_path.is_relative_to(base): + if CFG.restrict_to_workspace and not joined_path.is_relative_to(base): raise ValueError( - f"Attempted to access path '{joined_path}' outside of working directory '{base}'." + f"Attempted to access path '{joined_path}' outside of workspace '{base}'." ) return joined_path From 8532307b2f73e864f77b90c63a4deadf44b48404 Mon Sep 17 00:00:00 2001 From: Will Callender <29266431+willcallender@users.noreply.github.com> Date: Tue, 18 Apr 2023 20:16:08 -0400 Subject: [PATCH 5/5] Rename `evaluate_code` to `analyze_code` (#1371) ChatGPT is less confused by this phrasing From my own observations and others (ie #101 and #286) ChatGPT seems to think that `evaluate_code` will actually run code, rather than just provide feedback. Since changing the phrasing to `analyze_code` I haven't seen the AI make this mistake. --------- Co-authored-by: Reinier van der Leer --- autogpt/app.py | 6 +++--- autogpt/commands/{evaluate_code.py => analyze_code.py} | 2 +- autogpt/prompt.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename autogpt/commands/{evaluate_code.py => analyze_code.py} (94%) diff --git a/autogpt/app.py b/autogpt/app.py index 381f5a2a..246213a5 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -3,8 +3,8 @@ import json from typing import Dict, List, NoReturn, Union from autogpt.agent.agent_manager import AgentManager +from autogpt.commands.analyze_code import analyze_code from autogpt.commands.audio_text import read_audio_from_file -from autogpt.commands.evaluate_code import evaluate_code from autogpt.commands.execute_code import ( execute_python_file, execute_shell, @@ -181,8 +181,8 @@ def execute_command(command_name: str, arguments): # TODO: Change these to take in a file rather than pasted code, if # non-file is given, return instructions "Input should be a python # filepath, write your code to file and try again" - elif command_name == "evaluate_code": - return evaluate_code(arguments["code"]) + elif command_name == "analyze_code": + return analyze_code(arguments["code"]) elif command_name == "improve_code": return improve_code(arguments["suggestions"], arguments["code"]) elif command_name == "write_tests": diff --git a/autogpt/commands/evaluate_code.py b/autogpt/commands/analyze_code.py similarity index 94% rename from autogpt/commands/evaluate_code.py rename to autogpt/commands/analyze_code.py index 8f7cbca9..e02ea4c5 100644 --- a/autogpt/commands/evaluate_code.py +++ b/autogpt/commands/analyze_code.py @@ -4,7 +4,7 @@ from __future__ import annotations from autogpt.llm_utils import call_ai_function -def evaluate_code(code: str) -> list[str]: +def analyze_code(code: str) -> list[str]: """ A function that takes in a string and returns a response from create chat completion api call. diff --git a/autogpt/prompt.py b/autogpt/prompt.py index a0456305..03c132ac 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -73,7 +73,7 @@ def get_prompt() -> str: ("Append to file", "append_to_file", {"file": "", "text": ""}), ("Delete file", "delete_file", {"file": ""}), ("Search Files", "search_files", {"directory": ""}), - ("Evaluate Code", "evaluate_code", {"code": ""}), + ("Analyze Code", "analyze_code", {"code": ""}), ( "Get Improved Code", "improve_code",