Improve docker setup & config (#1843)

* Improve docker setup & config

* fix(browsing): Selenium needs access to home directory

* fix(docker): allow overriding memory backend settings

* simplify Dockerfile and docker-compose config

* add .dockerignore

* adjust Docker CI with release build type arg

* replace Chrome by Chromium in devcontainer

* update docs

* update bulletin

* use preinstalled chromedriver in web_selenium.py

* update installation.md

* fix code blocks for mkdocs

* fix links to docs

.devcontainer/Dockerfile

@@ -1,6 +1,6 @@
 # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster
 ARG VARIANT=3-bullseye
-FROM --platform=linux/amd64 python:3.10
+FROM python:3.10
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
@@ -10,10 +10,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 # They are installed by the base image (python) which does not have the patch.
 RUN python3 -m pip install --upgrade setuptools
 
-# Install Chrome for web browsing
+# Install Chromium for web browsing
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+RUN apt-get install -y chromium-driver
-    && curl -sSL https://dl.google.com/linux/direct/google-chrome-stable_current_$(dpkg --print-architecture).deb -o /tmp/chrome.deb \
-    && apt-get -y install /tmp/chrome.deb
 
 # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
 # COPY requirements.txt /tmp/pip-tmp/

.dockerignore

@@ -0,0 +1,8 @@
+.*
+*.template
+*.yaml
+*.yml
+
+*.md
+*.png
+!BULLETIN.md

.github/workflows/docker-image.yml

@@ -18,4 +18,4 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Build the Docker image
-      run: docker build . --file Dockerfile --tag autogpt:$(date +%s)
+      run: docker build . --build-arg BUILD_TYPE=release --tag autogpt:$(date +%s)

BULLETIN.md

@@ -1,2 +1,5 @@
 Welcome to Auto-GPT! We'll keep you informed of the latest news and features by printing messages here.
 If you don't wish to see this message, you can run Auto-GPT with the --skip-news flag
+
+## Changes to Docker configuration
+The workdir has been changed from /home/appuser to /app. Be sure to update any volume mounts accordingly.

Dockerfile

@@ -1,38 +1,33 @@
 # Use an official Python base image from the Docker Hub
 FROM python:3.10-slim
 
-# Install git
+# 'dev' or 'release' container build
-RUN apt-get -y update
+ARG BUILD_TYPE=dev
-RUN apt-get -y install git chromium-driver
 
-# Install Xvfb and other dependencies for headless browser testing
+# Install browsers
-RUN apt-get update \
+RUN apt-get update && apt-get install -y \
-    && apt-get install -y wget gnupg2 libgtk-3-0 libdbus-glib-1-2 dbus-x11 xvfb ca-certificates
+    chromium-driver firefox-esr \
+    ca-certificates
 
-# Install Firefox / Chromium
+# Install utilities
-RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+RUN apt-get install -y curl jq wget git
-    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
-    && apt-get update \
-    && apt-get install -y chromium firefox-esr
 
 # Set environment variables
 ENV PIP_NO_CACHE_DIR=yes \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
 
-# Create a non-root user and set permissions
+# Install the required python packages globally
-RUN useradd --create-home appuser
+ENV PATH="$PATH:/root/.local/bin"
-WORKDIR /home/appuser
+COPY requirements.txt .
-RUN chown appuser:appuser /home/appuser
-USER appuser
 
-# Copy the requirements.txt file and install the requirements
+# Only install dev dependencies in dev container builds
-COPY --chown=appuser:appuser requirements.txt .
+RUN [ '${BUILD_TYPE}' = 'dev' ] || sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \
-RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \
+	pip install --no-cache-dir -r requirements.txt
-	pip install --no-cache-dir --user -r requirements.txt
 
 # Copy the application files
-COPY --chown=appuser:appuser autogpt/ ./autogpt
+WORKDIR /app
+COPY autogpt/ ./autogpt
 
 # Set the entrypoint
 ENTRYPOINT ["python", "-m", "autogpt"]

README.md

@@ -59,37 +59,37 @@ Your support is greatly appreciated. Development of this free, open-source proje
 
 ## 📋 Requirements
 
-- Environment (pick one)
+Choose an environment to run Auto-GPT in (pick one):
+
+  - [Docker](https://docs.docker.com/get-docker/) (*recommended*)
   - Python 3.10 or later (instructions: [for Windows](https://www.tutorialspoint.com/how-to-install-python-in-windows))
-  - Docker [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt)
   - [VSCode + devcontainer](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
 
 ## Quickstart
 
 1. Set up your OpenAI [API Keys](https://platform.openai.com/account/api-keys)
 2. Download the [latest release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest)
-3. Install the [dependencies](/docs/installation.md) and set your [environment variables](/docs/installation.md)
+3. Follow the [installation instructions][docs/install]
-4. Configure any additional features you want, or install some [plugins](/docs/plugins.md)
+4. Configure any additional features you want, or install some [plugins][docs/plugins]
-5. [Run](/docs/usage.md) the app
+5. [Run][docs/usage] the app
 
-Please see the [documentation](/docs) linked below for full setup instructions and configuration options.
+Please see the [documentation][docs] linked below for full setup instructions and configuration options.
 
-## 💾 Documentation
+[docs]: https://significant-gravitas.github.io/Auto-GPT/
 
-[Installation](/docs/installation.md)
+## 📖 Documentation
-
+* [⚙️ Installation][docs/install]
-[Usage](/docs/usage.md)
+* [💻 Usage][docs/usage]
-
+* [🔌 Plugins][docs/plugins]
-[Speech](/docs/configuration/voice.md)
+* Configuration
-
+  * [🔍 Web Search](https://significant-gravitas.github.io/Auto-GPT/configuration/search/)
-[Search](/docs/configuration/search.md)
+  * [🧠 Memory](https://significant-gravitas.github.io/Auto-GPT/configuration/memory/)
-
+  * [🗣️ Voice (TTS)](https://significant-gravitas.github.io/Auto-GPT/configuration/voice/)
-[Plugins](/docs/plugins.md)
+  * [🖼️ Image Generation](https://significant-gravitas.github.io/Auto-GPT/configuration/imagegen/)
-
-[Memory](/docs/configuration/memory.md)
-
-[Image Generation](/docs/configuration/imagegen.md)
 
+[docs/install]: https://significant-gravitas.github.io/Auto-GPT/installation/
+[docs/usage]: https://significant-gravitas.github.io/Auto-GPT/usage/
+[docs/plugins]: https://significant-gravitas.github.io/Auto-GPT/plugins/
 
 ## ⚠️ Limitations
 

autogpt/commands/web_selenium.py

@@ -95,8 +95,13 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]:
             options.add_argument("--headless")
             options.add_argument("--disable-gpu")
 
+        chromium_driver_path = Path("/usr/bin/chromedriver")
+
         driver = webdriver.Chrome(
-            executable_path=ChromeDriverManager().install(), options=options
+            executable_path=chromium_driver_path
+            if chromium_driver_path.exists()
+            else ChromeDriverManager().install(),
+            options=options,
         )
     driver.get(url)
 

docker-compose.yml

@@ -9,9 +9,11 @@ services:
     build: ./
     env_file:
       - .env
+    environment:
+      MEMORY_BACKEND: ${MEMORY_BACKEND:-redis}
+      REDIS_HOST: ${REDIS_HOST:-redis}
     volumes:
-      - "./autogpt:/home/appuser"
+      - ./:/app
-      - ".env:/home/appuser/.env"
     profiles: ["exclude-from-up"]
 
   redis:

docs/installation.md

@@ -11,20 +11,17 @@ You can set up paid account at https://platform.openai.com/account/billing/overv
 Important: It's highly recommended that you track your usage on [the Usage page](https://platform.openai.com/account/usage)
 You can also set limits on how much you spend on [the Usage limits page](https://platform.openai.com/account/billing/limits).
 
-![For OpenAI API key to work, set up paid account at OpenAI API > Billing](./docs/imgs/openai-api-key-billing-paid-account.png)
+![For OpenAI API key to work, set up paid account at OpenAI API > Billing](./imgs/openai-api-key-billing-paid-account.png)
 
 **PLEASE ENSURE YOU HAVE DONE THIS STEP BEFORE PROCEEDING. OTHERWISE, NOTHING WILL WORK!**
 
-## Steps
+## General setup
 
-To install Auto-GPT, follow these steps:
+1. Make sure you have one of the environments listed under [**requirements**](../README.md#-requirements) set up.
-
-1. Make sure you have all the **requirements** listed in the [README](../README.md). If not, install/get them.
 
     _To execute the following commands, open a CMD, Bash, or Powershell window by navigating to a folder on your computer and typing `CMD` in the folder path at the top, then press enter._
 
-2. Clone the repository: For this step, you need Git installed. 
+2. Clone the repository using Git, or download the [latest stable release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) (`Source code (zip)`, at the bottom of the page).
-Note: If you don't have Git, you can just download the [latest stable release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) instead (`Source code (zip)`, at the bottom of the page).
 
     ``` shell
     git clone -b stable https://github.com/Significant-Gravitas/Auto-GPT.git
@@ -36,22 +33,16 @@ Note: If you don't have Git, you can just download the [latest stable release](h
     cd Auto-GPT
     ```
 
-4. Install the required dependencies.
-
-``` shell
-    pip install -r requirements.txt
-```
-
 5. Configure Auto-GPT:
-   1. Find the file named `.env.template` in the main /Auto-GPT folder. This file may be hidden by default in some operating systems due to the dot prefix. To reveal hidden files, follow the instructions for your specific operating system (e.g., in Windows, click on the "View" tab in File Explorer and check the "Hidden items" box; in macOS, press Cmd + Shift + .).
+   1. Find the file named `.env.template` in the main `Auto-GPT` folder. This file may be hidden by default in some operating systems due to the dot prefix. To reveal hidden files, follow the instructions for your specific operating system (e.g., in Windows, click on the "View" tab in File Explorer and check the "Hidden items" box; in macOS, press Cmd + Shift + .).
    2. Create a copy of this file and call it `.env` by removing the `template` extension.  The easiest way is to do this in a command prompt/terminal window `cp .env.template .env`.
    3. Open the `.env` file in a text editor.
    4. Find the line that says `OPENAI_API_KEY=`.
    5. After the `"="`, enter your unique OpenAI API Key (without any quotes or spaces).
-   6. Enter any other API keys or Tokens for services you would like to use.
+   6. Enter any other API keys or Tokens for services you would like to use. To activate and adjust a setting, remove the `# ` prefix.
    7. Save and close the `.env` file.
 
-   After you complete these steps, you'll have properly configured the API keys for your project.
+   You have now configured Auto-GPT.
 
    Notes:
    - See [OpenAI API Keys Configuration](#openai-api-keys-configuration) to get your OpenAI API key.
@@ -61,8 +52,8 @@ Note: If you don't have Git, you can just download the [latest stable release](h
        - `fast_llm_model_deployment_id` - your gpt-3.5-turbo or gpt-4 deployment ID
        - `smart_llm_model_deployment_id` - your gpt-4 deployment ID
        - `embedding_model_deployment_id` - your text-embedding-ada-002 v2 deployment ID
-     - Please specify all of these values as double-quoted strings
 
+     - Please specify all of these values as double-quoted strings
         ``` shell
         # Replace string in angled brackets (<>) to your own ID
         azure_model_map:
@@ -72,27 +63,43 @@ azure_model_map:
      - Details can be found here: https://pypi.org/project/openai/ in the `Microsoft Azure Endpoints` section and here: https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings?tabs=command-line for the embedding model.
      - If you're on Windows you may need to install https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170
 
-## Docker
+4. Follow the further instructions for running Auto-GPT with [Docker](#run-with-docker) (*recommended*), or [Docker-less](#run-docker-less)
 
-You can also build this into a docker image and run it:
+### Run with Docker
 
+Easiest is to run with `docker-compose`:
 ``` shell
-docker build -t autogpt .
+docker-compose build auto-gpt
-docker run -it --env-file=./.env -v $PWD/auto_gpt_workspace:/home/appuser/auto_gpt_workspace autogpt
+docker-compose run --rm auto-gpt
 ```
+By default, this will also start and attach a Redis memory backend.
+For related settings, see [Memory > Redis setup](./configuration/memory.md#redis-setup).
 
-Or if you have `docker-compose`:
+You can also build and run it with "vanilla" docker commands:
 ``` shell
-docker-compose run --build --rm auto-gpt
+docker build -t auto-gpt .
+docker run -it --env-file=.env -v $PWD:/app auto-gpt
 ```
 
 You can pass extra arguments, for instance, running with `--gpt3only` and `--continuous` mode:
 ``` shell
-docker run -it --env-file=./.env -v $PWD/auto_gpt_workspace:/home/appuser/auto_gpt_workspace autogpt --gpt3only --continuous
+docker-compose run --rm auto-gpt --gpt3only --continuous
 ```
-
 ``` shell
-docker-compose run --build --rm auto-gpt --gpt3only --continuous
+docker run -it --env-file=.env -v $PWD:/app --rm auto-gpt --gpt3only --continuous
 ```
 
-Alternatively, you can pull the latest release directly from [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt)
+Alternatively, you can pull the latest release directly from [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt) and run that:
+``` shell
+docker run -it --env OPENAI_API_KEY='your-key-here' --rm significantgravitas/auto-gpt
+```
+
+Or with `ai_settings.yml` presets mounted:
+``` shell
+docker run -it --env OPENAI_API_KEY='your-key-here' -v $PWD/ai_settings.yaml:/app/ai_settings.yaml --rm significantgravitas/auto-gpt
+```
+
+
+### Run Docker-less
+
+Simply run `./run.sh` (Linux/macOS) or `.\run.bat` (Windows) in your terminal.

docs/usage.md

@@ -1,15 +1,25 @@
 # Usage
 
-1. Run the `autogpt` Python module in your terminal.
+1. Open a terminal and run the startup script:
     * On Linux/MacOS:
-   ```   
+        ``` shell
         ./run.sh
         ```
     * On Windows:
-   ```   
+        ``` shell
         .\run.bat
         ```
-   Running with `--help` after `.\run.bat` lists all the possible command line arguments you can pass.
+    * Using Docker:
+        ``` shell
+        docker-compose run --rm auto-gpt
+        ```
+    Running with `--help` lists all the possible command line arguments you can pass:
+    ``` shell
+    ./run.sh --help
+
+    # or with docker
+    docker-compose run --rm auto-gpt --help
+    ```
 
 2. After each response from Auto-GPT, choose from the options to authorize command(s),
 exit the program, or provide feedback to the AI.
@@ -24,15 +34,15 @@ Here are some common arguments you can use when running Auto-GPT:
 > Replace anything in angled brackets (<>) to a value you want to specify
 
 * View all available command line arguments
-    ```    
+    ``` shell
     python -m autogpt --help
     ```
 * Run Auto-GPT with a different AI Settings file
-    ```    
+    ``` shell
     python -m autogpt --ai-settings <filename>
     ```
 * Specify a memory backend
-    ```    
+    ``` shell
     python -m autogpt --use-memory  <memory-backend>
     ```
 
@@ -55,7 +65,7 @@ Use at your own risk.
 
 1. Run the `autogpt` python module in your terminal:
 
-    ```    
+    ``` shell
     python -m autogpt --continuous
     ```