Merge pull request #83 from jina-ai/feat_base_docker_image

🐋 feat: docker base image

.github/workflows/ci.yml

@@ -59,3 +59,57 @@ jobs:
           SCENEX_API_KEY: ${{ secrets.SCENEX_API_KEY }}
           WHISPER_API_KEY: ${{ secrets.WHISPER_API_KEY }}
 
+  base-image-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Check if code relevant to executor has changed
+        uses: dorny/paths-filter@v2
+        id: check
+        with:
+          filters: |
+            changed:
+              - src/options/generate/static_files/base_image/**
+
+      - name: Get base image tag
+        if: steps.check.outputs.changed == 'true'
+        shell: bash
+        run: |
+          FILE='src/constants.py'
+          VERSION=$(sed -n '/DOCKER_BASE_IMAGE_VERSION =/p' $FILE | cut -d \' -f2)
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+
+      - name: Check that tag isn't used already for the docker base image
+        if: steps.check.outputs.changed == 'true'
+        env:
+          VERSION: ${{ env.VERSION }}
+        shell: bash
+        run: |
+          if docker pull jinaai/gpt-dev:$VERSION; then
+            echo "Executor version/tag is used already. Please update the tag"
+            exit 1
+          else
+            echo "Executor version/tag isn't used already, continue to build..."
+          fi
+
+      - name: Set up Docker Buildx
+        if: steps.check.outputs.changed == 'true'
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        if: steps.check.outputs.changed == 'true'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_DEVBOT_USER }}
+          password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }}
+
+      - name: Build and Push Docker Image
+        if: steps.check.outputs.changed == 'true'
+        uses: docker/build-push-action@v2
+        with:
+          context: src/options/generate/static_files/base_image
+          push: true
+          tags: jinaai/gpt-dev:${{ env.VERSION }}
+

src/constants.py

@@ -1,3 +1,5 @@
+DOCKER_BASE_IMAGE_VERSION = '0.0.4'
+
 EXECUTOR_FILE_NAME = '__init__.py'
 IMPLEMENTATION_FILE_NAME = 'microservice.py'
 TEST_EXECUTOR_FILE_NAME = 'test_microservice.py'
@@ -51,3 +53,4 @@ LANGUAGE_PACKAGES = [
     'pattern', 'polyglot', 'pytorch-transformers', 'rasa', 'sentence-transformers',
     'spacy', 'stanza', 'summarizer', 'sumy', 'textblob', 'textstat', 'transformers'
 ]
+

src/options/generate/generator.py

@@ -17,7 +17,7 @@ from src.apis.pypi import is_package_on_pypi, get_latest_package_version, clean_
 from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \
     BLACKLISTED_PACKAGES, EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \
     REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, IMPLEMENTATION_FILE_NAME, \
-    IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES
+    IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES, DOCKER_BASE_IMAGE_VERSION
 from src.options.generate.templates_system import system_task_iteration, system_task_introduction, system_test_iteration
 from src.options.generate.templates_user import template_generate_microservice_name, \
     template_generate_possible_packages, \
@@ -212,7 +212,7 @@ metas:
                   encoding='utf-8') as f:
             docker_file_template_lines = f.readlines()
         docker_file_template_lines = [
-            line.replace('{{apt_get_packages}}', '')
+            line.replace('{{APT_GET_PACKAGES}}', '').replace('{{DOCKER_BASE_IMAGE_VERSION}}', DOCKER_BASE_IMAGE_VERSION)
             for line in docker_file_template_lines
         ]
         docker_file_content = '\n'.join(docker_file_template_lines)
@@ -232,8 +232,8 @@ metas:
         packages = ' '.join(json.loads(json_string)['packages'])
 
         docker_file_template = self.read_docker_template()
-        return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(
-            apt_get_packages=packages)}
+        return {DOCKER_FILE_NAME: docker_file_template.replace('{{APT_GET_PACKAGES}}', '{APT_GET_PACKAGES}').replace('{{DOCKER_BASE_IMAGE_VERSION}}', DOCKER_BASE_IMAGE_VERSION).format(
+            APT_GET_PACKAGES=packages)}
 
     def parse_result_fn_requirements(self, content_raw: str):
         content_parsed = self.extract_content_from_result(content_raw, 'requirements.txt', match_single_block=True)

src/options/generate/static_files/base_image/Dockerfile

@@ -0,0 +1,7 @@
+FROM jinaai/jina:3.15.1-dev14-py39-standard
+
+# update pip
+RUN pip install --upgrade pip
+
+# install media dependencies
+RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev

src/options/generate/static_files/microservice/Dockerfile

@@ -1,8 +1,6 @@
-FROM jinaai/jina:3.14.1-py39-standard
-# update pip
-RUN pip install --upgrade pip
+FROM jinaai/gpt-dev:{{DOCKER_BASE_IMAGE_VERSION}}
 
-RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev {{apt_get_packages}} && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get install --no-install-recommends -y {{APT_GET_PACKAGES}}
 
 ## install requirements for the executor
 COPY requirements.txt .

src/options/generate/templates_user.py

@@ -170,7 +170,7 @@ template_generate_apt_get_install = PromptTemplate.from_template(
 
 {docker_file_wrapped}
 
-Name all packages which need to be installed via `apt-get install` in above Dockerfile (`{{apt_get_packages}}`) for the following requirements.txt file:
+Name all packages which need to be installed via `apt-get install` in above Dockerfile (`{{APT_GET_PACKAGES}}`) for the following requirements.txt file:
 
 {requirements_file_wrapped}
 
@@ -178,7 +178,7 @@ Note that you must not list apt-get packages that are already installed in the D
 Note that openai does not require any apt-get packages.
 Note that you are only allowed to list packages where you are highly confident that they are really needed.
 Note that you can assume that the standard python packages are already installed.
-Output the packages that need to me placed at {{apt_get_packages}} as json in the following format:
+Output the packages that need to me placed at {{APT_GET_PACKAGES}} as json in the following format:
 **apt-get-packages.json**
 ```json
 {{"packages": ["<package1>", "<package2>"]}}
@@ -275,7 +275,7 @@ Here is the summary of the error that occurred:
 {summarized_error}
 
 To solve this error, you should determine the list of packages that need to be installed via `apt-get install` in the Dockerfile.
-Output the apt-get packages that need to be placed at {{apt_get_packages}} as json in the following format:
+Output the apt-get packages that need to be placed at {{APT_GET_PACKAGES}} as json in the following format:
 **apt-get-packages.json**
 ```json
 {{"packages": ["<package1>", "<package2>"]}}