From 6204532a0b20172b359aac8032fc8e29465f1122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Wed, 3 May 2023 16:21:16 +0200 Subject: [PATCH 01/25] =?UTF-8?q?=E2=9C=85=20fix:=20docker=20dependency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 49 +++++++++++++++---------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 597fd89..f5eeb24 100644 --- a/README.md +++ b/README.md @@ -58,13 +58,11 @@ We are working on a way to use gpt-3.5-turbo as well. ```bash gptdeploy generate \ --description "" \ ---test "" \ --model \ --path ``` To generate your personal microservice two things are required: - A `description` of the task you want to accomplish. -- A `test` scenario that ensures the microservice works as expected. - The `model` you want to use - either `gpt-3.5` or `gpt-4`. `gpt-3.5` is ~10x cheaper, but will not be able to generate as complex microservices. - A `path` on the local drive where the microservice will be generated. @@ -105,7 +103,7 @@ In this section you can get a feeling for the kind of microservices that can be ```bash gptdeploy generate \ --description "The user writes something and gets a related deep compliment." \ ---test "Given the word test a deep compliment is generated" \ +--test "" \ --model gpt-4 \ --path microservice ``` @@ -115,8 +113,7 @@ gptdeploy generate \ ### Extract and summarize news articles given a URL ```bash gptdeploy generate \ ---description "Extract text from a news article URL using Newspaper3k library and generate a summary using gpt." \ ---test "input: 'http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/' output: assert a summarized version of the article exists" \ +--description "Extract text from a news article URL using Newspaper3k library and generate a summary using gpt. Example input: http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/" \ --model gpt-4 \ --path microservice ``` @@ -125,8 +122,7 @@ gptdeploy generate \ ### Chemical Formula Visualization ```bash gptdeploy generate \ ---description "Convert a chemical formula into a 2D chemical structure diagram" \ ---test "C=C, CN=C=O, CCC(=O)O" \ +--description "Convert a chemical formula into a 2D chemical structure diagram. Example inputs: C=C, CN=C=O, CCC(=O)O" \ --model gpt-4 \ --path microservice ``` @@ -135,8 +131,7 @@ gptdeploy generate \ ### 2d rendering of 3d model ```bash gptdeploy generate \ ---description "create a 2d rendering of a whole 3d object and x,y,z object rotation using trimesh and pyrender.OffscreenRenderer with os.environ['PYOPENGL_PLATFORM'] = 'egl' and freeglut3-dev library" \ ---test "input: https://graphics.stanford.edu/courses/cs148-10-summer/as3/code/as3/teapot.obj output: assert the image is not completely white or black" \ +--description "create a 2d rendering of a whole 3d object and x,y,z object rotation using trimesh and pyrender.OffscreenRenderer with os.environ['PYOPENGL_PLATFORM'] = 'egl' and freeglut3-dev library - example input: https://graphics.stanford.edu/courses/cs148-10-summer/as3/code/as3/teapot.obj" \ --model gpt-4 \ --path microservice ``` @@ -145,8 +140,7 @@ gptdeploy generate \ ### Product Recommendation ```bash gptdeploy generate \ ---description "Generate personalized product recommendations based on user product browsing history and the product categories fashion, electronics and sport" \ ---test "Test that a user how visited p1(electronics),p2(fashion),p3(fashion) is more likely to buy p4(fashion) than p5(sports)" \ +--description "Generate personalized product recommendations based on user product browsing history and the product categories fashion, electronics and sport. Example: Input: browsing history: prod1(electronics),prod2(fashion),prod3(fashion), output: p4(fashion)" \ --model gpt-4 \ --path microservice ``` @@ -156,7 +150,6 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a search query, find articles on hacker news using the hacker news api and return a list of (title, author, website_link, first_image_on_the_website)" \ ---test "searching for GPT gives results" \ --model gpt-4 \ --path microservice ```` @@ -166,8 +159,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ ---description "Given an image, return the image with bounding boxes of all animals (https://pjreddie.com/media/files/yolov3.weights, https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg)" \ ---test "https://images.unsplash.com/photo-1444212477490-ca407925329e contains animals" \ +--description "Given an image, return the image with bounding boxes of all animals (https://pjreddie.com/media/files/yolov3.weights, https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg), Example input: https://images.unsplash.com/photo-1444212477490-ca407925329e" \ --model gpt-4 \ --path microservice ``` @@ -176,8 +168,7 @@ gptdeploy generate \ ### Meme Generator ```bash gptdeploy generate \ ---description "Generate a meme from an image and a caption" \ ---test "Surprised Pikachu: https://media.wired.com/photos/5f87340d114b38fa1f8339f9/master/w_1600%2Cc_limit/Ideas_Surprised_Pikachu_HD.jpg, TOP:When you discovered GPTDeploy" \ +--description "Generate a meme from an image and a caption. Example input: https://media.wired.com/photos/5f87340d114b38fa1f8339f9/master/w_1600%2Cc_limit/Ideas_Surprised_Pikachu_HD.jpg, TOP:When you discovered GPTDeploy" \ --model gpt-4 \ --path microservice ``` @@ -187,7 +178,6 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a word, return a list of rhyming words using the datamuse api" \ ---test "hello" \ --model gpt-4 \ --path microservice ``` @@ -197,7 +187,6 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Generate a word cloud from a given text" \ ---test "Lorem ipsum dolor sit amet, consectetur adipiscing elit." \ --model gpt-4 \ --path microservice ``` @@ -206,8 +195,7 @@ gptdeploy generate \ ### 3d model info ```bash gptdeploy generate \ ---description "Given a 3d object, return vertex count and face count" \ ---test "https://raw.githubusercontent.com/polygonjs/polygonjs-assets/master/models/wolf.obj" \ +--description "Given a 3d object, return vertex count and face count. Example: https://raw.githubusercontent.com/polygonjs/polygonjs-assets/master/models/wolf.obj" \ --model gpt-4 \ --path microservice ``` @@ -216,8 +204,7 @@ gptdeploy generate \ ### Table extraction ```bash gptdeploy generate \ ---description "Given a URL, extract all tables as csv" \ ---test "http://www.ins.tn/statistiques/90" \ +--description "Given a URL, extract all tables as csv. Example: http://www.ins.tn/statistiques/90" \ --model gpt-4 \ --path microservice ``` @@ -226,8 +213,7 @@ gptdeploy generate \ ### Audio to mel spectrogram ```bash gptdeploy generate \ ---description "Create mel spectrograms from audio file" \ ---test "https://cdn.pixabay.com/download/audio/2023/02/28/audio_550d815fa5.mp3" \ +--description "Create mel spectrogram from audio file. Example: https://cdn.pixabay.com/download/audio/2023/02/28/audio_550d815fa5.mp3" \ --model gpt-4 \ --path microservice ``` @@ -237,7 +223,6 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Convert text to speech" \ ---test "Hello, welcome to GPT Deploy!" \ --model gpt-4 \ --path microservice ``` @@ -251,8 +236,7 @@ gptdeploy generate \ ### Heatmap Generator ```bash gptdeploy generate \ ---description "Create a heatmap from an image and a list of relative coordinates" \ ---test "https://images.unsplash.com/photo-1574786198875-49f5d09fe2d2, [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.2, 0.1], [0.7, 0.2], [0.4, 0.2]]" \ +--description "Create a heatmap from an image and a list of relative coordinates. Example input: https://images.unsplash.com/photo-1574786198875-49f5d09fe2d2, [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.2, 0.1], [0.7, 0.2], [0.4, 0.2]]" \ --model gpt-4 \ --path microservice ``` @@ -261,10 +245,8 @@ gptdeploy generate \ ### QR Code Generator ```bash gptdeploy generate \ ---description "Generate QR code from URL" \ ---test "https://www.example.com" \ ---model gpt-4 \ ---path microservice +--description "Generate QR code from URL. Example input: https://www.example.com" \ +--model gpt-4 ``` QR Code Generator @@ -272,8 +254,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ ---description "Visualize the Mandelbrot set with custom parameters" \ ---test "center=-0+1i, zoom=1.0, size=800x800, iterations=1000" \ +--description "Visualize the Mandelbrot set with custom parameters. Example input: center=-0+1i, zoom=1.0, size=800x800, iterations=1000" \ --model gpt-4 \ --path microservice ``` @@ -283,7 +264,7 @@ gptdeploy generate \ ### Markdown to HTML Converter ```bash -gptdeploy generate --description "Convert markdown to HTML" --test "# Hello, welcome to GPT Deploy!" +gptdeploy generate --description "Convert markdown to HTML" ``` Markdown to HTML Converter From da133bf48323db76b5cb70e2e3df51472f647d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Wed, 3 May 2023 16:25:04 +0200 Subject: [PATCH 02/25] =?UTF-8?q?=E2=9C=85=20docs:=20path=20not=20required?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 49 ++++++++++++++++--------------------------------- src/cli.py | 2 +- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index f5eeb24..5b04cf8 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,7 @@ In this section you can get a feeling for the kind of microservices that can be ```bash gptdeploy generate \ --description "The user writes something and gets a related deep compliment." \ ---test "" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Compliment Generator @@ -114,8 +112,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Extract text from a news article URL using Newspaper3k library and generate a summary using gpt. Example input: http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` News Article Example @@ -123,8 +120,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Convert a chemical formula into a 2D chemical structure diagram. Example inputs: C=C, CN=C=O, CCC(=O)O" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Chemical Formula Visualization @@ -132,8 +128,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "create a 2d rendering of a whole 3d object and x,y,z object rotation using trimesh and pyrender.OffscreenRenderer with os.environ['PYOPENGL_PLATFORM'] = 'egl' and freeglut3-dev library - example input: https://graphics.stanford.edu/courses/cs148-10-summer/as3/code/as3/teapot.obj" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` 2D Rendering of 3D Model @@ -141,8 +136,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Generate personalized product recommendations based on user product browsing history and the product categories fashion, electronics and sport. Example: Input: browsing history: prod1(electronics),prod2(fashion),prod3(fashion), output: p4(fashion)" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Product Recommendation @@ -150,8 +144,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a search query, find articles on hacker news using the hacker news api and return a list of (title, author, website_link, first_image_on_the_website)" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ```` Hacker News Search @@ -160,8 +153,7 @@ gptdeploy generate \ gptdeploy generate \ --description "Given an image, return the image with bounding boxes of all animals (https://pjreddie.com/media/files/yolov3.weights, https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg), Example input: https://images.unsplash.com/photo-1444212477490-ca407925329e" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Animal Detector @@ -169,8 +161,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Generate a meme from an image and a caption. Example input: https://media.wired.com/photos/5f87340d114b38fa1f8339f9/master/w_1600%2Cc_limit/Ideas_Surprised_Pikachu_HD.jpg, TOP:When you discovered GPTDeploy" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Meme Generator @@ -178,8 +169,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a word, return a list of rhyming words using the datamuse api" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Rhyme Generator @@ -187,8 +177,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Generate a word cloud from a given text" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Word Cloud Generator @@ -196,8 +185,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a 3d object, return vertex count and face count. Example: https://raw.githubusercontent.com/polygonjs/polygonjs-assets/master/models/wolf.obj" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` 3D Model Info @@ -205,8 +193,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Given a URL, extract all tables as csv. Example: http://www.ins.tn/statistiques/90" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Table Extraction @@ -214,8 +201,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Create mel spectrogram from audio file. Example: https://cdn.pixabay.com/download/audio/2023/02/28/audio_550d815fa5.mp3" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Audio to Mel Spectrogram @@ -223,8 +209,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Convert text to speech" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Text to Speech @@ -237,8 +222,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Create a heatmap from an image and a list of relative coordinates. Example input: https://images.unsplash.com/photo-1574786198875-49f5d09fe2d2, [[0.1, 0.2], [0.3, 0.4], [0.5, 0.6], [0.2, 0.1], [0.7, 0.2], [0.4, 0.2]]" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Heatmap Generator @@ -255,8 +239,7 @@ gptdeploy generate \ ```bash gptdeploy generate \ --description "Visualize the Mandelbrot set with custom parameters. Example input: center=-0+1i, zoom=1.0, size=800x800, iterations=1000" \ ---model gpt-4 \ ---path microservice +--model gpt-4 ``` Mandelbrot Set Visualizer diff --git a/src/cli.py b/src/cli.py index 4ff16e3..e908013 100644 --- a/src/cli.py +++ b/src/cli.py @@ -30,7 +30,7 @@ https://github.com/jina-ai/gptdeploy/issues/new return wrapper def path_param(func): - @click.option('--path', required=True, help='Path to the generated microservice.') + @click.option('--path', default='microservice', help='Path to the generated microservice.') @functools.wraps(func) def wrapper(*args, **kwargs): path = os.path.expanduser(kwargs['path']) From 5a859e0cbbac80e19856bb4ff7c48375734866e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Wed, 3 May 2023 22:24:22 +0200 Subject: [PATCH 03/25] =?UTF-8?q?=E2=AD=90=20fix:=20content=20extraction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- src/options/generate/generator.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5b04cf8..a4ffffb 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,10 @@ gptdeploy generate \ --path ``` To generate your personal microservice two things are required: -- A `description` of the task you want to accomplish. +- A `description` of the task you want to accomplish. (optional) - The `model` you want to use - either `gpt-3.5` or `gpt-4`. `gpt-3.5` is ~10x cheaper, -but will not be able to generate as complex microservices. -- A `path` on the local drive where the microservice will be generated. +but will not be able to generate as complex microservices. (default: largest you have access to) +- A `path` on the local drive where the microservice will be generated. (default: ./microservice) The creation process should take between 5 and 15 minutes. During this time, GPT iteratively builds your microservice until it finds a strategy that make your test scenario pass. diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index d293079..5df2921 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -46,7 +46,7 @@ class Generator: def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break - pattern = fr"\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" + pattern = fr"(?:\*|\*\*| ){file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" matches = re.findall(pattern, plain_text, re.MULTILINE) if matches: return matches[-1].strip() From 037bb22a738547a28c6a131ceb55669d36feb659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 09:30:58 +0200 Subject: [PATCH 04/25] =?UTF-8?q?=F0=9F=93=A6=20fix:=20packages=20without?= =?UTF-8?q?=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/pypi.py | 4 +++- src/options/generate/templates_system.py | 2 +- test/unit/test_api.py | 18 +++++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/apis/pypi.py b/src/apis/pypi.py index 7c57a82..14d3251 100644 --- a/src/apis/pypi.py +++ b/src/apis/pypi.py @@ -35,8 +35,10 @@ def get_latest_package_version(package_name): # Get package versions not older than 2021 valid_versions = [] for v, release_info in releases.items(): + if not release_info: + continue upload_time = datetime.strptime(release_info[0]['upload_time'], '%Y-%m-%dT%H:%M:%S') - if upload_time.year <= 2021: + if upload_time.year <= 2020 or (upload_time.year == 2021 and upload_time.month <= 9): # knowledge cutoff 2021-09 (including september) valid_versions.append(v) v = max(valid_versions, key=version.parse) if valid_versions else None diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 1d31a5b..12bc7f7 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -1,7 +1,7 @@ from src.options.generate.templates_user import not_allowed_docker_string, not_allowed_function_string -template_system_message_base = f'''It is the year 2021. +template_system_message_base = f'''It is September 2021. You are a principal engineer working at Jina - an open source company. You accurately satisfy all of the user's requirements. To be more specific, you help the user to build a microservice with the following requirements: diff --git a/test/unit/test_api.py b/test/unit/test_api.py index 8ebb124..47f9e89 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -38,19 +38,27 @@ def test_filter_packages_list(): def test_precheck_requirements_txt(tmpdir): requirements_content = """\ +# version does not exist but jina and docarray should not be verified jina==1.2.3 docarray==1.2.3 -requests~=2.26.0 +# package that actually exists in that version gtts~=2.2.3 +# package with non-existing version pydub~=123.123.123 +# non-existing package with correct version base64~=3.3.0 +# existing package without version +requests +# another existing package without version +streamlit """ requirements_clean = """\ -jina==1.2.3 -docarray==1.2.3 -requests~=2.26.0 +jina==111.222.333 +docarray==111.222.333 gtts~=2.2.3 -pydub~=0.25.1""" +pydub~=0.25.1 +requests~=2.26.0 +streamlit~=1.3.1""" requirements_txt_path = os.path.join(tmpdir, "requirements.txt") with open(requirements_txt_path, "w", encoding="utf-8") as f: f.write(requirements_content) From 8d87af091d7e3c64af3ac836bbf17b6b3d944b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 09:36:25 +0200 Subject: [PATCH 05/25] =?UTF-8?q?=F0=9F=93=A6=20fix:=20packages=20without?= =?UTF-8?q?=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/unit/test_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/test_api.py b/test/unit/test_api.py index 47f9e89..f94782c 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -39,8 +39,8 @@ def test_filter_packages_list(): def test_precheck_requirements_txt(tmpdir): requirements_content = """\ # version does not exist but jina and docarray should not be verified -jina==1.2.3 -docarray==1.2.3 +jina==111.222.333 +docarray==111.222.333 # package that actually exists in that version gtts~=2.2.3 # package with non-existing version @@ -58,7 +58,7 @@ docarray==111.222.333 gtts~=2.2.3 pydub~=0.25.1 requests~=2.26.0 -streamlit~=1.3.1""" +streamlit~=0.89.0""" requirements_txt_path = os.path.join(tmpdir, "requirements.txt") with open(requirements_txt_path, "w", encoding="utf-8") as f: f.write(requirements_content) From 028cf1897e477ae254365a896cd02f3fe5f1ec9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 10:23:36 +0200 Subject: [PATCH 06/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push-docker-base-image.yml | 30 +++++++++++++++++++ src/options/generate/static_files/__init__.py | 0 .../static_files/base_image/Dockerfile | 7 +++++ .../static_files/microservice/Dockerfile | 4 +-- 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/push-docker-base-image.yml delete mode 100644 src/options/generate/static_files/__init__.py create mode 100644 src/options/generate/static_files/base_image/Dockerfile diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml new file mode 100644 index 0000000..927c939 --- /dev/null +++ b/.github/workflows/push-docker-base-image.yml @@ -0,0 +1,30 @@ +name: Push docker base image + +on: + push: + branches: + - main + pull_request: + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} + password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} + + - name: Build and Push Docker Image + uses: docker/build-push-action@v2 + with: + context: src/options/generate/static_files/base_image + push: true + tags: jina/gpt-dev:latest \ No newline at end of file diff --git a/src/options/generate/static_files/__init__.py b/src/options/generate/static_files/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/options/generate/static_files/base_image/Dockerfile b/src/options/generate/static_files/base_image/Dockerfile new file mode 100644 index 0000000..8373c07 --- /dev/null +++ b/src/options/generate/static_files/base_image/Dockerfile @@ -0,0 +1,7 @@ +FROM jinaai/jina:3.14.1-py39-standard + +# update pip +RUN pip install --upgrade pip + +# install media dependencies +RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev {{apt_get_packages}} \ No newline at end of file diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 681ad0c..5d4e97e 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,8 +1,8 @@ -FROM jinaai/jina:3.14.1-py39-standard +FROM jinaai/gpt-dev:latest # update pip RUN pip install --upgrade pip -RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev {{apt_get_packages}} && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get install --no-install-recommends -y {{apt_get_packages}} ## install requirements for the executor COPY requirements.txt . From f049ab61346d6e46b62e5df899493370bca5d541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 10:30:36 +0200 Subject: [PATCH 07/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/static_files/base_image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/static_files/base_image/Dockerfile b/src/options/generate/static_files/base_image/Dockerfile index 8373c07..33924d6 100644 --- a/src/options/generate/static_files/base_image/Dockerfile +++ b/src/options/generate/static_files/base_image/Dockerfile @@ -4,4 +4,4 @@ FROM jinaai/jina:3.14.1-py39-standard RUN pip install --upgrade pip # install media dependencies -RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev {{apt_get_packages}} \ No newline at end of file +RUN apt-get update && apt-get install --no-install-recommends -y ffmpeg build-essential pkg-config libpoppler-cpp-dev \ No newline at end of file From 87e5e6a99d4dad5f768c0a438ac8e7ff4cdc6b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 10:33:28 +0200 Subject: [PATCH 08/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push-docker-base-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml index 927c939..a7d4c14 100644 --- a/.github/workflows/push-docker-base-image.yml +++ b/.github/workflows/push-docker-base-image.yml @@ -27,4 +27,4 @@ jobs: with: context: src/options/generate/static_files/base_image push: true - tags: jina/gpt-dev:latest \ No newline at end of file + tags: jinaai/gpt-dev:latest \ No newline at end of file From e44459e13f2cddf23e93c7fecb55363eefb65d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:00:16 +0200 Subject: [PATCH 09/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push-docker-base-image.yml | 34 +++++++++++++++++++- src/constants.py | 3 ++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml index a7d4c14..48fcb2c 100644 --- a/.github/workflows/push-docker-base-image.yml +++ b/.github/workflows/push-docker-base-image.yml @@ -13,18 +13,50 @@ jobs: - name: Checkout Repository uses: actions/checkout@v2 + - name: Check if code relevant to executor has changed + uses: dorny/paths-filter@v2 + id: check + with: + filters: | + changed: + - src/options/generate/static_files/base_image/** + + - name: Get base image tag + if: steps.check.outputs.changed == 'true' + shell: bash + run: | + FILE='scr/constants.py' + VERSION=$(sed -n '/DOCKER_BASE_IMAGE_VERSION =/p' $FILE | cut -d \' -f2) + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Check that tag isn't used already for the docker base image + if: steps.check.outputs.changed == 'true' + env: + VERSION: ${{ env.VERSION }} + shell: bash + run: | + if jina hub pull jinahub+docker://jinaai/gpt-dev/$VERSION; then + echo "Executor version/tag is used already. Please update the tag" + exit 1 + else + echo "Executor version/tag isn't used already, continue to build..." + fi + - name: Set up Docker Buildx + if: steps.check.outputs.changed == 'true' uses: docker/setup-buildx-action@v1 - name: Login to Docker Hub + if: steps.check.outputs.changed == 'true' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} - name: Build and Push Docker Image + if: steps.check.outputs.changed == 'true' uses: docker/build-push-action@v2 with: context: src/options/generate/static_files/base_image push: true - tags: jinaai/gpt-dev:latest \ No newline at end of file + tags: jinaai/gpt-dev:$VERSION \ No newline at end of file diff --git a/src/constants.py b/src/constants.py index 08833b4..ef527c3 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,3 +1,5 @@ +DOCKER_BASE_IMAGE_VERSION = '0.0.1' + EXECUTOR_FILE_NAME = '__init__.py' IMPLEMENTATION_FILE_NAME = 'microservice.py' TEST_EXECUTOR_FILE_NAME = 'test_microservice.py' @@ -51,3 +53,4 @@ LANGUAGE_PACKAGES = [ 'pattern', 'polyglot', 'pytorch-transformers', 'rasa', 'sentence-transformers', 'spacy', 'stanza', 'summarizer', 'sumy', 'textblob', 'textstat', 'transformers' ] + From 223defe9fb26358b48be3b29d965faa72b9ddc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:01:45 +0200 Subject: [PATCH 10/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push-docker-base-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml index 48fcb2c..41bbefb 100644 --- a/.github/workflows/push-docker-base-image.yml +++ b/.github/workflows/push-docker-base-image.yml @@ -25,7 +25,7 @@ jobs: if: steps.check.outputs.changed == 'true' shell: bash run: | - FILE='scr/constants.py' + FILE='src/constants.py' VERSION=$(sed -n '/DOCKER_BASE_IMAGE_VERSION =/p' $FILE | cut -d \' -f2) echo "VERSION=$VERSION" >> $GITHUB_ENV From bb7878bdb3f818ccac5ade3b284db5b132f4fe23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:06:00 +0200 Subject: [PATCH 11/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- .github/workflows/push-docker-base-image.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a56cc2c..025dbb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: workflow_dispatch: - pull_request: +# pull_request: jobs: test_cognitive_level: diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml index 41bbefb..0c52390 100644 --- a/.github/workflows/push-docker-base-image.yml +++ b/.github/workflows/push-docker-base-image.yml @@ -35,7 +35,7 @@ jobs: VERSION: ${{ env.VERSION }} shell: bash run: | - if jina hub pull jinahub+docker://jinaai/gpt-dev/$VERSION; then + if docker pull jinaai/gpt-dev:$VERSION; then echo "Executor version/tag is used already. Please update the tag" exit 1 else From 0854e9441c41bfc11bcec0b8f027a11553a7722c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:13:43 +0200 Subject: [PATCH 12/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/push-docker-base-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml index 0c52390..8c22bd7 100644 --- a/.github/workflows/push-docker-base-image.yml +++ b/.github/workflows/push-docker-base-image.yml @@ -59,4 +59,4 @@ jobs: with: context: src/options/generate/static_files/base_image push: true - tags: jinaai/gpt-dev:$VERSION \ No newline at end of file + tags: jinaai/gpt-dev:${{ env.VERSION }} \ No newline at end of file From 3204d893e18ac1b364650edd94d57830004cc285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:23:00 +0200 Subject: [PATCH 13/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 54 ++++++++++++++++ .github/workflows/push-docker-base-image.yml | 62 ------------------- src/constants.py | 2 +- src/options/generate/generator.py | 8 +-- .../static_files/microservice/Dockerfile | 6 +- src/options/generate/templates_user.py | 6 +- 6 files changed, 64 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/push-docker-base-image.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 025dbb0..621d825 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,57 @@ jobs: SCENEX_API_KEY: ${{ secrets.SCENEX_API_KEY }} WHISPER_API_KEY: ${{ secrets.WHISPER_API_KEY }} + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + + - name: Check if code relevant to executor has changed + uses: dorny/paths-filter@v2 + id: check + with: + filters: | + changed: + - src/options/generate/static_files/base_image/** + + - name: Get base image tag + if: steps.check.outputs.changed == 'true' + shell: bash + run: | + FILE='src/constants.py' + VERSION=$(sed -n '/DOCKER_BASE_IMAGE_VERSION =/p' $FILE | cut -d \' -f2) + echo "VERSION=$VERSION" >> $GITHUB_ENV + + - name: Check that tag isn't used already for the docker base image + if: steps.check.outputs.changed == 'true' + env: + VERSION: ${{ env.VERSION }} + shell: bash + run: | + if docker pull jinaai/gpt-dev:$VERSION; then + echo "Executor version/tag is used already. Please update the tag" + exit 1 + else + echo "Executor version/tag isn't used already, continue to build..." + fi + + - name: Set up Docker Buildx + if: steps.check.outputs.changed == 'true' + uses: docker/setup-buildx-action@v1 + + - name: Login to Docker Hub + if: steps.check.outputs.changed == 'true' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} + password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} + + - name: Build and Push Docker Image + if: steps.check.outputs.changed == 'true' + uses: docker/build-push-action@v2 + with: + context: src/options/generate/static_files/base_image + push: true + tags: jinaai/gpt-dev:${{ env.VERSION }} + diff --git a/.github/workflows/push-docker-base-image.yml b/.github/workflows/push-docker-base-image.yml deleted file mode 100644 index 8c22bd7..0000000 --- a/.github/workflows/push-docker-base-image.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Push docker base image - -on: - push: - branches: - - main - pull_request: - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v2 - - - name: Check if code relevant to executor has changed - uses: dorny/paths-filter@v2 - id: check - with: - filters: | - changed: - - src/options/generate/static_files/base_image/** - - - name: Get base image tag - if: steps.check.outputs.changed == 'true' - shell: bash - run: | - FILE='src/constants.py' - VERSION=$(sed -n '/DOCKER_BASE_IMAGE_VERSION =/p' $FILE | cut -d \' -f2) - echo "VERSION=$VERSION" >> $GITHUB_ENV - - - name: Check that tag isn't used already for the docker base image - if: steps.check.outputs.changed == 'true' - env: - VERSION: ${{ env.VERSION }} - shell: bash - run: | - if docker pull jinaai/gpt-dev:$VERSION; then - echo "Executor version/tag is used already. Please update the tag" - exit 1 - else - echo "Executor version/tag isn't used already, continue to build..." - fi - - - name: Set up Docker Buildx - if: steps.check.outputs.changed == 'true' - uses: docker/setup-buildx-action@v1 - - - name: Login to Docker Hub - if: steps.check.outputs.changed == 'true' - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} - password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} - - - name: Build and Push Docker Image - if: steps.check.outputs.changed == 'true' - uses: docker/build-push-action@v2 - with: - context: src/options/generate/static_files/base_image - push: true - tags: jinaai/gpt-dev:${{ env.VERSION }} \ No newline at end of file diff --git a/src/constants.py b/src/constants.py index ef527c3..4ccd58e 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,4 +1,4 @@ -DOCKER_BASE_IMAGE_VERSION = '0.0.1' +DOCKER_BASE_IMAGE_VERSION = '0.0.2' EXECUTOR_FILE_NAME = '__init__.py' IMPLEMENTATION_FILE_NAME = 'microservice.py' diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 5df2921..7af4515 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -17,7 +17,7 @@ from src.apis.pypi import is_package_on_pypi, get_latest_package_version, clean_ from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \ BLACKLISTED_PACKAGES, EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, IMPLEMENTATION_FILE_NAME, \ - IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES + IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES, DOCKER_BASE_IMAGE_VERSION from src.options.generate.templates_system import system_task_iteration, system_task_introduction, system_test_iteration from src.options.generate.templates_user import template_generate_microservice_name, \ template_generate_possible_packages, \ @@ -211,7 +211,7 @@ metas: encoding='utf-8') as f: docker_file_template_lines = f.readlines() docker_file_template_lines = [ - line.replace('{{apt_get_packages}}', '') + line.replace('{{APT_GET_PACKAGES}}', '').replace('{{DOCKER_BASE_IMAGE_VERSION}}', DOCKER_BASE_IMAGE_VERSION) for line in docker_file_template_lines ] docker_file_content = '\n'.join(docker_file_template_lines) @@ -231,8 +231,8 @@ metas: packages = ' '.join(json.loads(json_string)['packages']) docker_file_template = self.read_docker_template() - return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format( - apt_get_packages=packages)} + return {DOCKER_FILE_NAME: docker_file_template.replace('{{APT_GET_PACKAGES}}', '{APT_GET_PACKAGES}').replace('{{DOCKER_BASE_IMAGE_VERSION}}', DOCKER_BASE_IMAGE_VERSION).format( + APT_GET_PACKAGES=packages)} def parse_result_fn_requirements(self, content_raw: str): content_parsed = self.extract_content_from_result(content_raw, 'requirements.txt', match_single_block=True) diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 5d4e97e..176f5f6 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,8 +1,6 @@ -FROM jinaai/gpt-dev:latest -# update pip -RUN pip install --upgrade pip +FROM jinaai/gpt-dev:{{DOCKER_BASE_IMAGE_VERSION}} -RUN apt-get install --no-install-recommends -y {{apt_get_packages}} +RUN apt-get install --no-install-recommends -y {{APT_GET_PACKAGES}} ## install requirements for the executor COPY requirements.txt . diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 12f34e7..d4be807 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -170,7 +170,7 @@ template_generate_apt_get_install = PromptTemplate.from_template( {docker_file_wrapped} -Name all packages which need to be installed via `apt-get install` in above Dockerfile (`{{apt_get_packages}}`) for the following requirements.txt file: +Name all packages which need to be installed via `apt-get install` in above Dockerfile (`{{APT_GET_PACKAGES}}`) for the following requirements.txt file: {requirements_file_wrapped} @@ -178,7 +178,7 @@ Note that you must not list apt-get packages that are already installed in the D Note that openai does not require any apt-get packages. Note that you are only allowed to list packages where you are highly confident that they are really needed. Note that you can assume that the standard python packages are already installed. -Output the packages that need to me placed at {{apt_get_packages}} as json in the following format: +Output the packages that need to me placed at {{APT_GET_PACKAGES}} as json in the following format: **apt-get-packages.json** ```json {{"packages": ["", ""]}} @@ -275,7 +275,7 @@ Here is the summary of the error that occurred: {summarized_error} To solve this error, you should determine the list of packages that need to be installed via `apt-get install` in the Dockerfile. -Output the apt-get packages that need to be placed at {{apt_get_packages}} as json in the following format: +Output the apt-get packages that need to be placed at {{APT_GET_PACKAGES}} as json in the following format: **apt-get-packages.json** ```json {{"packages": ["", ""]}} From 41002443a1379736387ea7f4ae97ca81f986a9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:23:16 +0200 Subject: [PATCH 14/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 621d825..e7749a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: workflow_dispatch: -# pull_request: + pull_request: jobs: test_cognitive_level: From 517793dde4ba3d9ccf3ab2bf04b6c1db06ad31df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:24:54 +0200 Subject: [PATCH 15/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7749a3..ec2ce7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: SCENEX_API_KEY: ${{ secrets.SCENEX_API_KEY }} WHISPER_API_KEY: ${{ secrets.WHISPER_API_KEY }} - build-and-push: + base-image-push: runs-on: ubuntu-latest steps: - name: Checkout Repository From e28ee71e03513b099778e3eb54dcedcd0020c460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:26:08 +0200 Subject: [PATCH 16/25] =?UTF-8?q?=F0=9F=90=8B=20feat:=20docker=20base=20im?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants.py b/src/constants.py index 4ccd58e..f2ac232 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,4 +1,4 @@ -DOCKER_BASE_IMAGE_VERSION = '0.0.2' +DOCKER_BASE_IMAGE_VERSION = '0.0.3' EXECUTOR_FILE_NAME = '__init__.py' IMPLEMENTATION_FILE_NAME = 'microservice.py' From ea5a4dd9b7568114dc0af628508bee5e83befd31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 11:44:44 +0200 Subject: [PATCH 17/25] =?UTF-8?q?=F0=9F=97=84=20fix:=20dont=20output=20oth?= =?UTF-8?q?er=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 12f34e7..fa2ab9c 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -280,18 +280,14 @@ Output the apt-get packages that need to be placed at {{apt_get_packages}} as js ```json {{"packages": ["", ""]}} ``` -Example for the following requirements.txt file: -**requirements.txt** -``` -numpy==1.19.5 -fitz -``` -The output would be: +Example: +Error is about missing package `libgl1-mesa-glx`. +The output is: **apt-get-packages.json** ```json -{{"packages": []}} +{{"packages": [libgl1-mesa-glx]}} ``` -Note that you must not output any other files. Only output the apt-get-packages.json file. +Note that you must not output the content of any other files. Only output the apt-get-packages.json file. ''' ) From 36cb8169b4e3ea16c536302e3276c94c993cfbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 12:13:50 +0200 Subject: [PATCH 18/25] =?UTF-8?q?=F0=9F=97=84=20fix:=20dont=20output=20oth?= =?UTF-8?q?er=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index fa2ab9c..267e6cf 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -287,7 +287,8 @@ The output is: ```json {{"packages": [libgl1-mesa-glx]}} ``` -Note that you must not output the content of any other files. Only output the apt-get-packages.json file. +Note that you must not output the content of any other files like the Dockerfile or requirements.txt. +Only output the apt-get-packages.json file. ''' ) From 179b73d5e553fdd14f6486cfe348b91616d849d1 Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 4 May 2023 14:17:36 +0200 Subject: [PATCH 19/25] :bug: fix: improve error message --- src/apis/jina_cloud.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/apis/jina_cloud.py b/src/apis/jina_cloud.py index 9684f9a..3255ebc 100644 --- a/src/apis/jina_cloud.py +++ b/src/apis/jina_cloud.py @@ -164,10 +164,10 @@ def deploy_on_jcloud(executor_name, microservice_path): time.sleep(5) except SystemExit as e: raise SystemExit(f''' -Looks like your free credits ran out. -Please add payment information to your account and try again. -Visit https://cloud.jina.ai/ - ''') from e +Looks like something went wrong in the generation and we didn't catch it. +Please, create an issue here: https://github.com/jina-ai/gptdeploy/issues/new/choose +And add details on the microservice you are trying to create. +''') from e if i == 2: raise Exception(''' Could not deploy on Jina Cloud. From af2fa8ea7a0e8055fe240d3824fa105292c81016 Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 4 May 2023 14:18:29 +0200 Subject: [PATCH 20/25] :bug: fix: improve error message --- src/apis/jina_cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apis/jina_cloud.py b/src/apis/jina_cloud.py index 3255ebc..8628ae4 100644 --- a/src/apis/jina_cloud.py +++ b/src/apis/jina_cloud.py @@ -166,7 +166,8 @@ def deploy_on_jcloud(executor_name, microservice_path): raise SystemExit(f''' Looks like something went wrong in the generation and we didn't catch it. Please, create an issue here: https://github.com/jina-ai/gptdeploy/issues/new/choose -And add details on the microservice you are trying to create. +nd add details on the microservice you are trying to create. +In the meantime, you can upgrade your GPT Deploy version, if not using latest, and try again. ''') from e if i == 2: raise Exception(''' From e65f958cededb73a1078102c1647571ef3933b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=B6nicke?= Date: Thu, 4 May 2023 14:29:01 +0200 Subject: [PATCH 21/25] =?UTF-8?q?=F0=9F=96=8A=20fix:=20requirements=20pars?= =?UTF-8?q?ing=20error=20(#85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/pypi.py | 9 ++++----- test/unit/test_api.py | 3 +++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/apis/pypi.py b/src/apis/pypi.py index 14d3251..6f1fc5c 100644 --- a/src/apis/pypi.py +++ b/src/apis/pypi.py @@ -63,13 +63,12 @@ def clean_requirements_txt(previous_microservice_path): continue split = re.split(r'==|>=|<=|>|<|~=', line) - if len(split) == 1: + if len(split) == 1 or len(split) > 2: version = None package_name = split[0] - elif len(split) == 2: - package_name, version = split else: - raise ValueError(f'Could not parse line {line} in requirements.txt') + package_name, version = split + # Keep lines with jina, docarray, openai, pytest unchanged if package_name in {'jina', 'docarray', 'openai', 'pytest'}: @@ -79,7 +78,7 @@ def clean_requirements_txt(previous_microservice_path): if version is None or not is_package_on_pypi(package_name, version): latest_version = get_latest_package_version(package_name) if latest_version is None: - raise ValueError(f'Package {package_name} not found on PyPI') + continue updated_requirements.append(f'{package_name}~={latest_version}') else: updated_requirements.append(line) diff --git a/test/unit/test_api.py b/test/unit/test_api.py index f94782c..c3c1962 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -47,6 +47,8 @@ gtts~=2.2.3 pydub~=123.123.123 # non-existing package with correct version base64~=3.3.0 +# not parsable version +pdfminer.six>=20201018,<20211018 # existing package without version requests # another existing package without version @@ -57,6 +59,7 @@ jina==111.222.333 docarray==111.222.333 gtts~=2.2.3 pydub~=0.25.1 +pdfminer.six~=20201018 requests~=2.26.0 streamlit~=0.89.0""" requirements_txt_path = os.path.join(tmpdir, "requirements.txt") From 8aadb2127704d63489dfd3ac3106425cde52359b Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Thu, 4 May 2023 14:50:35 +0200 Subject: [PATCH 22/25] :bug: fix: improve error message --- src/apis/jina_cloud.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/apis/jina_cloud.py b/src/apis/jina_cloud.py index 8628ae4..8c4b780 100644 --- a/src/apis/jina_cloud.py +++ b/src/apis/jina_cloud.py @@ -164,10 +164,11 @@ def deploy_on_jcloud(executor_name, microservice_path): time.sleep(5) except SystemExit as e: raise SystemExit(f''' -Looks like something went wrong in the generation and we didn't catch it. -Please, create an issue here: https://github.com/jina-ai/gptdeploy/issues/new/choose -nd add details on the microservice you are trying to create. -In the meantime, you can upgrade your GPT Deploy version, if not using latest, and try again. +Looks like you either ran out of credits or something went wrong in the generation and we didn't catch it. +To check if you ran out of credits, please go to https://cloud.jina.ai. +If you have credits left, please create an issue here https://github.com/jina-ai/gptdeploy/issues/new/choose +and add details on the microservice you are trying to create. +In that case, you can upgrade your GPT Deploy version, if not using latest, and try again. ''') from e if i == 2: raise Exception(''' From ca8494c9da61292907257394901e2af40eab01b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 15:48:23 +0200 Subject: [PATCH 23/25] =?UTF-8?q?=E2=9C=8F=20test:=20response=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 3 +- test/unit/test_response_parsing.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 test/unit/test_response_parsing.py diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 5df2921..da1fee3 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -44,7 +44,8 @@ class Generator: self.microservice_specification = TaskSpecification(task=task_description, test=None) self.microservice_root_path = path - def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): + @staticmethod + def extract_content_from_result(plain_text, file_name, match_single_block=False, can_contain_code_block=True): optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break pattern = fr"(?:\*|\*\*| ){file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" matches = re.findall(pattern, plain_text, re.MULTILINE) diff --git a/test/unit/test_response_parsing.py b/test/unit/test_response_parsing.py new file mode 100644 index 0000000..6c326ce --- /dev/null +++ b/test/unit/test_response_parsing.py @@ -0,0 +1,44 @@ +import pytest + +from src.options.generate.generator import Generator + +def create_code_block(with_backticks, asterisks, with_highlight_info, file_name, start_inline, content): + code_block = f''' +{{ + "content": "{content}", +}} +''' + if with_highlight_info: + high_light_info = 'json' + else: + high_light_info = '' + if with_backticks: + code_block = f'```{high_light_info}\n{code_block}\n```' + if file_name: + code_block = f'{asterisks}{file_name}{asterisks}\n{code_block}' + if start_inline: + code_block = f'This is your file: {code_block}' + return code_block + + +@pytest.mark.parametrize( + 'plain_text, expected1, expected2', + [ + ( + f"""{create_code_block(True, '', False, 'test1.json', True, 'content1')}\n{create_code_block(True, '', True, '', False, 'content2')}""", + f'{create_code_block(False, "", False, "", False, content="content1")}'.strip(), + '' + ), + ( + f"""{create_code_block(True, '', True, '', False, 'content2')}""", + '', + f'{create_code_block(False, "", False, "", False, content="content2")}'.strip() + ) + ] +) +def test_extract_content_from_result(plain_text, expected1, expected2): + parsed_result1 = Generator.extract_content_from_result(plain_text, 'test1.json', False, True) + assert parsed_result1 == expected1 + parsed_result2 = Generator.extract_content_from_result(plain_text, 'test100.json', True, False) + assert parsed_result2 == expected2 + From 6a67882e048d050167efce78a2cad4301760a9a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 15:51:39 +0200 Subject: [PATCH 24/25] =?UTF-8?q?=F0=9F=90=8B=20feat=20docker=20base=20ima?= =?UTF-8?q?ge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/static_files/base_image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/static_files/base_image/Dockerfile b/src/options/generate/static_files/base_image/Dockerfile index 33924d6..edd0c73 100644 --- a/src/options/generate/static_files/base_image/Dockerfile +++ b/src/options/generate/static_files/base_image/Dockerfile @@ -1,4 +1,4 @@ -FROM jinaai/jina:3.14.1-py39-standard +FROM jinaai/jina:3.15.1-dev14-py39-standard # update pip RUN pip install --upgrade pip From 5a1f7ed144e0fc30aa4089bfbc11913247af3b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 4 May 2023 15:53:38 +0200 Subject: [PATCH 25/25] =?UTF-8?q?=F0=9F=90=8B=20feat=20docker=20base=20ima?= =?UTF-8?q?ge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants.py b/src/constants.py index f2ac232..7b7f6d0 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,4 +1,4 @@ -DOCKER_BASE_IMAGE_VERSION = '0.0.3' +DOCKER_BASE_IMAGE_VERSION = '0.0.4' EXECUTOR_FILE_NAME = '__init__.py' IMPLEMENTATION_FILE_NAME = 'microservice.py'