From 81b67851da902b654757e79065d26f9c1ef04831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sat, 29 Apr 2023 16:51:57 +0200 Subject: [PATCH 01/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_generator.py b/test/test_generator.py index 098dc18..de7c866 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -25,7 +25,7 @@ def test_generation_level_0(tmpdir): ) generator.generate() -@pytest.mark.skip(reason="not possible") + def test_generation_level_1(tmpdir): """ Requirements: @@ -53,7 +53,7 @@ Enjoy your day! #variety' ) generator.generate() -@pytest.mark.skip(reason="not possible") + def test_generation_level_2(tmpdir): """ Requirements: From 271782a1cfe58441741927f7d51327f8b73bd83f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 01:58:58 +0200 Subject: [PATCH 02/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 23 ++++++++---- src/options/generate/templates_system.py | 48 ++++++++---------------- src/options/generate/templates_user.py | 4 +- test/test_generator.py | 5 ++- 4 files changed, 36 insertions(+), 44 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index f4170bb..a153c9d 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -44,7 +44,7 @@ class Generator: def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break - pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" + pattern = fr"^\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" match = re.search(pattern, plain_text, re.MULTILINE) if match: return match.group(1).strip() @@ -422,8 +422,16 @@ gptdeploy deploy --path {self.microservice_root_path} if not original_task: self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') - self.refine_requirements(pm, system_task_iteration, 'task') - self.refine_requirements(pm, system_test_iteration, 'test') + messages = [SystemMessage(content=system_task_introduction + system_task_iteration)] + self.refine_requirements(pm, messages, 'task', '') + messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration) + self.refine_requirements( + pm, + messages, + 'test', + '''Note that the test scenario must not contain information that was already mentioned in the microservice description. +Note that if the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.''' + ) break except self.TaskRefinementException as e: @@ -437,11 +445,8 @@ Test scenario: {self.microservice_specification.test} ''') - def refine_requirements(self, pm, template_init, refinement_type): + def refine_requirements(self, pm, messages, refinement_type, custom_suffix): user_input = self.microservice_specification.task - messages = [ - SystemMessage(content=system_task_introduction + template_init), - ] num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) @@ -449,7 +454,8 @@ Test scenario: agent_response_raw = conversation.chat( template_refinement.format( user_input=user_input, - _optional_test=' test' if refinement_type == 'test' else '' + _optional_test=' test' if refinement_type == 'test' else '', + custom_suffix=custom_suffix, ), role='user' ) @@ -457,6 +463,7 @@ Test scenario: agent_question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False) if final: + messages.append(AIMessage(content=final)) setattr(self.microservice_specification, refinement_type, final) break elif agent_question: diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 41d1cba..7fd4685 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -96,7 +96,7 @@ system_test_iteration = f''' The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: -a) If unit test requires an example input file as input: +a) If the input of the unit test requires an example input file or e.g. a certain textual description that is not mentioned before: In this case you must ask the client to provide the example input file as URL. You must not accept files that are not URLs. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): @@ -107,60 +107,42 @@ Your response must exactly match the following block code format (double asteris ``` If you did a, you must not do b. -b) Any strings, ints, or bools can be used as input for the unit test. +b) If the input can be determined from the previous messages: In this case you must describe the unit test verbally. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **final.txt** ```text - +input: "" +assertion of output: "" ``` If you did b, you must not do a. -Example 1: -Client: -**client-response.txt** -``` -given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22 -``` -PM: +Example for: "given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22": **final.txt** ```text -The test takes the city "Berlin" as input and asserts that the weather report for the next 5 days exists in the response. +input: "Berlin" +assertion of output: "contains weather report for the next 5 days" ``` -Example 2: -Client: -**client-response.txt** -``` -The user inserts a png and gets an svg as response. -``` -PM: +Example for "The user inserts a png and gets an svg as response.": **prompt.txt** ```text Please provide a png example input file as url. ``` -Client: -**client-response.txt** -``` -https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png -``` -PM: + +Example for "The user inserts a png like https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png and gets an svg as response.": **final.txt** ```text -The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg. +input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png" +assertion of output: "is an svg" ``` -Example 3: -Client: -**client-response.txt** -``` -The microservice takes nothing as input and returns the current time. -``` -PM: +Example for "The microservice takes nothing as input and returns the current time.": **final.txt** ```text -The test takes nothing as input and asserts that the output is a string. +input: "nothing" +assertion of output: "is a string" ``` ''' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9e078c3..9818865 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -368,10 +368,12 @@ Or write the summarized microservice{_optional_test} description like this: **final.txt** ```text -``` +``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. +Note that the last sequence of characters in your response must be ``` (triple backtick). Note that prompt.txt must not only contain one question. Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary. +{custom_suffix} ''' ) diff --git a/test/test_generator.py b/test/test_generator.py index de7c866..298cd1c 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -40,11 +40,12 @@ def test_generation_level_1(tmpdir): generator = Generator( ''' Input is a tweet that might contain passive aggressive language like: -'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 But hey, at least SOMEONE's enjoying their lunch. #officelife' +'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 +But hey, at least SOMEONE's enjoying their lunch. #officelife' The output is a tweet that is not passive aggressive like: 'Hi coworker, I hope you're having an amazing day! -Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. +Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. 😜 If you're up for trying different lunch options, that could be a fun way to mix things up. Enjoy your day! #variety' ''', From 4b9e9a92ee86f9189ba6dd768d49e50650a62145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 17:40:08 +0200 Subject: [PATCH 03/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_system.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 7fd4685..fc1669f 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -110,20 +110,18 @@ If you did a, you must not do b. b) If the input can be determined from the previous messages: In this case you must describe the unit test verbally. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): - **final.txt** ```text input: "" -assertion of output: "" +weak assertion of output: "" ``` - If you did b, you must not do a. Example for: "given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22": **final.txt** ```text input: "Berlin" -assertion of output: "contains weather report for the next 5 days" +weak assertion of output: "contains weather report for the next 5 days" ``` Example for "The user inserts a png and gets an svg as response.": @@ -136,13 +134,13 @@ Example for "The user inserts a png like https://aquasecurity.github.io/kube-ben **final.txt** ```text input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png" -assertion of output: "is an svg" +weak assertion of output: "is an svg" ``` Example for "The microservice takes nothing as input and returns the current time.": **final.txt** ```text input: "nothing" -assertion of output: "is a string" +weak assertion of output: "is a string" ``` ''' From c7424211d06fc7ed9784add0de40d961b6eb6ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 17:41:21 +0200 Subject: [PATCH 04/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_generator.py b/test/test_generator.py index 298cd1c..7a03112 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -67,7 +67,7 @@ def test_generation_level_2(tmpdir): """ os.environ['VERBOSE'] = 'true' generator = Generator( - "The input is a PDF like https://www.africau.edu/images/default/sample.pdf and the output the summarized text.", + "The input is a PDF like https://www.africau.edu/images/default/sample.pdf and the output the summarized text (50 words).", str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) From e4fc2e67850ef1487951974924a23eb0a8a1b248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 17:52:47 +0200 Subject: [PATCH 05/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/gpt.py | 9 +++++---- src/options/generate/generator.py | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index c386f6a..4111386 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -108,14 +108,14 @@ class _GPTConversation: self.messages = messages self.print_stream = print_stream self.print_costs = print_costs - for message in messages: + for i, message in enumerate(messages): if os.environ['VERBOSE'].lower() == 'true': if isinstance(message, SystemMessage): - print_colored('system - prompt', message.content, 'magenta') + print_colored(f'({i}) system - prompt', message.content, 'magenta') elif isinstance(message, HumanMessage): - print_colored('user - prompt', message.content, 'blue') + print_colored(f'({i}) user - prompt', message.content, 'blue') elif isinstance(message, AIMessage): - print_colored('assistant - prompt', message.content, 'green') + print_colored(f'({i}) assistant - prompt', message.content, 'green') def chat(self, prompt: str, role: str = 'user'): MassageClass = HumanMessage if role == 'user' else SystemMessage @@ -126,6 +126,7 @@ class _GPTConversation: print_colored(role, prompt, color) if self.print_stream: print_colored('assistant', '', 'green', end='') + print('thinking...') for i in range(10): try: response = self._chat(self.messages) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index a153c9d..d7577e7 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -430,7 +430,7 @@ gptdeploy deploy --path {self.microservice_root_path} messages, 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. -Note that if the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.''' +Note that the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.''' ) break except self.TaskRefinementException as e: @@ -450,7 +450,6 @@ Test scenario: num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) - print('thinking...') agent_response_raw = conversation.chat( template_refinement.format( user_input=user_input, From cb738c93329ca5f7d5d359ae7ea2211bde5a496b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 18:10:05 +0200 Subject: [PATCH 06/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/gpt.py | 7 ++++--- test/test_generator.py | 14 +++----------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index 4111386..16a53af 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -108,6 +108,9 @@ class _GPTConversation: self.messages = messages self.print_stream = print_stream self.print_costs = print_costs + + + def print_messages(self, messages): for i, message in enumerate(messages): if os.environ['VERBOSE'].lower() == 'true': if isinstance(message, SystemMessage): @@ -121,9 +124,7 @@ class _GPTConversation: MassageClass = HumanMessage if role == 'user' else SystemMessage chat_message = MassageClass(content=prompt) self.messages.append(chat_message) - if os.environ['VERBOSE'].lower() == 'true': - color = 'blue' if role == 'user' else 'magenta' - print_colored(role, prompt, color) + self.print_messages(self.messages) if self.print_stream: print_colored('assistant', '', 'green', end='') print('thinking...') diff --git a/test/test_generator.py b/test/test_generator.py index 7a03112..f8a021c 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -38,17 +38,9 @@ def test_generation_level_1(tmpdir): """ os.environ['VERBOSE'] = 'true' generator = Generator( - ''' -Input is a tweet that might contain passive aggressive language like: -'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 -But hey, at least SOMEONE's enjoying their lunch. #officelife' -The output is a tweet that is not passive aggressive like: -'Hi coworker, -I hope you're having an amazing day! -Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. 😜 -If you're up for trying different lunch options, that could be a fun way to mix things up. -Enjoy your day! #variety' -''', + '''Input is a tweet that might contain passive aggressive language like: +\'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 +But hey, at least SOMEONE's enjoying their lunch. #officelife\'''', str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) From 810fe67d7867fa267e257f1af979b61731e4e53d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 18:12:59 +0200 Subject: [PATCH 07/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9818865..21f8e5f 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -24,7 +24,7 @@ The implemented function and the test must not contain prototype or placeholder The implemented function and the test must run in a docker container based on debian. The implemented function and the test must not use libraries like Flask. The implemented function and the test must not have a __main__ function. -The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding or generation of natural language or using any language model. Other language models are not allowed. +The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding, generation or summrization of natural language or using any language model. Other language models are not allowed. The implemented function and the test must not use gpt_3_5_turbo_api or any other language model if the task does not require understanding or generation of natural language.''' From 930d755d38181de093aca204dcdfb65dfe714924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 18:39:44 +0200 Subject: [PATCH 08/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 5 +++-- src/options/generate/templates_user.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index d7577e7..7c52e76 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -430,7 +430,7 @@ gptdeploy deploy --path {self.microservice_root_path} messages, 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. -Note that the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.''' +Note that you must not ask for information that were already mentioned before.''' ) break except self.TaskRefinementException as e: @@ -453,7 +453,8 @@ Test scenario: agent_response_raw = conversation.chat( template_refinement.format( user_input=user_input, - _optional_test=' test' if refinement_type == 'test' else '', + final_placeholder='''input: "" +weak assertion of output: ""''' if refinement_type == 'test' else '', custom_suffix=custom_suffix, ), role='user' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 21f8e5f..c1c81a1 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -367,7 +367,7 @@ Either ask for clarification like this: Or write the summarized microservice{_optional_test} description like this: **final.txt** ```text - +{final_placeholder} ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. From 9d22511b6f1903babee033d547246bbc604db18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 18:40:33 +0200 Subject: [PATCH 09/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c62554b..d06be3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - group: [1, 2, 3, 4, 5] + group: [1, 2, 3] steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -27,7 +27,7 @@ jobs: - name: Test id: test run: | - pytest -v -s -m "not gpu" --splits 5 --group ${{ matrix.group }} --splitting-algorithm least_duration test/ + pytest -v -s -m "not gpu" --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration test/ timeout-minutes: 10 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From 6ae41d6ae232c415aea07e91e39ac3fb818823c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 19:42:50 +0200 Subject: [PATCH 10/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 28 +++++++++++------ src/options/generate/templates_system.py | 24 +++++++------- src/options/generate/templates_user.py | 40 ++++++++++++++++++------ test/test_generator.py | 2 ++ 4 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 7c52e76..d2a129b 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -24,7 +24,8 @@ from src.options.generate.templates_user import template_generate_microservice_n template_solve_pip_dependency_issue, template_is_dependency_issue, template_generate_playground, \ template_generate_function, template_generate_test, template_generate_requirements, \ template_chain_of_thought, template_summarize_error, \ - template_generate_apt_get_install, template_solve_apt_get_dependency_issue, template_refinement + template_generate_apt_get_install, template_solve_apt_get_dependency_issue, template_pm_task_iteration, \ + template_pm_test_iteration from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path @@ -422,15 +423,26 @@ gptdeploy deploy --path {self.microservice_root_path} if not original_task: self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') - messages = [SystemMessage(content=system_task_introduction + system_task_iteration)] - self.refine_requirements(pm, messages, 'task', '') + messages = [ + SystemMessage(content=system_task_introduction + system_task_iteration), + ] + self.refine_requirements( + pm, + messages, + 'task', + '', + template_pm_task_iteration, + micro_service_initial_description=f'Microservice description: {self.microservice_specification.task}' + ) + # replacing the system message of the task with the system message of the test messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration) self.refine_requirements( pm, messages, 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. -Note that you must not ask for information that were already mentioned before.''' +Note that you must not ask for information that were already mentioned before.''', + template_pm_test_iteration ) break except self.TaskRefinementException as e: @@ -445,17 +457,15 @@ Test scenario: {self.microservice_specification.test} ''') - def refine_requirements(self, pm, messages, refinement_type, custom_suffix): + def refine_requirements(self, pm, messages, refinement_type, custom_suffix, template_pm_iteration, micro_service_initial_description=None): user_input = self.microservice_specification.task num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) agent_response_raw = conversation.chat( - template_refinement.format( - user_input=user_input, - final_placeholder='''input: "" -weak assertion of output: ""''' if refinement_type == 'test' else '', + template_pm_iteration.format( custom_suffix=custom_suffix, + **{'micro_service_initial_description': micro_service_initial_description} if refinement_type == 'task' and len(messages) == 1 else {} ), role='user' ) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index fc1669f..6c4e84b 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -28,11 +28,11 @@ You must not output anything else than what you got told in the following steps. 1. You must create a check list for the requirements of the microservice. Input and output have to be accurately specified. -You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: -input: -output: -api access: -database access: +You must use the following format (insert defined, not defined or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: +input: +output: +api access: +database access: 2. You must do either a or b. @@ -59,9 +59,9 @@ The character sequence ``` must always be at the beginning of the line. You must not add information that was not provided by the client. Example for the description "given a city, get the weather report for the next 5 days": -input: ✅ -output: ✅ -api access: ❌ +input: defined +output: defined +api access: not defined database access: n/a **prompt.txt** @@ -70,8 +70,8 @@ Please provide the url of the weather api and a valid api key or some other way ``` Example for the description "convert png to svg": -input: ✅ -output: ✅ +input: defined +output: defined api access: n/a database access: n/a @@ -81,8 +81,8 @@ The user inserts a png and gets an svg as response. ``` Example for the description "parser": -input: ❌ -output: ❌ +input: not defined +output: not defined api access: n/a database access: n/a diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index c1c81a1..ab2a913 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -349,14 +349,9 @@ The playground (app.py) must not import the executor. ''' ) -# Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. -template_refinement = PromptTemplate.from_template( - ''' +template_pm_task_iteration = PromptTemplate.from_template( + '''{micro_service_initial_description} 1.Quickly go through the checklist (input/output well defined? api or db access needed?) and think about if you should ask something to the client or if you should write the final description. -**client-response.txt** -```text -{user_input} -``` 2.Either write the prompt.txt or the final.txt file. Either ask for clarification like this: **prompt.txt** @@ -364,10 +359,10 @@ Either ask for clarification like this: ``` -Or write the summarized microservice{_optional_test} description like this: +Or write the summarized microservice description like this: **final.txt** ```text -{final_placeholder} + ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. @@ -377,3 +372,30 @@ Note that if urls, secrets, database names, etc. are mentioned, they must be par {custom_suffix} ''' ) + +template_pm_test_iteration = PromptTemplate.from_template( + '''If the example input for the microservice was mentioned already, then output final.txt. +Otherwise, output prompt.txt where you ask for the example input file as URL. +Except for urls, you should come up with your own example input that makes sense for the microservice description. + +Example for the case where you have to ask for the example input file: +**prompt.txt** +```text +Can you please provide an example input file as URL? +``` + +Example for the case where the example input was already mentioned: +**final.txt** +```text +input: "" +weak assertion of output: "" +``` +Note that your response must be either prompt.txt or final.txt. You must not write both. +Note that you must obey the double asterisk and tripple backtick syntax from above. +Note that the last sequence of characters in your response must be ``` (triple backtick). +Note that your response must start with the character sequence ** (double asterisk). +Note that prompt.txt must not only contain one question. +Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary. +{custom_suffix} +''' +) diff --git a/test/test_generator.py b/test/test_generator.py index f8a021c..febab1c 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -65,6 +65,8 @@ def test_generation_level_2(tmpdir): ) generator.generate() +#Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. + @pytest.mark.skip(reason="not possible") def test_generation_level_3(tmpdir): """ From b197d6dd5ab94aeb22f4341b17a03d6a9ae4aeaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 21:45:31 +0200 Subject: [PATCH 11/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 20 +++++++++++++++----- src/options/generate/templates_user.py | 9 +++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index d2a129b..cfb0a24 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -432,17 +432,27 @@ gptdeploy deploy --path {self.microservice_root_path} 'task', '', template_pm_task_iteration, - micro_service_initial_description=f'Microservice description: {self.microservice_specification.task}' + micro_service_initial_description=f'''Microservice description: +{self.microservice_specification.task} +''', ) - # replacing the system message of the task with the system message of the test - messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration) + + messages = [ + SystemMessage(content=system_task_introduction + system_test_iteration), + + ] self.refine_requirements( pm, messages, 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. Note that you must not ask for information that were already mentioned before.''', - template_pm_test_iteration + template_pm_test_iteration, + micro_service_initial_description=f'''Microservice original description: +{original_task} +Microservice refined description: +{self.microservice_specification.task} +''', ) break except self.TaskRefinementException as e: @@ -465,7 +475,7 @@ Test scenario: agent_response_raw = conversation.chat( template_pm_iteration.format( custom_suffix=custom_suffix, - **{'micro_service_initial_description': micro_service_initial_description} if refinement_type == 'task' and len(messages) == 1 else {} + **{'micro_service_initial_description': micro_service_initial_description} if len(messages) == 1 else {} ), role='user' ) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index ab2a913..b6a5876 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -177,7 +177,9 @@ Name all packages which need to be installed via `apt-get install` in above Dock {requirements_file_wrapped} -Output them as a white space separated list:''' +Note that you must not list packages that are already installed in the Dockerfile. +Output the packages as a white space separated list: +''' ) @@ -374,7 +376,10 @@ Note that if urls, secrets, database names, etc. are mentioned, they must be par ) template_pm_test_iteration = PromptTemplate.from_template( - '''If the example input for the microservice was mentioned already, then output final.txt. + '''{micro_service_initial_description} +1. Look at the original description and the refined description. +2. find out if the original description and the refined description contain an example input for the microservice. +If the example input for the microservice is mentioned in the refined description or the original description, then output final.txt. Otherwise, output prompt.txt where you ask for the example input file as URL. Except for urls, you should come up with your own example input that makes sense for the microservice description. From 2c01c496afcaa8c4f31f73e8895ab2151e7994fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:23:16 +0200 Subject: [PATCH 12/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 6 +++--- .test_durations | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d06be3c..b81c1bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,12 +5,12 @@ on: pull_request: jobs: - test: + test level: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - group: [1, 2, 3] + group: [0, 1, 2] steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -27,7 +27,7 @@ jobs: - name: Test id: test run: | - pytest -v -s -m "not gpu" --splits 3 --group ${{ matrix.group }} --splitting-algorithm least_duration test/ + pytest -vs test/test_generator::test_generation_level_${{ matrix.group }} timeout-minutes: 10 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/.test_durations b/.test_durations index 9ee8723..313ea55 100644 --- a/.test_durations +++ b/.test_durations @@ -2,8 +2,6 @@ "test/test_generator.py::test_generation_level_0": 100, "test/test_generator.py::test_generation_level_1": 100, "test/test_generator.py::test_generation_level_2": 100, - "test/test_generator.py::test_generation_level_3": 100, - "test/test_generator.py::test_generation_level_4": 100, "test/test_hub.py::test_is_microservice_in_hub": 1, "test/test_strings.py::test_clean_color_codes": 1 } \ No newline at end of file From 8d9580a7d09c02583e5aed205d86de3a501a7232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:26:38 +0200 Subject: [PATCH 13/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b81c1bc..9fe3645 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Test id: test run: | - pytest -vs test/test_generator::test_generation_level_${{ matrix.group }} + pytest -vs test/test_generator.py::test_generation_level_${{ matrix.group }} timeout-minutes: 10 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From 8d81b1788376b3946d0df175edfe3a9599a13f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:27:18 +0200 Subject: [PATCH 14/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fe3645..76e6d66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - test level: + test_level: runs-on: ubuntu-latest strategy: fail-fast: false From 158650d2d93437b32ca448245f8b9366ad5eae55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:32:17 +0200 Subject: [PATCH 15/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index cfb0a24..987160f 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -439,7 +439,6 @@ gptdeploy deploy --path {self.microservice_root_path} messages = [ SystemMessage(content=system_task_introduction + system_test_iteration), - ] self.refine_requirements( pm, @@ -475,7 +474,7 @@ Test scenario: agent_response_raw = conversation.chat( template_pm_iteration.format( custom_suffix=custom_suffix, - **{'micro_service_initial_description': micro_service_initial_description} if len(messages) == 1 else {} + micro_service_initial_description=micro_service_initial_description if len(messages) == 1 else '', ), role='user' ) From 4c33a3f2989303813c08a9470636e51592a84389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:39:50 +0200 Subject: [PATCH 16/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index b6a5876..9e25920 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -62,7 +62,7 @@ template_generate_possible_packages = PromptTemplate.from_template( "{description}" 1. Write down the different strategies to solve the task. For each strategy write down all the non-trivial subtasks you need to solve. If there is a natural language understanding or generation stragegy, write it down. 2. Find out what is the core problem to solve. -3. List up to 15 Python packages that are specifically designed or have functionalities to solve the complete core problem with one of the defined strategies. You must add gpt_3_5_turbo_api if the task involves generating or understanding natural language or using a (pre-trained) language model. +3. List up to 10 Python packages that are specifically designed or have functionalities to solve the complete core problem with one of the defined strategies. You must add gpt_3_5_turbo_api if the task involves generating or understanding natural language or using a (pre-trained) language model. 4. Exclude any package that can generate or understand natural language or enables using any language model, but you must not exclude gpt_3_5_turbo_api. Print the cleaned list of packages and give a brief reason for keeping it after its name. 5. For each cleaned package think if it fulfills the following requirements: a) specifically designed or have functionalities to solve the complete core problem. From 1cf4ddb625dd73caf784f38baae1eb562a5ac040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 22:59:37 +0200 Subject: [PATCH 17/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/static_files/microservice/Dockerfile | 2 +- src/options/generate/templates_user.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 190e8ff..613ad49 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,6 +1,6 @@ FROM jinaai/jina:3.14.1-py39-standard -RUN apt-get update && apt-get install --no-install-recommends -y {{apt_get_packages}} && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install --no-install-recommends -y {apt_get_packages} && apt-get clean && rm -rf /var/lib/apt/lists/* ## install requirements for the executor COPY requirements.txt . diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9e25920..f69c939 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -60,7 +60,7 @@ package5a ... template_generate_possible_packages = PromptTemplate.from_template( '''Here is the task description of the problem you need to solve: "{description}" -1. Write down the different strategies to solve the task. For each strategy write down all the non-trivial subtasks you need to solve. If there is a natural language understanding or generation stragegy, write it down. +1. Write down ut to 3 different strategies to solve the task. For each strategy write down all the non-trivial subtasks you need to solve. If there is a natural language understanding or generation stragegy, write it down. 2. Find out what is the core problem to solve. 3. List up to 10 Python packages that are specifically designed or have functionalities to solve the complete core problem with one of the defined strategies. You must add gpt_3_5_turbo_api if the task involves generating or understanding natural language or using a (pre-trained) language model. 4. Exclude any package that can generate or understand natural language or enables using any language model, but you must not exclude gpt_3_5_turbo_api. Print the cleaned list of packages and give a brief reason for keeping it after its name. @@ -177,8 +177,9 @@ Name all packages which need to be installed via `apt-get install` in above Dock {requirements_file_wrapped} -Note that you must not list packages that are already installed in the Dockerfile. -Output the packages as a white space separated list: +Note that you must not list apt-get packages that are already installed in the Dockerfile. +Note that openai does not require any apt-get packages. +Output the packages that need to me placed at {apt_get_packages} as a white space separated list: ''' ) From f9e2ef04e8c506b303dec64dc63e06dccd0ed37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 23:17:14 +0200 Subject: [PATCH 18/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 3 ++- src/options/generate/templates_user.py | 2 +- test/test_generator.py | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 987160f..27910be 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -399,6 +399,7 @@ metas: print_colored('', f'Could not debug the Microservice with any of the approaches: {packages} giving up.', 'red') + return -1 continue print(f''' You can now run or deploy your microservice: @@ -406,7 +407,7 @@ gptdeploy run --path {self.microservice_root_path} gptdeploy deploy --path {self.microservice_root_path} ''' ) - break + return 0 def summarize_error(self, error): conversation = self.gpt_session.get_conversation() diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index f69c939..39642fd 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -179,7 +179,7 @@ Name all packages which need to be installed via `apt-get install` in above Dock Note that you must not list apt-get packages that are already installed in the Dockerfile. Note that openai does not require any apt-get packages. -Output the packages that need to me placed at {apt_get_packages} as a white space separated list: +Output the packages that need to me placed at {{apt_get_packages}} as a white space separated list: ''' ) diff --git a/test/test_generator.py b/test/test_generator.py index febab1c..31c9df3 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -23,7 +23,7 @@ def test_generation_level_0(tmpdir): str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) - generator.generate() + assert generator.generate() == 0 def test_generation_level_1(tmpdir): @@ -44,7 +44,7 @@ But hey, at least SOMEONE's enjoying their lunch. #officelife\'''', str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) - generator.generate() + assert generator.generate() == 0 def test_generation_level_2(tmpdir): @@ -63,7 +63,7 @@ def test_generation_level_2(tmpdir): str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) - generator.generate() + assert generator.generate() == 0 #Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. @@ -98,7 +98,7 @@ Create an audio file of the summarized text. str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) - generator.generate() + assert generator.generate() == 0 @pytest.mark.skip(reason="not possible") def test_generation_level_4(tmpdir): @@ -136,4 +136,4 @@ The output is the image with the joke on it.''', str(tmpdir) + 'microservice', 'gpt-3.5-turbo' ) - generator.generate() + assert generator.generate() == 0 From 9f83d5a230866dbb92eefb1bcb956c3eafaf550c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 23:52:45 +0200 Subject: [PATCH 19/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 35 +++++++++++-------- .../static_files/microservice/Dockerfile | 2 -- src/options/generate/templates_user.py | 22 +++++++----- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 27910be..143facb 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -1,3 +1,4 @@ +import json import os import random import re @@ -185,16 +186,22 @@ metas: tag_name=REQUIREMENTS_FILE_TAG, )[REQUIREMENTS_FILE_NAME] - self.generate_and_persist_file( - section_title='Generate Dockerfile', - template=template_generate_apt_get_install, - destination_folder=MICROSERVICE_FOLDER_v1, - file_name_s=None, - parse_result_fn=self.parse_result_fn_dockerfile, - docker_file_wrapped=self.read_docker_template(), - requirements_file_wrapped=self.files_to_string({ - REQUIREMENTS_FILE_NAME: requirements_content, - }) + # self.generate_and_persist_file( + # section_title='Generate Dockerfile', + # template=template_generate_apt_get_install, + # destination_folder=MICROSERVICE_FOLDER_v1, + # file_name_s=None, + # parse_result_fn=self.parse_result_fn_dockerfile, + # docker_file_wrapped=self.read_docker_template(), + # requirements_file_wrapped=self.files_to_string({ + # REQUIREMENTS_FILE_NAME: requirements_content, + # }) + # ) + + # docker file from curdir + shutil.copy( + src=os.path.join(os.path.dirname(__file__), 'static_files', 'microservice', 'Dockerfile'), + dst=MICROSERVICE_FOLDER_v1 + '/Dockerfile' ) self.write_config_yml(microservice_name, MICROSERVICE_FOLDER_v1) @@ -364,15 +371,15 @@ metas: def get_possible_packages(self): print_colored('', '\n\n############# What packages to use? #############', 'blue') - packages_csv_string = self.generate_and_persist_file( + packages_json_string = self.generate_and_persist_file( section_title='Generate possible packages', template=template_generate_possible_packages, destination_folder=self.microservice_root_path, - file_name_s=['packages.csv'], + file_name_s=['strategies.json'], system_definition_examples=[], description=self.microservice_specification.task - )['packages.csv'] - packages_list = [[pkg.strip().lower() for pkg in packages_string.split(',')] for packages_string in packages_csv_string.split('\n')] + )['strategies.json'] + packages_list = [[pkg.strip().lower() for pkg in packages] for packages in json.loads(packages_json_string)] packages_list = [ packages for packages in packages_list if len(set(packages).intersection(set(PROBLEMATIC_PACKAGES))) == 0 ] diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 613ad49..3ba6f3a 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,7 +1,5 @@ FROM jinaai/jina:3.14.1-py39-standard -RUN apt-get update && apt-get install --no-install-recommends -y {apt_get_packages} && apt-get clean && rm -rf /var/lib/apt/lists/* - ## install requirements for the executor COPY requirements.txt . RUN pip install --compile -r requirements.txt diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 39642fd..bf5d472 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -45,15 +45,21 @@ PDFParserExecutor ) -template_generate_possible_packages_output_format_string = '''You must output the package combinations as a list of lists wrapped into ``` and name it **packages.csv**. Do not use quotation marks around packages names in the output. Separate packages in a combination by comma. The output looks this: -**{file_name}** +template_generate_possible_packages_output_format_string = '''You must output the package combinations as a \ +list of lists wrapped into ``` and name it **strategies.json**. \ +Do not use quotation marks around packages names in the output. \ +Separate packages in a combination by comma. \ +Note that you can also leave a line empty to indicate that one of the strategies does not require any package and can be done in plain python. +The output looks like this: +**strategies.json** ``` -package1a, package1b ... -package2a, package2b, package2c -package3a ... -package4a ... -package5a ... -... +[ + ["package1", "package2", "package3"], + ["package4", "package5"], + ["package6", "package7", "package8", "package9"], + [], + ["package10"] +] ```''' From 177cb9a094c77702119eed12abde3a25bc8a8661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 00:21:21 +0200 Subject: [PATCH 20/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index bf5d472..34a5bb7 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -24,8 +24,7 @@ The implemented function and the test must not contain prototype or placeholder The implemented function and the test must run in a docker container based on debian. The implemented function and the test must not use libraries like Flask. The implemented function and the test must not have a __main__ function. -The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding, generation or summrization of natural language or using any language model. Other language models are not allowed. -The implemented function and the test must not use gpt_3_5_turbo_api or any other language model if the task does not require understanding or generation of natural language.''' +The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding, generation, summarization, transformation, translation or classification of natural language or using any language model. Other language models are not allowed.''' template_generate_microservice_name = PromptTemplate.from_template( @@ -400,7 +399,7 @@ Example for the case where the example input was already mentioned: **final.txt** ```text input: "" -weak assertion of output: "" +assertion: "the output must be of type " ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. From f41fce9368c50dd7bad6168ef513dd333c9e39e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 00:29:52 +0200 Subject: [PATCH 21/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 34a5bb7..9dc9b41 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -389,13 +389,19 @@ If the example input for the microservice is mentioned in the refined descriptio Otherwise, output prompt.txt where you ask for the example input file as URL. Except for urls, you should come up with your own example input that makes sense for the microservice description. -Example for the case where you have to ask for the example input file: +Example for the case where an example input file is required and was not mentioned before: **prompt.txt** ```text Can you please provide an example input file as URL? ``` -Example for the case where the example input was already mentioned: +Example for the case where the example input string is required and was not mentioned before: +**prompt.txt** +```text +Can you please provide an example input string? +``` + +Example for the case where the example input string or the example input file URL was already mentioned: **final.txt** ```text input: "" From 7a67bd987b8aa3b33e14a96a009844d4f417eebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 00:33:28 +0200 Subject: [PATCH 22/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_generator.py b/test/test_generator.py index 31c9df3..e0750c0 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -38,7 +38,8 @@ def test_generation_level_1(tmpdir): """ os.environ['VERBOSE'] = 'true' generator = Generator( - '''Input is a tweet that might contain passive aggressive language like: + '''Input is a tweet that might contain passive aggressive language. +Example tweet: \'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 But hey, at least SOMEONE's enjoying their lunch. #officelife\'''', str(tmpdir) + 'microservice', From 4ba38892db2ca4ced9bccfec123f1252617b879a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 00:43:28 +0200 Subject: [PATCH 23/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 6c4e84b..caaac73 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -96,9 +96,9 @@ system_test_iteration = f''' The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: -a) If the input of the unit test requires an example input file or e.g. a certain textual description that is not mentioned before: -In this case you must ask the client to provide the example input file as URL. +a) If no example input is provided in the description, then you must ask the client to provide an example input file URL or example string depending on the use-case. You must not accept files that are not URLs. +You must not ask for an example input in case the input can be determined from the conversation with the client. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **prompt.txt** From a88c15a393828a40c8a442a1be15952986f545cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 00:46:13 +0200 Subject: [PATCH 24/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_generator.py b/test/test_generator.py index e0750c0..877cc7a 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -38,7 +38,7 @@ def test_generation_level_1(tmpdir): """ os.environ['VERBOSE'] = 'true' generator = Generator( - '''Input is a tweet that might contain passive aggressive language. + '''Input is a tweet that might contain passive aggressive language. The output is the positive version of that tweet. Example tweet: \'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 But hey, at least SOMEONE's enjoying their lunch. #officelife\'''', From 3afef5b26f305b6437fda43489af03a67bd53036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 01:38:18 +0200 Subject: [PATCH 25/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 28 +++++++++++++++-------- src/options/generate/templates_system.py | 29 ++++++++++++++++++++---- src/options/generate/templates_user.py | 12 +++++----- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 143facb..2828323 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -46,7 +46,7 @@ class Generator: def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break - pattern = fr"^\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" + pattern = fr"\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" match = re.search(pattern, plain_text, re.MULTILINE) if match: return match.group(1).strip() @@ -431,12 +431,11 @@ gptdeploy deploy --path {self.microservice_root_path} if not original_task: self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') - messages = [ - SystemMessage(content=system_task_introduction + system_task_iteration), - ] self.refine_requirements( pm, - messages, + [ + SystemMessage(content=system_task_introduction + system_task_iteration), + ], 'task', '', template_pm_task_iteration, @@ -445,20 +444,31 @@ gptdeploy deploy --path {self.microservice_root_path} ''', ) - messages = [ - SystemMessage(content=system_task_introduction + system_test_iteration), - ] + # while True: + # example_input = get_example_input() + # if example_input: + # break + # else: + # user_input = ask_client_for_example() + # + # is example input already provided self.refine_requirements( pm, - messages, + [ + SystemMessage(content=system_task_introduction + system_test_iteration), + ], 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. Note that you must not ask for information that were already mentioned before.''', template_pm_test_iteration, micro_service_initial_description=f'''Microservice original description: +``` {original_task} +``` Microservice refined description: +``` {self.microservice_specification.task} +``` ''', ) break diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index caaac73..a0a1dfc 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -101,6 +101,9 @@ You must not accept files that are not URLs. You must not ask for an example input in case the input can be determined from the conversation with the client. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): +1. +contains example: no +2. **prompt.txt** ```text @@ -110,37 +113,53 @@ If you did a, you must not do b. b) If the input can be determined from the previous messages: In this case you must describe the unit test verbally. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): + +1. +contains example: yes () +2. **final.txt** ```text input: "" -weak assertion of output: "" +assertion: "" ``` + If you did b, you must not do a. -Example for: "given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22": +Example for: "given a city like "Berlin", get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22": +1. +contains example: yes (Berlin) +2. **final.txt** ```text input: "Berlin" -weak assertion of output: "contains weather report for the next 5 days" +assertion: "contains weather report for the next 5 days" ``` Example for "The user inserts a png and gets an svg as response.": +1. +contains example: no +2. **prompt.txt** ```text Please provide a png example input file as url. ``` Example for "The user inserts a png like https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png and gets an svg as response.": +1. +contains example: yes (https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png) +2. **final.txt** ```text input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png" -weak assertion of output: "is an svg" +assertion: "is an svg" ``` Example for "The microservice takes nothing as input and returns the current time.": +1. +contains example: n/a **final.txt** ```text input: "nothing" -weak assertion of output: "is a string" +assertion: "is a string" ``` ''' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9dc9b41..6978eff 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -383,10 +383,10 @@ Note that if urls, secrets, database names, etc. are mentioned, they must be par template_pm_test_iteration = PromptTemplate.from_template( '''{micro_service_initial_description} -1. Look at the original description and the refined description. -2. find out if the original description and the refined description contain an example input for the microservice. +1. write down if the original description and the refined description contain an example input for the microservice. +2. write down either prompt.txt or final.txt. If the example input for the microservice is mentioned in the refined description or the original description, then output final.txt. -Otherwise, output prompt.txt where you ask for the example input file as URL. +Otherwise, output prompt.txt where you ask for the example input file as URL or the example string. Except for urls, you should come up with your own example input that makes sense for the microservice description. Example for the case where an example input file is required and was not mentioned before: @@ -400,8 +400,9 @@ Example for the case where the example input string is required and was not ment ```text Can you please provide an example input string? ``` +Note that you must not ask for an example input in case the example input is already mentioned in the refined description or the original description. -Example for the case where the example input string or the example input file URL was already mentioned: +Example for the case where the example is already mentioned in the refined description or the original description: **final.txt** ```text input: "" @@ -411,8 +412,7 @@ Note that your response must be either prompt.txt or final.txt. You must not wri Note that you must obey the double asterisk and tripple backtick syntax from above. Note that the last sequence of characters in your response must be ``` (triple backtick). Note that your response must start with the character sequence ** (double asterisk). -Note that prompt.txt must not only contain one question. -Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary. +Note that prompt.txt must only contain one question. {custom_suffix} ''' ) From 36ccd79bc89ec3134e920f082cbb758e152f40a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 01:39:51 +0200 Subject: [PATCH 26/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 2828323..a5eeb40 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -443,15 +443,6 @@ gptdeploy deploy --path {self.microservice_root_path} {self.microservice_specification.task} ''', ) - - # while True: - # example_input = get_example_input() - # if example_input: - # break - # else: - # user_input = ask_client_for_example() - # - # is example input already provided self.refine_requirements( pm, [ From 30e0b8cf1e136420835467a4d2585c9ea88c4f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 02:06:09 +0200 Subject: [PATCH 27/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 7 +++++++ src/options/generate/templates_system.py | 8 ++++---- src/options/generate/templates_user.py | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index a5eeb40..04541cb 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -380,6 +380,8 @@ metas: description=self.microservice_specification.task )['strategies.json'] packages_list = [[pkg.strip().lower() for pkg in packages] for packages in json.loads(packages_json_string)] + packages_list = [[self.replace_with_gpt_3_5_turbo_if_possible(pkg) for pkg in packages] for packages in packages_list] + packages_list = [ packages for packages in packages_list if len(set(packages).intersection(set(PROBLEMATIC_PACKAGES))) == 0 ] @@ -512,3 +514,8 @@ Test scenario: while not val: val = input('you: ') return val + + @staticmethod + def replace_with_gpt_3_5_turbo_if_possible(pkg): + if pkg in ['nltk', 'textblob', 'spacy', 'transformers']: + return 'gpt_3_5_turbo_api' diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index a0a1dfc..7dabc54 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -120,7 +120,7 @@ contains example: yes () **final.txt** ```text input: "" -assertion: "" +assertion: the output is of type ``` If you did b, you must not do a. @@ -132,7 +132,7 @@ contains example: yes (Berlin) **final.txt** ```text input: "Berlin" -assertion: "contains weather report for the next 5 days" +assertion: the output is of type string ``` Example for "The user inserts a png and gets an svg as response.": @@ -151,7 +151,7 @@ contains example: yes (https://aquasecurity.github.io/kube-bench/v0.6.5/images/k **final.txt** ```text input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png" -assertion: "is an svg" +assertion: the output is of type svg ``` Example for "The microservice takes nothing as input and returns the current time.": @@ -160,6 +160,6 @@ contains example: n/a **final.txt** ```text input: "nothing" -assertion: "is a string" +assertion: the output is of type string ``` ''' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 6978eff..f3bd32a 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -405,8 +405,8 @@ Note that you must not ask for an example input in case the example input is alr Example for the case where the example is already mentioned in the refined description or the original description: **final.txt** ```text -input: "" -assertion: "the output must be of type " +input: +assertion: the output is of type ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. From ecedc1c6bf28c4183ee61c0ad6af2110597b356f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 02:27:53 +0200 Subject: [PATCH 28/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 04541cb..7860a48 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -519,3 +519,4 @@ Test scenario: def replace_with_gpt_3_5_turbo_if_possible(pkg): if pkg in ['nltk', 'textblob', 'spacy', 'transformers']: return 'gpt_3_5_turbo_api' + return pkg From 222ca015689fade8b71aea24e6d1d65cdaffca72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 10:51:12 +0200 Subject: [PATCH 29/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 7860a48..f9a6539 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -183,6 +183,7 @@ metas: }), file_name_purpose=REQUIREMENTS_FILE_NAME, file_name_s=[REQUIREMENTS_FILE_NAME], + parse_result_fn=self.parse_result_fn_requirements, tag_name=REQUIREMENTS_FILE_TAG, )[REQUIREMENTS_FILE_NAME] @@ -217,6 +218,16 @@ metas: docker_file_template = self.read_docker_template() return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(apt_get_packages=content_raw)} + def parse_result_fn_requirements(self, content_raw: str): + lines = content_raw.split('\n') + lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest']])] + content_modified = f'''jina==3.15.1.dev14 +docarray==0.21.0 +openai>=0.26.0 +pytest +{os.linesep.join(lines)}''' + return {REQUIREMENTS_FILE_NAME: content_modified} + def generate_playground(self, microservice_name, microservice_path): print_colored('', '\n\n############# Playground #############', 'blue') From ce8f6a1418a94681b84d06ce8f3fbbe4425864ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 10:55:16 +0200 Subject: [PATCH 30/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index f9a6539..a9f2bc9 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -219,7 +219,9 @@ metas: return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(apt_get_packages=content_raw)} def parse_result_fn_requirements(self, content_raw: str): - lines = content_raw.split('\n') + content_parsed = self.extract_content_from_result(content_raw, 'requirements.txt', match_single_block=True) + + lines = content_parsed.split('\n') lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest']])] content_modified = f'''jina==3.15.1.dev14 docarray==0.21.0 From 9b02b853240c9fc2bdbf60ddef412d991a1c8c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 11:04:08 +0200 Subject: [PATCH 31/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index a9f2bc9..8ea9300 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -530,6 +530,6 @@ Test scenario: @staticmethod def replace_with_gpt_3_5_turbo_if_possible(pkg): - if pkg in ['nltk', 'textblob', 'spacy', 'transformers']: + if pkg in ['nltk', 'textblob', 'spacy', 'transformers', 'textstat']: return 'gpt_3_5_turbo_api' return pkg From c8aa95e4ceaabc389c9f7736fb50fc5d22b388d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 11:08:01 +0200 Subject: [PATCH 32/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 65abdd2..321834d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ openai>=0.26.0 psutil jcloud jina-hubble-sdk -langchain \ No newline at end of file +langchain==0.0.153 \ No newline at end of file From d52e7489c589bbce99565ab51811f0b7643ad4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 11:41:26 +0200 Subject: [PATCH 33/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 41 ++++++++++--------- .../static_files/microservice/Dockerfile | 2 + src/options/generate/templates_user.py | 21 ++++++---- test/test_generator.py | 5 +++ 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 8ea9300..f3ef5fe 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -187,24 +187,24 @@ metas: tag_name=REQUIREMENTS_FILE_TAG, )[REQUIREMENTS_FILE_NAME] - # self.generate_and_persist_file( - # section_title='Generate Dockerfile', - # template=template_generate_apt_get_install, - # destination_folder=MICROSERVICE_FOLDER_v1, - # file_name_s=None, - # parse_result_fn=self.parse_result_fn_dockerfile, - # docker_file_wrapped=self.read_docker_template(), - # requirements_file_wrapped=self.files_to_string({ - # REQUIREMENTS_FILE_NAME: requirements_content, - # }) - # ) - - # docker file from curdir - shutil.copy( - src=os.path.join(os.path.dirname(__file__), 'static_files', 'microservice', 'Dockerfile'), - dst=MICROSERVICE_FOLDER_v1 + '/Dockerfile' + self.generate_and_persist_file( + section_title='Generate Dockerfile', + template=template_generate_apt_get_install, + destination_folder=MICROSERVICE_FOLDER_v1, + file_name_s=None, + parse_result_fn=self.parse_result_fn_dockerfile, + docker_file_wrapped=self.read_docker_template(), + requirements_file_wrapped=self.files_to_string({ + REQUIREMENTS_FILE_NAME: requirements_content, + }) ) + # # docker file from curdir + # shutil.copy( + # src=os.path.join(os.path.dirname(__file__), 'static_files', 'microservice', 'Dockerfile'), + # dst=MICROSERVICE_FOLDER_v1 + '/Dockerfile' + # ) + self.write_config_yml(microservice_name, MICROSERVICE_FOLDER_v1) print('\nFirst version of the microservice generated. Start iterating on it to make the tests pass...') @@ -215,14 +215,17 @@ metas: return f.read() def parse_result_fn_dockerfile(self, content_raw: str): + json_string = self.extract_content_from_result(content_raw, 'apt-get-packages.json', match_single_block=True) + packages = json.loads(json_string)['packages'] + docker_file_template = self.read_docker_template() - return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(apt_get_packages=content_raw)} + return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(apt_get_packages=packages)} def parse_result_fn_requirements(self, content_raw: str): content_parsed = self.extract_content_from_result(content_raw, 'requirements.txt', match_single_block=True) lines = content_parsed.split('\n') - lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest']])] + lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest', 'gpt_3_5_turbo_api']])] content_modified = f'''jina==3.15.1.dev14 docarray==0.21.0 openai>=0.26.0 @@ -322,7 +325,7 @@ pytest section_title='Debugging apt-get dependency issue', template=template_solve_apt_get_dependency_issue, destination_folder=next_microservice_path, - file_name_s=None, + file_name_s=['apt-get-packages.json'], parse_result_fn=self.parse_result_fn_dockerfile, system_definition_examples=[], summarized_error=summarized_error, diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 3ba6f3a..190e8ff 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,5 +1,7 @@ FROM jinaai/jina:3.14.1-py39-standard +RUN apt-get update && apt-get install --no-install-recommends -y {{apt_get_packages}} && apt-get clean && rm -rf /var/lib/apt/lists/* + ## install requirements for the executor COPY requirements.txt . RUN pip install --compile -r requirements.txt diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index f3bd32a..aba7c60 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -157,13 +157,10 @@ template_generate_requirements = PromptTemplate.from_template( {code_files_wrapped} -Write the content of the requirements.txt file. -The requirements.txt file must include the following packages in that specified version: +Write the content of the requirements.txt file like this: +**requirements.txt** ``` -jina==3.15.1.dev14 -docarray==0.21.0 -openai>=0.26.0 -pytest +... ``` Add any more packages that are needed to run the code. You must not add gpt_3_5_turbo_api to the requirements.txt file. @@ -184,7 +181,17 @@ Name all packages which need to be installed via `apt-get install` in above Dock Note that you must not list apt-get packages that are already installed in the Dockerfile. Note that openai does not require any apt-get packages. -Output the packages that need to me placed at {{apt_get_packages}} as a white space separated list: +Note that you only list packages where you are highly confident that they are really needed. +Output the packages that need to me placed at {{apt_get_packages}} as json in the following format: +**apt-get-packages.json** +```json +{{"packages": ["", ""]}} +``` +Have in mind that the packages list can be empty like this: +**apt-get-packages.json** +```json +{{"packages": []}} +``` ''' ) diff --git a/test/test_generator.py b/test/test_generator.py index 877cc7a..6a5ad87 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -26,6 +26,11 @@ def test_generation_level_0(tmpdir): assert generator.generate() == 0 +# fixture +@pytest.fixture +def tmpdir(): + return 'microservice' + def test_generation_level_1(tmpdir): """ Requirements: From 080468cbc2a1a641428f670616c01a4f4d3c05c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 11:53:47 +0200 Subject: [PATCH 34/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 2 +- src/options/generate/templates_user.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index f3ef5fe..4174ce7 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -216,7 +216,7 @@ metas: def parse_result_fn_dockerfile(self, content_raw: str): json_string = self.extract_content_from_result(content_raw, 'apt-get-packages.json', match_single_block=True) - packages = json.loads(json_string)['packages'] + packages = ' '.join(json.loads(json_string)['packages']) docker_file_template = self.read_docker_template() return {DOCKER_FILE_NAME: docker_file_template.replace('{{apt_get_packages}}', '{apt_get_packages}').format(apt_get_packages=packages)} diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index aba7c60..266408b 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -181,16 +181,23 @@ Name all packages which need to be installed via `apt-get install` in above Dock Note that you must not list apt-get packages that are already installed in the Dockerfile. Note that openai does not require any apt-get packages. -Note that you only list packages where you are highly confident that they are really needed. +Note that you are only allowed to list packages where you are highly confident that they are really needed. +Note that you can assume that the standard python packages are already installed. Output the packages that need to me placed at {{apt_get_packages}} as json in the following format: **apt-get-packages.json** ```json {{"packages": ["", ""]}} ``` -Have in mind that the packages list can be empty like this: +Example for the following requirements.txt file: +**requirements.txt** +``` +numpy==1.19.5 +fitz +``` +The output would be: **apt-get-packages.json** ```json -{{"packages": []}} +{"packages": []} ``` ''' ) From d61242b15c28dc0bc0c713896d441205c318b7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 12:01:40 +0200 Subject: [PATCH 35/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 266408b..a505435 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -197,7 +197,7 @@ fitz The output would be: **apt-get-packages.json** ```json -{"packages": []} +{{"packages": []}} ``` ''' ) From c7b0cf78fa2a9dbd80d76a5ade10a2db2bd1eff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 12:09:16 +0200 Subject: [PATCH 36/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 34 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 4174ce7..37eb171 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -187,24 +187,26 @@ metas: tag_name=REQUIREMENTS_FILE_TAG, )[REQUIREMENTS_FILE_NAME] - self.generate_and_persist_file( - section_title='Generate Dockerfile', - template=template_generate_apt_get_install, - destination_folder=MICROSERVICE_FOLDER_v1, - file_name_s=None, - parse_result_fn=self.parse_result_fn_dockerfile, - docker_file_wrapped=self.read_docker_template(), - requirements_file_wrapped=self.files_to_string({ - REQUIREMENTS_FILE_NAME: requirements_content, - }) - ) - - # # docker file from curdir - # shutil.copy( - # src=os.path.join(os.path.dirname(__file__), 'static_files', 'microservice', 'Dockerfile'), - # dst=MICROSERVICE_FOLDER_v1 + '/Dockerfile' + # self.generate_and_persist_file( + # section_title='Generate Dockerfile', + # template=template_generate_apt_get_install, + # destination_folder=MICROSERVICE_FOLDER_v1, + # file_name_s=None, + # parse_result_fn=self.parse_result_fn_dockerfile, + # docker_file_wrapped=self.read_docker_template(), + # requirements_file_wrapped=self.files_to_string({ + # REQUIREMENTS_FILE_NAME: requirements_content, + # }) # ) + + with open(os.path.join(os.path.dirname(__file__), 'static_files', 'microservice', 'Dockerfile'), 'r', encoding='utf-8') as f: + docker_file_template_lines = f.readlines() + docker_file_template_lines = [line for line in docker_file_template_lines if not line.startswith('RUN apt-get update')] + docker_file_content = '\n'.join(docker_file_template_lines) + persist_file(docker_file_content, os.path.join(MICROSERVICE_FOLDER_v1, 'Dockerfile')) + + self.write_config_yml(microservice_name, MICROSERVICE_FOLDER_v1) print('\nFirst version of the microservice generated. Start iterating on it to make the tests pass...') From e99131973d4230e0fdfb8075396fc0b2a3279ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 12:26:33 +0200 Subject: [PATCH 37/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 37eb171..60bfc76 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -187,6 +187,8 @@ metas: tag_name=REQUIREMENTS_FILE_TAG, )[REQUIREMENTS_FILE_NAME] + # I deactivated this because 3.5-turbo was halucinating packages that were not needed + # now, in the first iteration the default dockerfile is used # self.generate_and_persist_file( # section_title='Generate Dockerfile', # template=template_generate_apt_get_install, @@ -535,6 +537,6 @@ Test scenario: @staticmethod def replace_with_gpt_3_5_turbo_if_possible(pkg): - if pkg in ['nltk', 'textblob', 'spacy', 'transformers', 'textstat']: + if pkg in ['nltk', 'textblob', 'spacy', 'transformers', 'textstat', 'gensim']: return 'gpt_3_5_turbo_api' return pkg From 836255c204e042994f0101f79fd68f1c94529d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 13:17:12 +0200 Subject: [PATCH 38/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 7 +++++-- src/options/generate/static_files/microservice/Dockerfile | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 60bfc76..96a9aee 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -232,7 +232,7 @@ metas: lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest', 'gpt_3_5_turbo_api']])] content_modified = f'''jina==3.15.1.dev14 docarray==0.21.0 -openai>=0.26.0 +openai==0.26.0 pytest {os.linesep.join(lines)}''' return {REQUIREMENTS_FILE_NAME: content_modified} @@ -537,6 +537,9 @@ Test scenario: @staticmethod def replace_with_gpt_3_5_turbo_if_possible(pkg): - if pkg in ['nltk', 'textblob', 'spacy', 'transformers', 'textstat', 'gensim']: + if pkg in ['allennlp', 'bertopic', 'fasttext', 'flair', 'gensim', 'nltk', + 'pattern', 'polyglot', 'pytorch-transformers', 'rasa', 'sentence-transformers', + 'spacy', 'stanza', 'textblob', 'textstat', 'transformers']: + return 'gpt_3_5_turbo_api' return pkg diff --git a/src/options/generate/static_files/microservice/Dockerfile b/src/options/generate/static_files/microservice/Dockerfile index 190e8ff..d0de26d 100644 --- a/src/options/generate/static_files/microservice/Dockerfile +++ b/src/options/generate/static_files/microservice/Dockerfile @@ -1,4 +1,6 @@ FROM jinaai/jina:3.14.1-py39-standard +# update pip +RUN pip install --upgrade pip RUN apt-get update && apt-get install --no-install-recommends -y {{apt_get_packages}} && apt-get clean && rm -rf /var/lib/apt/lists/* From 5d6e5810f5a31e332e361b880c3da475e58b9d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 18:03:37 +0200 Subject: [PATCH 39/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 2 +- src/options/generate/generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 321834d..8dbf199 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ jina==3.15.1.dev14 click streamlit==1.9.0 -openai>=0.26.0 +openai>=0.27.5 psutil jcloud jina-hubble-sdk diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 96a9aee..ad6b45b 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -232,7 +232,7 @@ metas: lines = [line for line in lines if not any([pkg in line for pkg in ['jina', 'docarray', 'openai', 'pytest', 'gpt_3_5_turbo_api']])] content_modified = f'''jina==3.15.1.dev14 docarray==0.21.0 -openai==0.26.0 +openai==0.27.5 pytest {os.linesep.join(lines)}''' return {REQUIREMENTS_FILE_NAME: content_modified} From d960496e1765d0b47b66e5bf40964fda8857f08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Mon, 1 May 2023 18:19:25 +0200 Subject: [PATCH 40/40] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .test_durations | 7 ------- test/test_generator.py | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 .test_durations diff --git a/.test_durations b/.test_durations deleted file mode 100644 index 313ea55..0000000 --- a/.test_durations +++ /dev/null @@ -1,7 +0,0 @@ -{ - "test/test_generator.py::test_generation_level_0": 100, - "test/test_generator.py::test_generation_level_1": 100, - "test/test_generator.py::test_generation_level_2": 100, - "test/test_hub.py::test_is_microservice_in_hub": 1, - "test/test_strings.py::test_clean_color_codes": 1 -} \ No newline at end of file diff --git a/test/test_generator.py b/test/test_generator.py index 6a5ad87..cac1950 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -71,7 +71,6 @@ def test_generation_level_2(tmpdir): ) assert generator.generate() == 0 -#Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. @pytest.mark.skip(reason="not possible") def test_generation_level_3(tmpdir): @@ -143,3 +142,7 @@ The output is the image with the joke on it.''', 'gpt-3.5-turbo' ) assert generator.generate() == 0 + + +# further ideas: +# Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. \ No newline at end of file