From 6ae41d6ae232c415aea07e91e39ac3fb818823c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Sun, 30 Apr 2023 19:42:50 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20test:=20level=202?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 28 +++++++++++------ src/options/generate/templates_system.py | 24 +++++++------- src/options/generate/templates_user.py | 40 ++++++++++++++++++------ test/test_generator.py | 2 ++ 4 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 7c52e76..d2a129b 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -24,7 +24,8 @@ from src.options.generate.templates_user import template_generate_microservice_n template_solve_pip_dependency_issue, template_is_dependency_issue, template_generate_playground, \ template_generate_function, template_generate_test, template_generate_requirements, \ template_chain_of_thought, template_summarize_error, \ - template_generate_apt_get_install, template_solve_apt_get_dependency_issue, template_refinement + template_generate_apt_get_install, template_solve_apt_get_dependency_issue, template_pm_task_iteration, \ + template_pm_test_iteration from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path @@ -422,15 +423,26 @@ gptdeploy deploy --path {self.microservice_root_path} if not original_task: self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') - messages = [SystemMessage(content=system_task_introduction + system_task_iteration)] - self.refine_requirements(pm, messages, 'task', '') + messages = [ + SystemMessage(content=system_task_introduction + system_task_iteration), + ] + self.refine_requirements( + pm, + messages, + 'task', + '', + template_pm_task_iteration, + micro_service_initial_description=f'Microservice description: {self.microservice_specification.task}' + ) + # replacing the system message of the task with the system message of the test messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration) self.refine_requirements( pm, messages, 'test', '''Note that the test scenario must not contain information that was already mentioned in the microservice description. -Note that you must not ask for information that were already mentioned before.''' +Note that you must not ask for information that were already mentioned before.''', + template_pm_test_iteration ) break except self.TaskRefinementException as e: @@ -445,17 +457,15 @@ Test scenario: {self.microservice_specification.test} ''') - def refine_requirements(self, pm, messages, refinement_type, custom_suffix): + def refine_requirements(self, pm, messages, refinement_type, custom_suffix, template_pm_iteration, micro_service_initial_description=None): user_input = self.microservice_specification.task num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) agent_response_raw = conversation.chat( - template_refinement.format( - user_input=user_input, - final_placeholder='''input: "" -weak assertion of output: ""''' if refinement_type == 'test' else '', + template_pm_iteration.format( custom_suffix=custom_suffix, + **{'micro_service_initial_description': micro_service_initial_description} if refinement_type == 'task' and len(messages) == 1 else {} ), role='user' ) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index fc1669f..6c4e84b 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -28,11 +28,11 @@ You must not output anything else than what you got told in the following steps. 1. You must create a check list for the requirements of the microservice. Input and output have to be accurately specified. -You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: -input: -output: -api access: -database access: +You must use the following format (insert defined, not defined or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: +input: +output: +api access: +database access: 2. You must do either a or b. @@ -59,9 +59,9 @@ The character sequence ``` must always be at the beginning of the line. You must not add information that was not provided by the client. Example for the description "given a city, get the weather report for the next 5 days": -input: ✅ -output: ✅ -api access: ❌ +input: defined +output: defined +api access: not defined database access: n/a **prompt.txt** @@ -70,8 +70,8 @@ Please provide the url of the weather api and a valid api key or some other way ``` Example for the description "convert png to svg": -input: ✅ -output: ✅ +input: defined +output: defined api access: n/a database access: n/a @@ -81,8 +81,8 @@ The user inserts a png and gets an svg as response. ``` Example for the description "parser": -input: ❌ -output: ❌ +input: not defined +output: not defined api access: n/a database access: n/a diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index c1c81a1..ab2a913 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -349,14 +349,9 @@ The playground (app.py) must not import the executor. ''' ) -# Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. -template_refinement = PromptTemplate.from_template( - ''' +template_pm_task_iteration = PromptTemplate.from_template( + '''{micro_service_initial_description} 1.Quickly go through the checklist (input/output well defined? api or db access needed?) and think about if you should ask something to the client or if you should write the final description. -**client-response.txt** -```text -{user_input} -``` 2.Either write the prompt.txt or the final.txt file. Either ask for clarification like this: **prompt.txt** @@ -364,10 +359,10 @@ Either ask for clarification like this: ``` -Or write the summarized microservice{_optional_test} description like this: +Or write the summarized microservice description like this: **final.txt** ```text -{final_placeholder} + ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. @@ -377,3 +372,30 @@ Note that if urls, secrets, database names, etc. are mentioned, they must be par {custom_suffix} ''' ) + +template_pm_test_iteration = PromptTemplate.from_template( + '''If the example input for the microservice was mentioned already, then output final.txt. +Otherwise, output prompt.txt where you ask for the example input file as URL. +Except for urls, you should come up with your own example input that makes sense for the microservice description. + +Example for the case where you have to ask for the example input file: +**prompt.txt** +```text +Can you please provide an example input file as URL? +``` + +Example for the case where the example input was already mentioned: +**final.txt** +```text +input: "" +weak assertion of output: "" +``` +Note that your response must be either prompt.txt or final.txt. You must not write both. +Note that you must obey the double asterisk and tripple backtick syntax from above. +Note that the last sequence of characters in your response must be ``` (triple backtick). +Note that your response must start with the character sequence ** (double asterisk). +Note that prompt.txt must not only contain one question. +Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary. +{custom_suffix} +''' +) diff --git a/test/test_generator.py b/test/test_generator.py index f8a021c..febab1c 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -65,6 +65,8 @@ def test_generation_level_2(tmpdir): ) generator.generate() +#Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. + @pytest.mark.skip(reason="not possible") def test_generation_level_3(tmpdir): """