diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index f4170bb..a153c9d 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -44,7 +44,7 @@ class Generator: def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break - pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" + pattern = fr"^\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" match = re.search(pattern, plain_text, re.MULTILINE) if match: return match.group(1).strip() @@ -422,8 +422,16 @@ gptdeploy deploy --path {self.microservice_root_path} if not original_task: self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') - self.refine_requirements(pm, system_task_iteration, 'task') - self.refine_requirements(pm, system_test_iteration, 'test') + messages = [SystemMessage(content=system_task_introduction + system_task_iteration)] + self.refine_requirements(pm, messages, 'task', '') + messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration) + self.refine_requirements( + pm, + messages, + 'test', + '''Note that the test scenario must not contain information that was already mentioned in the microservice description. +Note that if the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.''' + ) break except self.TaskRefinementException as e: @@ -437,11 +445,8 @@ Test scenario: {self.microservice_specification.test} ''') - def refine_requirements(self, pm, template_init, refinement_type): + def refine_requirements(self, pm, messages, refinement_type, custom_suffix): user_input = self.microservice_specification.task - messages = [ - SystemMessage(content=system_task_introduction + template_init), - ] num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) @@ -449,7 +454,8 @@ Test scenario: agent_response_raw = conversation.chat( template_refinement.format( user_input=user_input, - _optional_test=' test' if refinement_type == 'test' else '' + _optional_test=' test' if refinement_type == 'test' else '', + custom_suffix=custom_suffix, ), role='user' ) @@ -457,6 +463,7 @@ Test scenario: agent_question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False) if final: + messages.append(AIMessage(content=final)) setattr(self.microservice_specification, refinement_type, final) break elif agent_question: diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 41d1cba..7fd4685 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -96,7 +96,7 @@ system_test_iteration = f''' The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: -a) If unit test requires an example input file as input: +a) If the input of the unit test requires an example input file or e.g. a certain textual description that is not mentioned before: In this case you must ask the client to provide the example input file as URL. You must not accept files that are not URLs. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): @@ -107,60 +107,42 @@ Your response must exactly match the following block code format (double asteris ``` If you did a, you must not do b. -b) Any strings, ints, or bools can be used as input for the unit test. +b) If the input can be determined from the previous messages: In this case you must describe the unit test verbally. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **final.txt** ```text - +input: "" +assertion of output: "" ``` If you did b, you must not do a. -Example 1: -Client: -**client-response.txt** -``` -given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22 -``` -PM: +Example for: "given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22": **final.txt** ```text -The test takes the city "Berlin" as input and asserts that the weather report for the next 5 days exists in the response. +input: "Berlin" +assertion of output: "contains weather report for the next 5 days" ``` -Example 2: -Client: -**client-response.txt** -``` -The user inserts a png and gets an svg as response. -``` -PM: +Example for "The user inserts a png and gets an svg as response.": **prompt.txt** ```text Please provide a png example input file as url. ``` -Client: -**client-response.txt** -``` -https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png -``` -PM: + +Example for "The user inserts a png like https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png and gets an svg as response.": **final.txt** ```text -The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg. +input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png" +assertion of output: "is an svg" ``` -Example 3: -Client: -**client-response.txt** -``` -The microservice takes nothing as input and returns the current time. -``` -PM: +Example for "The microservice takes nothing as input and returns the current time.": **final.txt** ```text -The test takes nothing as input and asserts that the output is a string. +input: "nothing" +assertion of output: "is a string" ``` ''' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9e078c3..9818865 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -368,10 +368,12 @@ Or write the summarized microservice{_optional_test} description like this: **final.txt** ```text -``` +``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. +Note that the last sequence of characters in your response must be ``` (triple backtick). Note that prompt.txt must not only contain one question. Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary. +{custom_suffix} ''' ) diff --git a/test/test_generator.py b/test/test_generator.py index de7c866..298cd1c 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -40,11 +40,12 @@ def test_generation_level_1(tmpdir): generator = Generator( ''' Input is a tweet that might contain passive aggressive language like: -'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 But hey, at least SOMEONE's enjoying their lunch. #officelife' +'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 +But hey, at least SOMEONE's enjoying their lunch. #officelife' The output is a tweet that is not passive aggressive like: 'Hi coworker, I hope you're having an amazing day! -Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. +Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. 😜 If you're up for trying different lunch options, that could be a fun way to mix things up. Enjoy your day! #variety' ''',