aljaz/dev-gpt
1
0
Fork 0
mirror of https://github.com/aljazceru/dev-gpt.git synced 2026-01-23 23:44:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

🧪 test: level 2

Browse Source
This commit is contained in:
Florian Hönicke
2023-04-30 01:58:58 +02:00
parent 81b67851da
commit 271782a1cf
4 changed files with 36 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/options/generate/generator.py
View File

@@ -44,7 +44,7 @@ class Generator:
def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True):
optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break
pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```"
pattern = fr"^\*?\*?{file_name}\*?\*?\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```"
match = re.search(pattern, plain_text, re.MULTILINE)
if match:
return match.group(1).strip()
@@ -422,8 +422,16 @@ gptdeploy deploy --path {self.microservice_root_path}
if not original_task:
self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?')
self.refine_requirements(pm, system_task_iteration, 'task')
self.refine_requirements(pm, system_test_iteration, 'test')
messages = [SystemMessage(content=system_task_introduction + system_task_iteration)]
self.refine_requirements(pm, messages, 'task', '')
messages[0] = SystemMessage(content=system_task_introduction + system_test_iteration)
self.refine_requirements(
pm,
messages,
'test',
'''Note that the test scenario must not contain information that was already mentioned in the microservice description.
Note that if the test scenario must contain the full description of the concrete example in case it was mentioned in the microservice description.'''
)
break
except self.TaskRefinementException as e:
@@ -437,11 +445,8 @@ Test scenario:
{self.microservice_specification.test}
''')
def refine_requirements(self, pm, template_init, refinement_type):
def refine_requirements(self, pm, messages, refinement_type, custom_suffix):
user_input = self.microservice_specification.task
messages = [
SystemMessage(content=system_task_introduction + template_init),
]
num_parsing_tries = 0
while True:
conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False)
@@ -449,7 +454,8 @@ Test scenario:
agent_response_raw = conversation.chat(
template_refinement.format(
user_input=user_input,
_optional_test=' test' if refinement_type == 'test' else ''
_optional_test=' test' if refinement_type == 'test' else '',
custom_suffix=custom_suffix,
),
role='user'
)
@@ -457,6 +463,7 @@ Test scenario:
agent_question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False)
final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False)
if final:
messages.append(AIMessage(content=final))
setattr(self.microservice_specification, refinement_type, final)
break
elif agent_question:

48
src/options/generate/templates_system.py
View File

@@ -96,7 +96,7 @@ system_test_iteration = f'''
The client gives you a description of the microservice (web service).
Your task is to describe verbally a unit test for that microservice.
There are two cases:
a) If unit test requires an example input file as input:
a) If the input of the unit test requires an example input file or e.g. a certain textual description that is not mentioned before:
In this case you must ask the client to provide the example input file as URL.
You must not accept files that are not URLs.
Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block):
@@ -107,60 +107,42 @@ Your response must exactly match the following block code format (double asteris
```
If you did a, you must not do b.
b) Any strings, ints, or bools can be used as input for the unit test.
b) If the input can be determined from the previous messages:
In this case you must describe the unit test verbally.
Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block):
**final.txt**
```text
<task here>
input: "<input here>"
assertion of output: "<assertion of output here>"
```
If you did b, you must not do a.
Example 1:
Client:
**client-response.txt**
```
given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22
```
PM:
Example for: "given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22":
**final.txt**
```text
The test takes the city "Berlin" as input and asserts that the weather report for the next 5 days exists in the response.
input: "Berlin"
assertion of output: "contains weather report for the next 5 days"
```
Example 2:
Client:
**client-response.txt**
```
The user inserts a png and gets an svg as response.
```
PM:
Example for "The user inserts a png and gets an svg as response.":
**prompt.txt**
```text
Please provide a png example input file as url.
```
Client:
**client-response.txt**
```
https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png
```
PM:
Example for "The user inserts a png like https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png and gets an svg as response.":
**final.txt**
```text
The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg.
input: "https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png"
assertion of output: "is an svg"
```
Example 3:
Client:
**client-response.txt**
```
The microservice takes nothing as input and returns the current time.
```
PM:
Example for "The microservice takes nothing as input and returns the current time.":
**final.txt**
```text
The test takes nothing as input and asserts that the output is a string.
input: "nothing"
assertion of output: "is a string"
```
'''

4
src/options/generate/templates_user.py
View File

@@ -368,10 +368,12 @@ Or write the summarized microservice{_optional_test} description like this:
**final.txt**
```text
<microservice{_optional_test} description here>
```
```
Note that your response must be either prompt.txt or final.txt. You must not write both.
Note that you must obey the double asterisk and tripple backtick syntax from above.
Note that the last sequence of characters in your response must be ``` (triple backtick).
Note that prompt.txt must not only contain one question.
Note that if urls, secrets, database names, etc. are mentioned, they must be part of the summary.
{custom_suffix}
'''
)

5
test/test_generator.py
View File

@@ -40,11 +40,12 @@ def test_generation_level_1(tmpdir):
generator = Generator(
'''
Input is a tweet that might contain passive aggressive language like:
'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢 But hey, at least SOMEONE's enjoying their lunch. #officelife'
'When your coworker microwaves fish in the break room... AGAIN. 🐟🤢
But hey, at least SOMEONE's enjoying their lunch. #officelife'
The output is a tweet that is not passive aggressive like:
'Hi coworker,
I hope you're having an amazing day!
Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room.
Just a quick note: sometimes microwaving fish can create an interesting aroma in the break room. 😜
If you're up for trying different lunch options, that could be a fun way to mix things up.
Enjoy your day! #variety'
''',