From 0153f5c3bfb1424aab5c9a7875c17dcc05686ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 10:11:16 +0200 Subject: [PATCH 01/22] =?UTF-8?q?=F0=9F=91=A8=E2=80=8D=F0=9F=92=BC?= =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:=20pm=20role?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/gpt.py | 69 ++++----- src/cli.py | 4 +- src/options/generate/generator.py | 238 ++++++++++++++++++++++++++++-- src/options/generate/ui.py | 82 ++++++++++ 4 files changed, 342 insertions(+), 51 deletions(-) create mode 100644 src/options/generate/ui.py diff --git a/src/apis/gpt.py b/src/apis/gpt.py index a3ec8e3..807f917 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -8,7 +8,7 @@ from langchain import PromptTemplate from langchain.callbacks import CallbackManager from langchain.chat_models import ChatOpenAI from openai.error import RateLimitError -from langchain.schema import HumanMessage, SystemMessage, BaseMessage +from langchain.schema import HumanMessage, SystemMessage, BaseMessage, AIMessage from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from requests.exceptions import ConnectionError from urllib3.exceptions import InvalidChunkLength @@ -48,13 +48,12 @@ class GPTSession: self.chars_prompt_so_far = 0 self.chars_generation_so_far = 0 - def get_conversation(self, system_definition_examples: List[str] = ['gpt', 'executor', 'docarray', 'client']): + def get_conversation(self, messages: List[BaseMessage] = [], print_stream: bool = True, print_costs: bool = True): return _GPTConversation( - self.model_name, self.cost_callback, self.task_description, self.test_description, system_definition_examples + self.model_name, self.cost_callback, messages, print_stream, print_costs ) - @staticmethod def is_gpt4_available(): try: @@ -75,14 +74,15 @@ class GPTSession: except openai.error.InvalidRequestError: return False - def cost_callback(self, chars_prompt, chars_generation): + def cost_callback(self, chars_prompt, chars_generation, print_costs: bool = True): self.chars_prompt_so_far += chars_prompt self.chars_generation_so_far += chars_generation - print('\n') - money_prompt = self._calculate_money_spent(self.chars_prompt_so_far, self.pricing_prompt) - money_generation = self._calculate_money_spent(self.chars_generation_so_far, self.pricing_generation) - print('Total money spent so far on openai.com:', f'${money_prompt + money_generation:.3f}') - print('\n') + if print_costs: + print('\n') + money_prompt = self._calculate_money_spent(self.chars_prompt_so_far, self.pricing_prompt) + money_generation = self._calculate_money_spent(self.chars_generation_so_far, self.pricing_generation) + print('Total money spent so far on openai.com:', f'${money_prompt + money_generation:.3f}') + print('\n') @staticmethod def _calculate_money_spent(num_chars, price): @@ -96,29 +96,39 @@ class AssistantStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler): class _GPTConversation: - def __init__(self, model: str, cost_callback, task_description, test_description, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): + def __init__(self, model: str, cost_callback, messages: List[BaseMessage], print_stream, print_costs): self._chat = ChatOpenAI( model_name=model, streaming=True, - callback_manager=CallbackManager([AssistantStreamingStdOutCallbackHandler()]), + callback_manager=CallbackManager([AssistantStreamingStdOutCallbackHandler()] if print_stream else []), verbose=True, temperature=0, ) self.cost_callback = cost_callback - self.messages: List[BaseMessage] = [] - self.system_message = self._create_system_message(task_description, test_description, system_definition_examples) - if os.environ['VERBOSE'].lower() == 'true': - print_colored('system', self.system_message.content, 'magenta') + self.messages = messages + self.print_stream = print_stream + self.print_costs = print_costs + for message in messages: + if os.environ['VERBOSE'].lower() == 'true': + if isinstance(message, SystemMessage): + print_colored('system - prompt', message.content, 'magenta') + elif isinstance(message, HumanMessage): + print_colored('user - prompt', message.content, 'blue') + elif isinstance(message, AIMessage): + print_colored('assistant - prompt', message.content, 'green') - def chat(self, prompt: str): - chat_message = HumanMessage(content=prompt) + def chat(self, prompt: str, role: str = 'user'): + MassageClass = HumanMessage if role == 'user' else SystemMessage + chat_message = MassageClass(content=prompt) self.messages.append(chat_message) if os.environ['VERBOSE'].lower() == 'true': - print_colored('user', prompt, 'blue') - print_colored('assistant', '', 'green', end='') + color = 'blue' if role == 'user' else 'magenta' + print_colored(role, prompt, color) + if self.print_stream: + print_colored('assistant', '', 'green', end='') for i in range(10): try: - response = self._chat([self.system_message] + self.messages) + response = self._chat(self.messages) break except (ConnectionError, InvalidChunkLength) as e: print('There was a connection error. Retrying...') @@ -128,22 +138,7 @@ class _GPTConversation: if os.environ['VERBOSE'].lower() == 'true': print() - self.cost_callback(sum([len(m.content) for m in self.messages]), len(response.content)) + self.cost_callback(sum([len(m.content) for m in self.messages]), len(response.content), self.print_costs) self.messages.append(response) return response.content - @staticmethod - def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: - system_message = PromptTemplate.from_template(template_system_message_base).format( - task_description=task_description, - test_description=test_description, - ) - if 'gpt' in system_definition_examples: - system_message += f'\n{gpt_example}' - if 'executor' in system_definition_examples: - system_message += f'\n{executor_example}' - if 'docarray' in system_definition_examples: - system_message += f'\n{docarray_example}' - if 'client' in system_definition_examples: - system_message += f'\n{client_example}' - return SystemMessage(content=system_message) diff --git a/src/cli.py b/src/cli.py index 13ecd75..5b8d763 100644 --- a/src/cli.py +++ b/src/cli.py @@ -50,8 +50,8 @@ def main(ctx): @openai_api_key_needed @main.command() -@click.option('--description', required=True, help='Description of the microservice.') -@click.option('--test', required=True, help='Test scenario for the microservice.') +@click.option('--description', required=False, help='Description of the microservice.') +@click.option('--test', required=False, help='Test scenario for the microservice.') @click.option('--model', default='gpt-4', help='GPT model to use (default: gpt-4).') @click.option('--verbose', default=False, is_flag=True, help='Verbose mode.') # only for development @path_param diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 26c5360..b0d0006 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -2,30 +2,46 @@ import os import random import re import shutil -from typing import List +from typing import List, Text, Optional + +from langchain import PromptTemplate +from langchain.schema import SystemMessage, HumanMessage, AIMessage +from pydantic.dataclasses import dataclass from src.apis import gpt from src.apis.jina_cloud import process_error_message, push_executor, is_executor_in_hub from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \ PROBLEMATIC_PACKAGES, EXECUTOR_FILE_NAME, EXECUTOR_FILE_TAG, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, DOCKER_FILE_TAG, UNNECESSARY_PACKAGES +from src.options.generate.templates_system import template_system_message_base, gpt_example, executor_example, \ + docarray_example, client_example from src.options.generate.templates_user import template_generate_microservice_name, \ template_generate_possible_packages, \ template_solve_code_issue, \ template_solve_dependency_issue, template_is_dependency_issue, template_generate_playground, \ template_generate_executor, template_generate_test, template_generate_requirements, template_generate_dockerfile, \ template_chain_of_thought, template_summarize_error, template_generate_possible_packages_output_format_string +from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path from src.utils.string_tools import print_colored +@dataclass +class TaskSpecification: + task: Optional[Text] + test: Optional[Text] + +system_task_introduction = f''' +You are a product manager who refines the requirements of a client who wants to create a microservice. +''' + class Generator: def __init__(self, task_description, test_description, model='gpt-4'): self.gpt_session = gpt.GPTSession(task_description, test_description, model=model) - self.task_description = task_description - self.test_description = test_description + self.microservice_specification = TaskSpecification(task=task_description, test=test_description) def extract_content_from_result(self, plain_text, file_name, match_single_block=False): + pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?)\n```" # the \n at the end makes sure that ``` within the generated code is not matched match = re.search(pattern, plain_text, re.MULTILINE) if match: @@ -56,9 +72,23 @@ metas: return all_microservice_files_string.strip() - def generate_and_persist_file(self, section_title, template, destination_folder=None, file_name=None, system_definition_examples: List[str] = ['gpt', 'executor', 'docarray', 'client'], **template_kwargs): + def generate_and_persist_file( + self, + section_title, + template, + destination_folder=None, + file_name=None, + system_definition_examples: List[str] = ['gpt', 'executor', 'docarray', 'client'], + **template_kwargs + ): + """ + Generates a file using the GPT-3 API and persists it to the destination folder if specified. + In case the content is not properly generated, it retries the generation. + It returns the generated content. + """ print_colored('', f'\n\n############# {section_title} #############', 'blue') - conversation = self.gpt_session.get_conversation(system_definition_examples=system_definition_examples) + system_introduction_message = self._create_system_message(self.microservice_specification.task, self.microservice_specification.test, system_definition_examples) + conversation = self.gpt_session.get_conversation(messages=[system_introduction_message]) template_kwargs = {k: v for k, v in template_kwargs.items() if k in template.input_variables} content_raw = conversation.chat( template.format( @@ -91,8 +121,8 @@ metas: template_generate_executor, MICROSERVICE_FOLDER_v1, microservice_name=microservice_name, - microservice_description=self.task_description, - test_description=self.test_description, + microservice_description=self.microservice_specification.task, + test_description=self.microservice_specification.test, packages=packages, file_name_purpose=EXECUTOR_FILE_NAME, tag_name=EXECUTOR_FILE_TAG, @@ -105,8 +135,8 @@ metas: MICROSERVICE_FOLDER_v1, code_files_wrapped=self.files_to_string({'microservice.py': microservice_content}), microservice_name=microservice_name, - microservice_description=self.task_description, - test_description=self.test_description, + microservice_description=self.microservice_specification.task, + test_description=self.microservice_specification.test, file_name_purpose=TEST_EXECUTOR_FILE_NAME, tag_name=TEST_EXECUTOR_FILE_TAG, file_name=TEST_EXECUTOR_FILE_NAME, @@ -235,7 +265,7 @@ metas: ) else: user_query = template_solve_code_issue.format( - task_description=self.task_description, test_description=self.test_description, + task_description=self.microservice_specification.task, test_description=self.microservice_specification.test, summarized_error=summarized_error, all_files_string=self.files_to_string(file_name_to_content), ) conversation = self.gpt_session.get_conversation() @@ -276,7 +306,7 @@ metas: None, file_name='packages.csv', system_definition_examples=['gpt'], - description=self.task_description + description=self.microservice_specification.task ) packages_list = [[pkg.strip() for pkg in packages_string.split(',')] for packages_string in packages_csv_string.split('\n')] @@ -284,7 +314,8 @@ metas: return packages_list def generate(self, microservice_path): - generated_name = self.generate_microservice_name(self.task_description) + self.refine_specification() + generated_name = self.generate_microservice_name(self.microservice_specification.task) microservice_name = f'{generated_name}{random.randint(0, 10_000_000)}' packages_list = self.get_possible_packages() packages_list = [ @@ -320,3 +351,186 @@ gptdeploy deploy --path {microservice_path} error_summary = conversation.chat(template_summarize_error.format(error=error)) return error_summary + def refine_specification(self): + pm = get_random_employee('pm') + print(f'{pm.emoji}👋 Hi, I\'m {pm.name}, a PM at Jina AI. Gathering the requirements for our engineers.') + self.refine_task(pm) + self.refine_test(pm) + print(f''' +{pm.emoji} 👍 Great, I will handover the following requirements to our engineers: +{self.microservice_specification.task} +The following test scenario will be tested: +{self.microservice_specification.test} +''') + + def refine_task(self, pm): + system_task_iteration = f''' +The client writes a description of the microservice. +You must only talk to the client about the microservice. +You must not output anything else than what you got told in the following steps. +1. +You must create a check list for the requirements of the microservice. +Input and output have to be accurately specified. +You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: +input: +output: +credentials: +database access: + +2. +You must do either a or b. +a) +If the description is not sufficiently specified, then ask for the missing information. +Your response must exactly match the following block code format: + +**prompt.txt** +```text + +``` + +b) +Otherwise you respond with the summarized description. +The summarized description must contain all the information mentioned by the client. +Your response must exactly match the following block code format: + +**task-final.txt** +```text + +``` <-- this is in a new line + +The character sequence ``` must always be at the beginning of the line. +You must not add information that was not provided by the client. + + +Example for the description "given a city, get the weather report for the next 5 days": +input: ✅ +output: ✅ +credentials: ❌ +database access: n/a + +**prompt.txt** +```text +Please provide the url of the weather api and a valid api key. Or let our engineers try to find a free api. +``` + + +Example for the description "convert png to svg" +input: ✅ +output: ✅ +credentials: n/a +database access: n/a + +**task-final.txt** +```text +The user inserts a png and gets an svg as response. +``` + + +Example for the description "parser" +input: ❌ +output: ❌ +credentials: n/a +database access: n/a + +**prompt.txt** +```text +Please provide the input and output format. +``` + +''' + + + + task_description = self.microservice_specification.task + if not task_description: + task_description = self.get_user_input(pm, 'What should your microservice do?') + messages = [ + SystemMessage(content=system_task_introduction + system_task_iteration), + ] + + while True: + conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) + print('thinking...') + agent_response_raw = conversation.chat(task_description, role='user') + + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt') + task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt') + if task_final: + self.microservice_specification.task = task_final + break + if question: + task_description = self.get_user_input(pm, question) + messages.extend([HumanMessage(content=task_description)]) + else: + task_description = self.get_user_input(pm, agent_response_raw + '\n: ') + + def refine_test(self, pm): + system_test_iteration = f''' +The client gives you a description of the microservice. +Your task is to describe verbally a unit test for that microservice. +There are two cases: +a) The unit test requires a file as input. +In this case you must ask the client to provide the file as URL. +Your response must exactly match the following block code format: + +**prompt.txt** +```text + +``` + +If you did a, you must not do b. +b) Any strings, ints, or bools can be used as input for the unit test. +In this case you must describe the unit test verbally. +Your response must exactly match the following block code format: + +**test-final.txt** +```text + +``` + +If you did b, you must not do a. + +Example for the description "given a city, get the weather report for the next 5 days using the ap": +''' + messages = [ + SystemMessage(content=system_task_introduction + system_test_iteration), + ] + user_input = self.microservice_specification.task + while True: + conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) + agent_response_raw = conversation.chat(user_input, role='user') + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt') + test_final = self.extract_content_from_result(agent_response_raw, 'test-final.txt') + if test_final: + self.microservice_specification.task = test_final + break + if question: + user_input = self.get_user_input(pm, question) + messages.extend([HumanMessage(content=user_input)]) + else: + user_input = self.get_user_input(pm, agent_response_raw + '\n: ') + + + + @staticmethod + def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: + system_message = PromptTemplate.from_template(template_system_message_base).format( + task_description=task_description, + test_description=test_description, + ) + if 'gpt' in system_definition_examples: + system_message += f'\n{gpt_example}' + if 'executor' in system_definition_examples: + system_message += f'\n{executor_example}' + if 'docarray' in system_definition_examples: + system_message += f'\n{docarray_example}' + if 'client' in system_definition_examples: + system_message += f'\n{client_example}' + return SystemMessage(content=system_message) + + @staticmethod + def get_user_input(employee, prompt_to_user): + val = input(f'{employee.emoji}❓ {prompt_to_user}\nyou: ') + while not val: + val = input('you: ') + return val diff --git a/src/options/generate/ui.py b/src/options/generate/ui.py new file mode 100644 index 0000000..81d7a75 --- /dev/null +++ b/src/options/generate/ui.py @@ -0,0 +1,82 @@ +import random +from dataclasses import dataclass + +product_manager_names = [ + ('Leon', 'm'), + ('Saahil', 'm',), + ('Susana', 'f') +] +engineer_names = [ + ('Joschka', 'm'), + ('Johannes', 'm'), + ('Johanna', 'f'), + ('Aaron', 'm'), + ('Alaeddine', 'm'), + ('Andrei', 'm'), + ('Anne', 'f'), + ('Bo', 'm'), + ('Charlotte', 'f'), + ('David', 'm'), + ('Deepankar', 'm'), + ('Delgermurun', 'm'), + ('Edward', 'm'), + ('Felix', 'm'), + ('Florian', 'm'), + ('Georgios', 'm'), + ('Girish', 'm'), + ('Guillaume', 'm'), + ('Isabelle', 'f'), + ('Jackmin', 'm'), + ('Jie', 'm'), + ('Joan', 'm'), + ('Johannes', 'm'), + ('Joschka', 'm'), + ('Lechun', 'm'), + ('Louis', 'm'), + ('Mark', 'm'), + ('Maximilian', 'm'), + ('Michael', 'm'), + ('Mohamed Aziz', 'm'), + ('Mohammad Kalim', 'm'), + ('Nikos', 'm'), + ('Ran', 'm'), + ('Saba', 'f'), + ('Sami', 'm'), + ('Sha', 'm'), + ('Subba Reddy', 'm'), + ('Tanguy', 'm'), + ('Winston', 'm'), + ('Yadh', 'm'), + ('Yanlong', 'm'), + ('Zac', 'm'), + ('Zhaofeng', 'm'), + ('Zihao', 'm'), + ('Ziniu', 'm') +] + +role_to_gender_to_emoji = { + 'engineer':{ + 'm': '👨‍💻', + 'f': '👩‍💻' + }, + 'pm': { + 'm': '👨‍💼', + 'f': '👩‍💼' + }, + 'qa_endineer': { + 'm': '👨‍🔧', + 'f': '👩‍🔧', + }, +} + +@dataclass +class Employee: + role: str + name: str + gender: str + emoji: str + +def get_random_employee(role: str) -> Employee: + name, gender = random.choice(product_manager_names) + emoji = role_to_gender_to_emoji[role][gender] + return Employee(role, name, gender, emoji) From 864340790b84bace334d24ea3e53867054cc76db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 11:42:30 +0200 Subject: [PATCH 02/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC?= =?UTF-8?q?=F0=9F=91=A8=E2=80=8D=F0=9F=92=BC=20feat:=20pm=20role?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 121 ++------------------ src/options/generate/templates_system.py | 139 +++++++++++++++++++++++ 2 files changed, 148 insertions(+), 112 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index b0d0006..c9cad85 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -14,7 +14,7 @@ from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX PROBLEMATIC_PACKAGES, EXECUTOR_FILE_NAME, EXECUTOR_FILE_TAG, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, DOCKER_FILE_TAG, UNNECESSARY_PACKAGES from src.options.generate.templates_system import template_system_message_base, gpt_example, executor_example, \ - docarray_example, client_example + docarray_example, client_example, system_task_iteration, system_task_introduction, system_test_iteration from src.options.generate.templates_user import template_generate_microservice_name, \ template_generate_possible_packages, \ template_solve_code_issue, \ @@ -31,10 +31,6 @@ class TaskSpecification: task: Optional[Text] test: Optional[Text] -system_task_introduction = f''' -You are a product manager who refines the requirements of a client who wants to create a microservice. -''' - class Generator: def __init__(self, task_description, test_description, model='gpt-4'): self.gpt_session = gpt.GPTSession(task_description, test_description, model=model) @@ -358,88 +354,13 @@ gptdeploy deploy --path {microservice_path} self.refine_test(pm) print(f''' {pm.emoji} 👍 Great, I will handover the following requirements to our engineers: +Description of the microservice: {self.microservice_specification.task} -The following test scenario will be tested: +Test scenario: {self.microservice_specification.test} ''') def refine_task(self, pm): - system_task_iteration = f''' -The client writes a description of the microservice. -You must only talk to the client about the microservice. -You must not output anything else than what you got told in the following steps. -1. -You must create a check list for the requirements of the microservice. -Input and output have to be accurately specified. -You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: -input: -output: -credentials: -database access: - -2. -You must do either a or b. -a) -If the description is not sufficiently specified, then ask for the missing information. -Your response must exactly match the following block code format: - -**prompt.txt** -```text - -``` - -b) -Otherwise you respond with the summarized description. -The summarized description must contain all the information mentioned by the client. -Your response must exactly match the following block code format: - -**task-final.txt** -```text - -``` <-- this is in a new line - -The character sequence ``` must always be at the beginning of the line. -You must not add information that was not provided by the client. - - -Example for the description "given a city, get the weather report for the next 5 days": -input: ✅ -output: ✅ -credentials: ❌ -database access: n/a - -**prompt.txt** -```text -Please provide the url of the weather api and a valid api key. Or let our engineers try to find a free api. -``` - - -Example for the description "convert png to svg" -input: ✅ -output: ✅ -credentials: n/a -database access: n/a - -**task-final.txt** -```text -The user inserts a png and gets an svg as response. -``` - - -Example for the description "parser" -input: ❌ -output: ❌ -credentials: n/a -database access: n/a - -**prompt.txt** -```text -Please provide the input and output format. -``` - -''' - - task_description = self.microservice_specification.task if not task_description: @@ -465,44 +386,21 @@ Please provide the input and output format. task_description = self.get_user_input(pm, agent_response_raw + '\n: ') def refine_test(self, pm): - system_test_iteration = f''' -The client gives you a description of the microservice. -Your task is to describe verbally a unit test for that microservice. -There are two cases: -a) The unit test requires a file as input. -In this case you must ask the client to provide the file as URL. -Your response must exactly match the following block code format: - -**prompt.txt** -```text - -``` - -If you did a, you must not do b. -b) Any strings, ints, or bools can be used as input for the unit test. -In this case you must describe the unit test verbally. -Your response must exactly match the following block code format: - -**test-final.txt** -```text - -``` - -If you did b, you must not do a. - -Example for the description "given a city, get the weather report for the next 5 days using the ap": -''' messages = [ SystemMessage(content=system_task_introduction + system_test_iteration), ] user_input = self.microservice_specification.task while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) - agent_response_raw = conversation.chat(user_input, role='user') + agent_response_raw = conversation.chat(f'''**client-response.txt** +``` +{user_input} +``` +''', role='user') question = self.extract_content_from_result(agent_response_raw, 'prompt.txt') test_final = self.extract_content_from_result(agent_response_raw, 'test-final.txt') if test_final: - self.microservice_specification.task = test_final + self.microservice_specification.test = test_final break if question: user_input = self.get_user_input(pm, question) @@ -511,7 +409,6 @@ Example for the description "given a city, get the weather report for the next 5 user_input = self.get_user_input(pm, agent_response_raw + '\n: ') - @staticmethod def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: system_message = PromptTemplate.from_template(template_system_message_base).format( diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index ce36bef..474b6ba 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -107,3 +107,142 @@ and the following test scenario: You must obey the following rules: {not_allowed_executor_string} {not_allowed_docker_string}''' + +system_task_introduction = f''' +You are a product manager who refines the requirements of a client who wants to create a microservice. +''' + +system_task_iteration = ''' +The client writes a description of the microservice. +You must only talk to the client about the microservice. +You must not output anything else than what you got told in the following steps. +1. +You must create a check list for the requirements of the microservice. +Input and output have to be accurately specified. +You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: +input: +output: +credentials: +database access: + +2. +You must do either a or b. +a) +If the description is not sufficiently specified, then ask for the missing information. +Your response must exactly match the following block code format: + +**prompt.txt** +```text + +``` + +b) +Otherwise you respond with the summarized description. +The summarized description must contain all the information mentioned by the client. +Your response must exactly match the following block code format: + +**task-final.txt** +```text + +``` <-- this is in a new line + +The character sequence ``` must always be at the beginning of the line. +You must not add information that was not provided by the client. + + +Example for the description "given a city, get the weather report for the next 5 days": +input: ✅ +output: ✅ +credentials: ❌ +database access: n/a + +**prompt.txt** +```text +Please provide the url of the weather api and a valid api key. Or let our engineers try to find a free api. +``` + + +Example for the description "convert png to svg": +input: ✅ +output: ✅ +credentials: n/a +database access: n/a + +**task-final.txt** +```text +The user inserts a png and gets an svg as response. +``` + + +Example for the description "parser": +input: ❌ +output: ❌ +credentials: n/a +database access: n/a + +**prompt.txt** +```text +Please provide the input and output format. +``` + +''' + +system_test_iteration = f''' +The client gives you a description of the microservice. +Your task is to describe verbally a unit test for that microservice. +There are two cases: +a) The unit test requires a file as input. +In this case you must ask the client to provide the file as URL. +Your response must exactly match the following block code format: + +**prompt.txt** +```text + +``` + +If you did a, you must not do b. +b) Any strings, ints, or bools can be used as input for the unit test. +In this case you must describe the unit test verbally. +Your response must exactly match the following block code format: + +**test-final.txt** +```text + +``` + +If you did b, you must not do a. + +Example 1: +Client: +**client-response.txt** +``` +given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22 +``` +PM: +**test-final.txt** +```text +The test takes the city "Berlin" as input and asserts that the weather report for the next 5 days exists in the response. +``` + +Example 2: +Client: +**client-response.txt** +``` +The user inserts a png and gets an svg as response. +``` +PM: +**prompt.txt** +```text +Please provide a png file as url. +``` +Client: +**client-response.txt** +``` +https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png +``` +PM: +**test-final.txt** +```text +The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg. +``` +''' \ No newline at end of file From c5022f7d67b082c7bc71d60c8503bf0b01ea4ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 12:11:20 +0200 Subject: [PATCH 03/22] =?UTF-8?q?=E2=99=BB=20refactor:=20rename=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 16 +++++++-------- src/options/generate/templates_system.py | 25 +++++++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index c9cad85..4e5fef5 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -36,9 +36,9 @@ class Generator: self.gpt_session = gpt.GPTSession(task_description, test_description, model=model) self.microservice_specification = TaskSpecification(task=task_description, test=test_description) - def extract_content_from_result(self, plain_text, file_name, match_single_block=False): - - pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?)\n```" # the \n at the end makes sure that ``` within the generated code is not matched + def extract_content_from_result(self, plain_text, file_name, match_single_block=False, can_contain_code_block=True): + optional_line_break = '\n' if can_contain_code_block else '' # the \n at the end makes sure that ``` within the generated code is not matched because it is not right before a line break + pattern = fr"^\*\*{file_name}\*\*\n```(?:\w+\n)?([\s\S]*?){optional_line_break}```" match = re.search(pattern, plain_text, re.MULTILINE) if match: return match.group(1).strip() @@ -361,7 +361,6 @@ Test scenario: ''') def refine_task(self, pm): - task_description = self.microservice_specification.task if not task_description: task_description = self.get_user_input(pm, 'What should your microservice do?') @@ -374,8 +373,8 @@ Test scenario: print('thinking...') agent_response_raw = conversation.chat(task_description, role='user') - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt') - task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt') + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) + task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt', can_contain_code_block=False) if task_final: self.microservice_specification.task = task_final break @@ -392,13 +391,14 @@ Test scenario: user_input = self.microservice_specification.task while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) + print('thinking...') agent_response_raw = conversation.chat(f'''**client-response.txt** ``` {user_input} ``` ''', role='user') - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt') - test_final = self.extract_content_from_result(agent_response_raw, 'test-final.txt') + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) + test_final = self.extract_content_from_result(agent_response_raw, 'test-final.txt', can_contain_code_block=False) if test_final: self.microservice_specification.test = test_final break diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 474b6ba..d8f5493 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -144,12 +144,11 @@ Your response must exactly match the following block code format: **task-final.txt** ```text -``` <-- this is in a new line +``` The character sequence ``` must always be at the beginning of the line. You must not add information that was not provided by the client. - Example for the description "given a city, get the weather report for the next 5 days": input: ✅ output: ✅ @@ -161,7 +160,6 @@ database access: n/a Please provide the url of the weather api and a valid api key. Or let our engineers try to find a free api. ``` - Example for the description "convert png to svg": input: ✅ output: ✅ @@ -173,7 +171,6 @@ database access: n/a The user inserts a png and gets an svg as response. ``` - Example for the description "parser": input: ❌ output: ❌ @@ -191,8 +188,8 @@ system_test_iteration = f''' The client gives you a description of the microservice. Your task is to describe verbally a unit test for that microservice. There are two cases: -a) The unit test requires a file as input. -In this case you must ask the client to provide the file as URL. +a) The unit test requires an example file as input. +In this case you must ask the client to provide the example file as URL. Your response must exactly match the following block code format: **prompt.txt** @@ -233,7 +230,7 @@ The user inserts a png and gets an svg as response. PM: **prompt.txt** ```text -Please provide a png file as url. +Please provide a png example file as url. ``` Client: **client-response.txt** @@ -245,4 +242,18 @@ PM: ```text The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg. ``` + +Example 3: +Client: +**client-response.txt** +``` +The microservice takes nothing as input and returns the current time. +``` +PM: +**test-final.txt** +```text +The test takes nothing as input and asserts that the output is a string. +``` + + ''' \ No newline at end of file From dbacfd6511e4696c03f0f21f3d78701904967c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 14:04:59 +0200 Subject: [PATCH 04/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/gpt.py | 19 ------------------- src/options/generate/generator.py | 21 +++++++++++---------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index 29c41d3..db11f52 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -141,22 +141,3 @@ class _GPTConversation: self.cost_callback(sum([len(m.content) for m in self.messages]), len(response.content), self.print_costs) self.messages.append(response) return response.content - - @staticmethod - def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: - if system_definition_examples is None: - return None - - system_message = PromptTemplate.from_template(template_system_message_base).format( - task_description=task_description, - test_description=test_description, - ) - if 'gpt' in system_definition_examples: - system_message += f'\n{gpt_example}' - if 'executor' in system_definition_examples: - system_message += f'\n{executor_example}' - if 'docarray' in system_definition_examples: - system_message += f'\n{docarray_example}' - if 'client' in system_definition_examples: - system_message += f'\n{client_example}' - return SystemMessage(content=system_message) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 131f2d4..b6eeb21 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -2,32 +2,29 @@ import os import random import re import shutil -from typing import List, Callable, Union +from typing import Callable, Union -from langchain import PromptTemplate from typing import List, Text, Optional from langchain import PromptTemplate -from langchain.schema import SystemMessage, HumanMessage, AIMessage +from langchain.schema import SystemMessage, HumanMessage from pydantic.dataclasses import dataclass from src.apis import gpt from src.apis.jina_cloud import process_error_message, push_executor, is_executor_in_hub from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \ PROBLEMATIC_PACKAGES, EXECUTOR_FILE_NAME, EXECUTOR_FILE_TAG, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ - REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, DOCKER_FILE_TAG, UNNECESSARY_PACKAGES + REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, UNNECESSARY_PACKAGES from src.options.generate.templates_system import template_system_message_base, gpt_example, executor_example, \ docarray_example, client_example, system_task_iteration, system_task_introduction, system_test_iteration from src.options.generate.templates_user import template_generate_microservice_name, \ template_generate_possible_packages, \ template_solve_code_issue, \ - template_solve_pip_dependency_issue, template_is_dependency_issue, template_generate_playground, \ + template_solve_pip_dependency_issue, \ + template_generate_apt_get_install, template_solve_apt_get_dependency_issue, \ + template_is_dependency_issue, template_generate_playground, \ template_generate_executor, template_generate_test, template_generate_requirements, \ - template_chain_of_thought, template_summarize_error, \ - template_generate_apt_get_install, template_solve_apt_get_dependency_issue - template_solve_dependency_issue, template_is_dependency_issue, template_generate_playground, \ - template_generate_executor, template_generate_test, template_generate_requirements, template_generate_dockerfile, \ - template_chain_of_thought, template_summarize_error, template_generate_possible_packages_output_format_string + template_chain_of_thought, template_summarize_error from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path from src.utils.string_tools import print_colored @@ -376,6 +373,7 @@ metas: return packages_list def generate(self): + self.refine_specification() os.makedirs(self.microservice_root_path) generated_name = self.generate_microservice_name(self.microservice_specification.task) microservice_name = f'{generated_name}{random.randint(0, 10_000_000)}' @@ -469,6 +467,9 @@ Test scenario: @staticmethod def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: + if system_definition_examples is None: + return None + system_message = PromptTemplate.from_template(template_system_message_base).format( task_description=task_description, test_description=test_description, From 4bbbb938f1ca0a7b10eebe7c4a08f8a2d07da09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 15:36:25 +0200 Subject: [PATCH 05/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20more=20robust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 16 ++++++++-------- src/options/generate/templates_system.py | 8 ++++---- src/options/generate/templates_user.py | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index b6eeb21..ccc4243 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -24,7 +24,7 @@ from src.options.generate.templates_user import template_generate_microservice_n template_generate_apt_get_install, template_solve_apt_get_dependency_issue, \ template_is_dependency_issue, template_generate_playground, \ template_generate_executor, template_generate_test, template_generate_requirements, \ - template_chain_of_thought, template_summarize_error + template_chain_of_thought, template_summarize_error, template_task_refinement from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path from src.utils.string_tools import print_colored @@ -417,9 +417,9 @@ Test scenario: ''') def refine_task(self, pm): - task_description = self.microservice_specification.task - if not task_description: - task_description = self.get_user_input(pm, 'What should your microservice do?') + user_input = self.microservice_specification.task + if not user_input: + user_input = self.get_user_input(pm, 'What should your microservice do?') messages = [ SystemMessage(content=system_task_introduction + system_task_iteration), ] @@ -427,7 +427,7 @@ Test scenario: while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) print('thinking...') - agent_response_raw = conversation.chat(task_description, role='user') + agent_response_raw = conversation.chat(template_task_refinement.format(user_input=user_input), role='user') question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt', can_contain_code_block=False) @@ -435,10 +435,10 @@ Test scenario: self.microservice_specification.task = task_final break if question: - task_description = self.get_user_input(pm, question) - messages.extend([HumanMessage(content=task_description)]) + user_input = self.get_user_input(pm, question) + messages.extend([HumanMessage(content=user_input)]) else: - task_description = self.get_user_input(pm, agent_response_raw + '\n: ') + user_input = self.get_user_input(pm, agent_response_raw + '\n: ') def refine_test(self, pm): messages = [ diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index d8f5493..06ed302 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -129,7 +129,7 @@ database access: You must do either a or b. a) If the description is not sufficiently specified, then ask for the missing information. -Your response must exactly match the following block code format: +Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **prompt.txt** ```text @@ -139,7 +139,7 @@ Your response must exactly match the following block code format: b) Otherwise you respond with the summarized description. The summarized description must contain all the information mentioned by the client. -Your response must exactly match the following block code format: +Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **task-final.txt** ```text @@ -190,7 +190,7 @@ Your task is to describe verbally a unit test for that microservice. There are two cases: a) The unit test requires an example file as input. In this case you must ask the client to provide the example file as URL. -Your response must exactly match the following block code format: +Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **prompt.txt** ```text @@ -200,7 +200,7 @@ Your response must exactly match the following block code format: If you did a, you must not do b. b) Any strings, ints, or bools can be used as input for the unit test. In this case you must describe the unit test verbally. -Your response must exactly match the following block code format: +Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **test-final.txt** ```text diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 6e26760..4e29753 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -356,3 +356,24 @@ The playground (app.py) must always use the host on http://localhost:8080 and mu The playground (app.py) must not import the executor. ''' ) + + +template_task_refinement = PromptTemplate.from_template( + ''' +**client response** +```text +{user_input} +``` +Either ask for clarification like this: +**prompt.txt** +```text + +``` + +Or write the summarized description like this: +**task-final.txt** +```text + +``` +''' +) From 2678a4bd1ea624857d2a01f12ed7b45ed1576651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Thu, 27 Apr 2023 18:21:14 +0200 Subject: [PATCH 06/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20more=20robust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/apis/gpt.py | 2 + src/options/generate/generator.py | 56 ++++++++++++++---------- src/options/generate/templates_system.py | 2 +- src/options/generate/templates_user.py | 2 +- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index db11f52..386e411 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -1,4 +1,5 @@ import os +from copy import deepcopy from time import sleep from typing import List, Any @@ -49,6 +50,7 @@ class GPTSession: self.chars_generation_so_far = 0 def get_conversation(self, messages: List[BaseMessage] = [], print_stream: bool = True, print_costs: bool = True): + messages = deepcopy(messages) return _GPTConversation( self.model_name, self.cost_callback, messages, print_stream, print_costs ) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index ccc4243..cff1f79 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -7,7 +7,7 @@ from typing import Callable, Union from typing import List, Text, Optional from langchain import PromptTemplate -from langchain.schema import SystemMessage, HumanMessage +from langchain.schema import SystemMessage, HumanMessage, AIMessage from pydantic.dataclasses import dataclass from src.apis import gpt @@ -327,6 +327,9 @@ metas: class MaxDebugTimeReachedException(BaseException): pass + class TaskRefinementException(BaseException): + pass + def is_dependency_issue(self, summarized_error, dock_req_string: str, package_manager: str): # a few heuristics to quickly jump ahead if any([error_message in summarized_error for error_message in ['AttributeError', 'NameError', 'AssertionError']]): @@ -417,34 +420,42 @@ Test scenario: ''') def refine_task(self, pm): - user_input = self.microservice_specification.task - if not user_input: - user_input = self.get_user_input(pm, 'What should your microservice do?') - messages = [ - SystemMessage(content=system_task_introduction + system_task_iteration), - ] + try: + user_input = self.microservice_specification.task + if not user_input: + user_input = self.get_user_input(pm, 'What should your microservice do?') + messages = [ + SystemMessage(content=system_task_introduction + system_task_iteration), + ] - while True: - conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) - print('thinking...') - agent_response_raw = conversation.chat(template_task_refinement.format(user_input=user_input), role='user') + while True: + conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) + print('thinking...') + agent_response_raw = conversation.chat(template_task_refinement.format(user_input=user_input), role='user') + + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) + task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt', can_contain_code_block=False) + if task_final: + self.microservice_specification.task = task_final + break + if question: + messages.append(HumanMessage(content=user_input),) + user_input = self.get_user_input(pm, question) + messages.append(AIMessage(content=question)) + elif task_final: + user_input = self.get_user_input(pm, agent_response_raw + '\n: ') + else: + raise self.TaskRefinementException() + except self.TaskRefinementException as e: + print_colored('', f'{pm.emoji} Could not refine the task. Please try again...', 'red') + self.refine_task(pm) - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) - task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt', can_contain_code_block=False) - if task_final: - self.microservice_specification.task = task_final - break - if question: - user_input = self.get_user_input(pm, question) - messages.extend([HumanMessage(content=user_input)]) - else: - user_input = self.get_user_input(pm, agent_response_raw + '\n: ') def refine_test(self, pm): + user_input = self.microservice_specification.task messages = [ SystemMessage(content=system_task_introduction + system_test_iteration), ] - user_input = self.microservice_specification.task while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) print('thinking...') @@ -487,6 +498,7 @@ Test scenario: @staticmethod def get_user_input(employee, prompt_to_user): val = input(f'{employee.emoji}❓ {prompt_to_user}\nyou: ') + print() while not val: val = input('you: ') return val diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 06ed302..beffbb8 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -185,7 +185,7 @@ Please provide the input and output format. ''' system_test_iteration = f''' -The client gives you a description of the microservice. +The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: a) The unit test requires an example file as input. diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 4e29753..850384d 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -370,7 +370,7 @@ Either ask for clarification like this: ``` -Or write the summarized description like this: +Or write the summarized microservice description like this: **task-final.txt** ```text From c417822eccdebe7622c4f968666683758d6110d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 10:28:58 +0200 Subject: [PATCH 07/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 80 ++++++++++-------------- src/options/generate/templates_system.py | 13 ++-- src/options/generate/templates_user.py | 8 +-- 3 files changed, 43 insertions(+), 58 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index cff1f79..05d9306 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -24,7 +24,7 @@ from src.options.generate.templates_user import template_generate_microservice_n template_generate_apt_get_install, template_solve_apt_get_dependency_issue, \ template_is_dependency_issue, template_generate_playground, \ template_generate_executor, template_generate_test, template_generate_requirements, \ - template_chain_of_thought, template_summarize_error, template_task_refinement + template_chain_of_thought, template_summarize_error, template_refinement from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path from src.utils.string_tools import print_colored @@ -409,8 +409,20 @@ gptdeploy deploy --path {self.microservice_root_path} def refine_specification(self): pm = get_random_employee('pm') print(f'{pm.emoji}👋 Hi, I\'m {pm.name}, a PM at Jina AI. Gathering the requirements for our engineers.') - self.refine_task(pm) - self.refine_test(pm) + original_task = self.microservice_specification.task + while True: + try: + self.microservice_specification.test = None + if not original_task: + self.microservice_specification.task = self.get_user_input(pm, 'What should your microservice do?') + + self.refine_requirements(pm, system_task_iteration, 'task') + self.refine_requirements(pm, system_test_iteration, 'test') + break + except self.TaskRefinementException as e: + + print_colored('', f'{pm.emoji} Could not refine your requirements. Please try again...', 'red') + print(f''' {pm.emoji} 👍 Great, I will handover the following requirements to our engineers: Description of the microservice: @@ -419,61 +431,33 @@ Test scenario: {self.microservice_specification.test} ''') - def refine_task(self, pm): - try: - user_input = self.microservice_specification.task - if not user_input: - user_input = self.get_user_input(pm, 'What should your microservice do?') - messages = [ - SystemMessage(content=system_task_introduction + system_task_iteration), - ] - - while True: - conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) - print('thinking...') - agent_response_raw = conversation.chat(template_task_refinement.format(user_input=user_input), role='user') - - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) - task_final = self.extract_content_from_result(agent_response_raw, 'task-final.txt', can_contain_code_block=False) - if task_final: - self.microservice_specification.task = task_final - break - if question: - messages.append(HumanMessage(content=user_input),) - user_input = self.get_user_input(pm, question) - messages.append(AIMessage(content=question)) - elif task_final: - user_input = self.get_user_input(pm, agent_response_raw + '\n: ') - else: - raise self.TaskRefinementException() - except self.TaskRefinementException as e: - print_colored('', f'{pm.emoji} Could not refine the task. Please try again...', 'red') - self.refine_task(pm) - - - def refine_test(self, pm): + def refine_requirements(self, pm, template_init, refinement_type): user_input = self.microservice_specification.task messages = [ - SystemMessage(content=system_task_introduction + system_test_iteration), + SystemMessage(content=system_task_introduction + template_init), ] while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) print('thinking...') - agent_response_raw = conversation.chat(f'''**client-response.txt** -``` -{user_input} -``` -''', role='user') + agent_response_raw = conversation.chat( + template_refinement.format( + user_input=user_input, + _optional_test=' test' if refinement_type == 'test' else '' + ), + role='user' + ) + question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) - test_final = self.extract_content_from_result(agent_response_raw, 'test-final.txt', can_contain_code_block=False) - if test_final: - self.microservice_specification.test = test_final + final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False) + if final: + setattr(self.microservice_specification, refinement_type, final) break - if question: + elif question: + messages.append(HumanMessage(content=user_input),) + messages.append(AIMessage(content=question)) user_input = self.get_user_input(pm, question) - messages.extend([HumanMessage(content=user_input)]) else: - user_input = self.get_user_input(pm, agent_response_raw + '\n: ') + raise self.TaskRefinementException() @staticmethod diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index beffbb8..cc27871 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -141,7 +141,7 @@ Otherwise you respond with the summarized description. The summarized description must contain all the information mentioned by the client. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): -**task-final.txt** +**final.txt** ```text ``` @@ -166,7 +166,7 @@ output: ✅ credentials: n/a database access: n/a -**task-final.txt** +**final.txt** ```text The user inserts a png and gets an svg as response. ``` @@ -190,6 +190,7 @@ Your task is to describe verbally a unit test for that microservice. There are two cases: a) The unit test requires an example file as input. In this case you must ask the client to provide the example file as URL. +You must not accept files that are not URLs. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): **prompt.txt** @@ -202,7 +203,7 @@ b) Any strings, ints, or bools can be used as input for the unit test. In this case you must describe the unit test verbally. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): -**test-final.txt** +**final.txt** ```text ``` @@ -216,7 +217,7 @@ Client: given a city, get the weather report for the next 5 days using OpenWeatherMap with the api key b6907d289e10d714a6e88b30761fae22 ``` PM: -**test-final.txt** +**final.txt** ```text The test takes the city "Berlin" as input and asserts that the weather report for the next 5 days exists in the response. ``` @@ -238,7 +239,7 @@ Client: https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png ``` PM: -**test-final.txt** +**final.txt** ```text The test takes the png https://aquasecurity.github.io/kube-bench/v0.6.5/images/kube-bench-logo-only.png as input and asserts the output is an svg. ``` @@ -250,7 +251,7 @@ Client: The microservice takes nothing as input and returns the current time. ``` PM: -**test-final.txt** +**final.txt** ```text The test takes nothing as input and asserts that the output is a string. ``` diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 850384d..9d3bbf9 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -358,7 +358,7 @@ The playground (app.py) must not import the executor. ) -template_task_refinement = PromptTemplate.from_template( +template_refinement = PromptTemplate.from_template( ''' **client response** ```text @@ -370,10 +370,10 @@ Either ask for clarification like this: ``` -Or write the summarized microservice description like this: -**task-final.txt** +Or write the summarized microservice{_optional_test} description like this: +**final.txt** ```text - + ``` ''' ) From 201aad4a1521b1bad80ecfcd6c6225441d12bcba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 10:36:56 +0200 Subject: [PATCH 08/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 1 - src/options/generate/ui.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 05d9306..e47551a 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -345,7 +345,6 @@ metas: return 'yes' in answer.lower() def generate_microservice_name(self, description): - print_colored('', '\n\n############# What should be the name of the Microservice? #############', 'blue') name = self.generate_and_persist_file( section_title='Generate microservice name', template=template_generate_microservice_name, diff --git a/src/options/generate/ui.py b/src/options/generate/ui.py index 81d7a75..71cf9a2 100644 --- a/src/options/generate/ui.py +++ b/src/options/generate/ui.py @@ -7,9 +7,6 @@ product_manager_names = [ ('Susana', 'f') ] engineer_names = [ - ('Joschka', 'm'), - ('Johannes', 'm'), - ('Johanna', 'f'), ('Aaron', 'm'), ('Alaeddine', 'm'), ('Andrei', 'm'), From c8b87ecdf0c97c3533b50876bcb80aca5549ef62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 10:39:35 +0200 Subject: [PATCH 09/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_system.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index cc27871..5beb328 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -181,7 +181,6 @@ database access: n/a ```text Please provide the input and output format. ``` - ''' system_test_iteration = f''' @@ -255,6 +254,4 @@ PM: ```text The test takes nothing as input and asserts that the output is a string. ``` - - -''' \ No newline at end of file +''' From 1ce5994abc20458223e96521fca2e6703aa9b03a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 11:05:27 +0200 Subject: [PATCH 10/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20stable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/templates_user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 9d3bbf9..46c1314 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -375,5 +375,7 @@ Or write the summarized microservice{_optional_test} description like this: ```text ``` +Note that your response must be either prompt.txt or final.txt. You must not write both. +Note that you must obey the double asterisk and tripple backtick syntax from above. ''' ) From 8a355ae68d2797d2498b94e0f7cf16c19cff3284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 14:36:51 +0200 Subject: [PATCH 11/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20stable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 8 ++++---- src/options/generate/templates_system.py | 16 ++++++++-------- src/options/generate/templates_user.py | 9 ++++++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index e47551a..72fcef8 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -445,18 +445,18 @@ Test scenario: ), role='user' ) - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) + messages.append(HumanMessage(content=user_input), ) + messages.append(AIMessage(content=question)) final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False) if final: setattr(self.microservice_specification, refinement_type, final) break elif question: - messages.append(HumanMessage(content=user_input),) - messages.append(AIMessage(content=question)) user_input = self.get_user_input(pm, question) else: - raise self.TaskRefinementException() + user_input = 'Put your answer into the right format.' + # raise self.TaskRefinementException() @staticmethod diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 5beb328..223b5bd 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -122,7 +122,7 @@ Input and output have to be accurately specified. You must use the following format (insert ✅, ❌ or n/a) depending on whether the requirement is fulfilled, not fulfilled or not applicable: input: output: -credentials: +api access: database access: 2. @@ -152,18 +152,18 @@ You must not add information that was not provided by the client. Example for the description "given a city, get the weather report for the next 5 days": input: ✅ output: ✅ -credentials: ❌ +api access: ❌ database access: n/a **prompt.txt** ```text -Please provide the url of the weather api and a valid api key. Or let our engineers try to find a free api. +Please provide the url of the weather api and a valid api key or some other way accessing the api. Or let our engineers try to find a free api. ``` Example for the description "convert png to svg": input: ✅ output: ✅ -credentials: n/a +api access: n/a database access: n/a **final.txt** @@ -174,7 +174,7 @@ The user inserts a png and gets an svg as response. Example for the description "parser": input: ❌ output: ❌ -credentials: n/a +api access: n/a database access: n/a **prompt.txt** @@ -187,8 +187,8 @@ system_test_iteration = f''' The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: -a) The unit test requires an example file as input. -In this case you must ask the client to provide the example file as URL. +a) The unit test requires an example input file as input. +In this case you must ask the client to provide the example input file as URL. You must not accept files that are not URLs. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): @@ -230,7 +230,7 @@ The user inserts a png and gets an svg as response. PM: **prompt.txt** ```text -Please provide a png example file as url. +Please provide a png example input file as url. ``` Client: **client-response.txt** diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 46c1314..95b76ea 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -357,17 +357,19 @@ The playground (app.py) must not import the executor. ''' ) - +# Create a wrapper around google called Joogle. It modifies the page summary preview text of the search results to insert the word Jina as much as possible. template_refinement = PromptTemplate.from_template( ''' -**client response** +1.Quickly go through the checklist (input/output well defined? api or db access needed?) and think about if you should ask something to the client or if you should write the final description. +**client-response.txt** ```text {user_input} ``` +2.Either write the prompt.txt or the final.txt file. Either ask for clarification like this: **prompt.txt** ```text - + ``` Or write the summarized microservice{_optional_test} description like this: @@ -377,5 +379,6 @@ Or write the summarized microservice{_optional_test} description like this: ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. +Note that prompt.txt must not contain multiple questions. ''' ) From 5eaf4e2caa15b58b9c54e14b49283b39bfccfacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:15:25 +0200 Subject: [PATCH 12/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20stable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 27 ++++++++++++++---------- src/options/generate/templates_system.py | 2 +- src/options/generate/templates_user.py | 4 ++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 72fcef8..6e876d6 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -206,7 +206,7 @@ metas: print_colored('', '\n\n############# Playground #############', 'blue') file_name_to_content = get_all_microservice_files_with_content(microservice_path) - conversation = self.gpt_session.get_conversation([]) + conversation = self.gpt_session.get_conversation() conversation.chat( template_generate_playground.format( code_files_wrapped=self.files_to_string(file_name_to_content, ['microservice.py', 'test_microservice.py']), @@ -296,7 +296,7 @@ metas: destination_folder=next_microservice_path, file_name_s=None, parse_result_fn=self.parse_result_fn_dockerfile, - system_definition_examples=None, + system_definition_examples=[], summarized_error=summarized_error, all_files_string=dock_req_string, ) @@ -338,7 +338,7 @@ metas: return True print_colored('', f'Is it a {package_manager} dependency issue?', 'blue') - conversation = self.gpt_session.get_conversation(None) + conversation = self.gpt_session.get_conversation() answer = conversation.chat( template_is_dependency_issue.format(summarized_error=summarized_error, all_files_string=dock_req_string).replace('PACKAGE_MANAGER', package_manager) ) @@ -401,7 +401,7 @@ gptdeploy deploy --path {self.microservice_root_path} break def summarize_error(self, error): - conversation = self.gpt_session.get_conversation(None) + conversation = self.gpt_session.get_conversation() error_summary = conversation.chat(template_summarize_error.format(error=error)) return error_summary @@ -435,6 +435,7 @@ Test scenario: messages = [ SystemMessage(content=system_task_introduction + template_init), ] + num_parsing_tries = 0 while True: conversation = self.gpt_session.get_conversation(messages, print_stream=os.environ['VERBOSE'].lower() == 'true', print_costs=False) print('thinking...') @@ -445,18 +446,22 @@ Test scenario: ), role='user' ) - question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) - messages.append(HumanMessage(content=user_input), ) - messages.append(AIMessage(content=question)) + messages.append(HumanMessage(content=user_input)) + agent_question = self.extract_content_from_result(agent_response_raw, 'prompt.txt', can_contain_code_block=False) final = self.extract_content_from_result(agent_response_raw, 'final.txt', can_contain_code_block=False) if final: setattr(self.microservice_specification, refinement_type, final) break - elif question: - user_input = self.get_user_input(pm, question) + elif agent_question: + messages.append(AIMessage(content=agent_question)) + user_input = self.get_user_input(pm, agent_question) else: - user_input = 'Put your answer into the right format.' - # raise self.TaskRefinementException() + if num_parsing_tries > 2: + raise self.TaskRefinementException() + num_parsing_tries += 1 + messages.append(AIMessage(content=agent_response_raw)) + messages.append(SystemMessage(content='You did not put your answer into the right format using *** and ```.')) + @staticmethod diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index 223b5bd..334b902 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -187,7 +187,7 @@ system_test_iteration = f''' The client gives you a description of the microservice (web service). Your task is to describe verbally a unit test for that microservice. There are two cases: -a) The unit test requires an example input file as input. +a) If unit test requires an example input file as input: In this case you must ask the client to provide the example input file as URL. You must not accept files that are not URLs. Your response must exactly match the following block code format (double asterisks for the file name and triple backticks for the file block): diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 95b76ea..72033f6 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -369,7 +369,7 @@ template_refinement = PromptTemplate.from_template( Either ask for clarification like this: **prompt.txt** ```text - + ``` Or write the summarized microservice{_optional_test} description like this: @@ -379,6 +379,6 @@ Or write the summarized microservice{_optional_test} description like this: ``` Note that your response must be either prompt.txt or final.txt. You must not write both. Note that you must obey the double asterisk and tripple backtick syntax from above. -Note that prompt.txt must not contain multiple questions. +Note that prompt.txt must not only contain one question. ''' ) From e300fb161589d6a591b18fe4ab169479bf3ae4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:21:44 +0200 Subject: [PATCH 13/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20merge=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/generator.py | 37 +++++-------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 0ba4b56..f0e0756 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -2,7 +2,7 @@ import os import random import re import shutil -from typing import Callable, Union +from typing import Callable from typing import List, Text, Optional @@ -14,21 +14,17 @@ from src.apis import gpt from src.apis.jina_cloud import process_error_message, push_executor, is_executor_in_hub from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \ PROBLEMATIC_PACKAGES, EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ - REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, UNNECESSARY_PACKAGES -from src.options.generate.templates_system import template_system_message_base, gpt_example, executor_example, \ - docarray_example, client_example, system_task_iteration, system_task_introduction, system_test_iteration + REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, UNNECESSARY_PACKAGES, IMPLEMENTATION_FILE_NAME, \ + IMPLEMENTATION_FILE_TAG +from src.options.generate.templates_system import system_task_iteration, system_task_introduction, system_test_iteration from src.options.generate.templates_user import template_generate_microservice_name, \ template_generate_possible_packages, \ template_solve_code_issue, \ template_solve_pip_dependency_issue, template_is_dependency_issue, template_generate_playground, \ template_generate_function, template_generate_test, template_generate_requirements, \ template_chain_of_thought, template_summarize_error, \ - template_generate_apt_get_install, template_solve_apt_get_dependency_issue - template_solve_pip_dependency_issue, \ - template_generate_apt_get_install, template_solve_apt_get_dependency_issue, \ - template_is_dependency_issue, template_generate_playground, \ - template_generate_executor, template_generate_test, template_generate_requirements, \ - template_chain_of_thought, template_summarize_error, template_refinement + template_generate_apt_get_install, template_solve_apt_get_dependency_issue, template_refinement + from src.options.generate.ui import get_random_employee from src.utils.io import persist_file, get_all_microservice_files_with_content, get_microservice_path from src.utils.string_tools import print_colored @@ -178,7 +174,6 @@ metas: 'Requirements', template_generate_requirements, MICROSERVICE_FOLDER_v1, - system_definition_examples=None, code_files_wrapped=self.files_to_string({ IMPLEMENTATION_FILE_NAME: microservice_content, TEST_EXECUTOR_FILE_NAME: test_microservice_content, @@ -474,26 +469,6 @@ Test scenario: messages.append(SystemMessage(content='You did not put your answer into the right format using *** and ```.')) - - @staticmethod - def _create_system_message(task_description, test_description, system_definition_examples: List[str] = []) -> SystemMessage: - if system_definition_examples is None: - return None - - system_message = PromptTemplate.from_template(template_system_message_base).format( - task_description=task_description, - test_description=test_description, - ) - if 'gpt' in system_definition_examples: - system_message += f'\n{gpt_example}' - if 'executor' in system_definition_examples: - system_message += f'\n{executor_example}' - if 'docarray' in system_definition_examples: - system_message += f'\n{docarray_example}' - if 'client' in system_definition_examples: - system_message += f'\n{client_example}' - return SystemMessage(content=system_message) - @staticmethod def get_user_input(employee, prompt_to_user): val = input(f'{employee.emoji}❓ {prompt_to_user}\nyou: ') From 1fefaaa48ff9a5032706662a8ff5f46e4ab7cfed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:31:45 +0200 Subject: [PATCH 14/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20merge=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..72f6073 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + workflow_dispatch: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + group: [1, 2] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Prepare environment + run: | + python -m pip install --upgrade pip + python -m pip install wheel + pip install --no-cache-dir ".[full,test]" + - name: Test + id: test + run: | + pytest --suppress-no-test-exit-code --cov=now --cov-report=xml -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration tests/ + timeout-minutes: 20 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} \ No newline at end of file From 1b9a76b78bfdbd3350b490c48b13805d8a31a7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:34:36 +0200 Subject: [PATCH 15/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f6073..2c06e92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - test: + test_✨: runs-on: ubuntu-latest strategy: fail-fast: false @@ -22,6 +22,7 @@ jobs: python -m pip install --upgrade pip python -m pip install wheel pip install --no-cache-dir ".[full,test]" + pip install pytest - name: Test id: test run: | From 502d207d150c8b6a94efdb05c9bdda9dfcb6efa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:35:19 +0200 Subject: [PATCH 16/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c06e92..b4b935e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - test_✨: + test: runs-on: ubuntu-latest strategy: fail-fast: false From bd21157f4fb060cf08c8504ac93d18d8c5e5174c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:37:53 +0200 Subject: [PATCH 17/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4b935e..e93af99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,10 +23,11 @@ jobs: python -m pip install wheel pip install --no-cache-dir ".[full,test]" pip install pytest + pip install pytest-split - name: Test id: test run: | - pytest --suppress-no-test-exit-code --cov=now --cov-report=xml -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration tests/ + pytest -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration tests/ timeout-minutes: 20 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} \ No newline at end of file From ab08785ae0ad6e2bf9c6db65f51acad19bb8529a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 17:41:37 +0200 Subject: [PATCH 18/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e93af99..b5e9b7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Test id: test run: | - pytest -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration tests/ + pytest -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration test/ timeout-minutes: 20 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} \ No newline at end of file From cca9c5caeca2d362a99cad29d3259e49ed53dc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 18:57:57 +0200 Subject: [PATCH 19/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements-test.txt | 2 ++ src/options/generate/generator.py | 3 ++- src/options/generate/templates_user.py | 11 +++++----- test/test_generator.py | 30 ++++---------------------- 4 files changed, 14 insertions(+), 32 deletions(-) create mode 100644 requirements-test.txt diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..7f6cef7 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest +pytest-split \ No newline at end of file diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index f0e0756..f9c824a 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -11,6 +11,7 @@ from langchain.schema import SystemMessage, HumanMessage, AIMessage from pydantic.dataclasses import dataclass from src.apis import gpt +from src.apis.gpt import _GPTConversation from src.apis.jina_cloud import process_error_message, push_executor, is_executor_in_hub from src.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \ PROBLEMATIC_PACKAGES, EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \ @@ -110,7 +111,7 @@ metas: parse_result_fn = self.get_default_parse_result_fn(file_name_s) print_colored('', f'\n\n############# {section_title} #############', 'blue') - system_introduction_message = self._create_system_message(self.microservice_specification.task, self.microservice_specification.test, system_definition_examples) + system_introduction_message = _GPTConversation._create_system_message(self.microservice_specification.task, self.microservice_specification.test, system_definition_examples) conversation = self.gpt_session.get_conversation(messages=[system_introduction_message]) template_kwargs = {k: v for k, v in template_kwargs.items() if k in template.input_variables} if 'file_name' in template.input_variables and len(file_name_s) == 1: diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index ea9737b..c878d43 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -22,8 +22,10 @@ The implemented function and the test must not use a pre-trained model unless it The implemented function and the test must not train a model. The implemented function and the test must not contain prototype or placeholder implementations. The implemented function and the test must run in a docker container based on debian. -The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding or generating natural language or using any language model. Other language models are not allowed.''' - +The implemented function and the test must not use libraries like Flask. +The implemented function and the test must not have a __main__ function. +The implemented function and the test must use gpt_3_5_turbo_api if the task requires understanding or generation of natural language or using any language model. Other language models are not allowed. +The implemented function and the test must not use gpt_3_5_turbo_api or any other language model if the task does not require understanding or generation of natural language.''' template_generate_microservice_name = PromptTemplate.from_template( @@ -87,7 +89,7 @@ template_code_wrapping_string = '''The code will go into {file_name_purpose}. Ma You must provide the complete file with the exact same syntax to wrap the code.''' -gpt_35_turbo_usage_string = """If you use gpt_3_5_turbo_api, then this is an example on how to use it: +gpt_35_turbo_usage_string = """If need to use gpt_3_5_turbo_api, then this is an example on how to use it: ``` from .apis import GPT_3_5_Turbo_API @@ -151,8 +153,7 @@ template_generate_requirements = PromptTemplate.from_template( {code_files_wrapped} Write the content of the requirements.txt file. -The requirements.txt file must include the following packages: -**requirements.txt** +The requirements.txt file must include the following packages in that specified version: ``` jina==3.15.1.dev14 docarray==0.21.0 diff --git a/test/test_generator.py b/test/test_generator.py index 0552e56..9ad2276 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -1,29 +1,7 @@ -import unittest.mock as mock +import os from src.options.generate.generator import Generator -from src.apis.gpt import GPTSession def test_generator(tmpdir): - # Define a mock response - mock_response = { - "choices": [ - { - "delta": { - "content": "This is a mock response." - } - } - ] - } - - # Define a function to replace openai.ChatCompletion.create - def mock_create(*args, **kwargs): - return [mock_response] * kwargs.get("stream", 1) - - # Define a function to replace get_openai_api_key - def mock_get_openai_api_key(*args, **kwargs): - pass - - # Use mock.patch as a context manager to replace the original methods with the mocks - with mock.patch("openai.ChatCompletion.create", side_effect=mock_create), \ - mock.patch.object(GPTSession, "configure_openai_api_key", side_effect=mock_get_openai_api_key): - generator = Generator("my description", "my test") - generator.generate(str(tmpdir)) + os.environ['VERBOSE'] = 'true' + generator = Generator("The microservice is very simple, it does not take anything as input and only outputs the word 'test'", "my test", str(tmpdir) + 'microservice', 'gpt-3.5-turbo') + generator.generate() From ca883768a9a58ef400a6536b4d2851e07d2ebe47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 19:04:08 +0200 Subject: [PATCH 20/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5e9b7d..67e845d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,4 +30,4 @@ jobs: pytest -v -s -m "not gpu" --splits 9 --group ${{ matrix.group }} --splitting-algorithm least_duration test/ timeout-minutes: 20 env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} \ No newline at end of file + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} From 047a40952942ebb35bb3bd9997bea9c6bf2db486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 19:24:48 +0200 Subject: [PATCH 21/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/options/generate/static_files/microservice/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/options/generate/static_files/microservice/__init__.py diff --git a/src/options/generate/static_files/microservice/__init__.py b/src/options/generate/static_files/microservice/__init__.py new file mode 100644 index 0000000..88777c8 --- /dev/null +++ b/src/options/generate/static_files/microservice/__init__.py @@ -0,0 +1 @@ +# if this file \ No newline at end of file From 56ce1e9618a90cf686eb0ee6e749f9e30d358c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Ho=CC=88nicke?= Date: Fri, 28 Apr 2023 19:42:52 +0200 Subject: [PATCH 22/22] =?UTF-8?q?=F0=9F=91=A9=E2=80=8D=F0=9F=92=BC=20feat:?= =?UTF-8?q?=20pm=20role=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ++-- MANIFEST.in | 2 +- test/.test_durations | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 test/.test_durations diff --git a/.gitignore b/.gitignore index 4c80df9..5419ea1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /microservice/ .env -config.yml -executor data build +gptdeploy.egg-info +dist diff --git a/MANIFEST.in b/MANIFEST.in index 4c8d5b5..11bbb69 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include requirements.txt include gptdeploy.cmd -recursive-include src/options/generate/static_files/gateway *.toml *.cmd *.conf *.txt Dockerfile \ No newline at end of file +recursive-include src/options/generate/static_files/ * \ No newline at end of file diff --git a/test/.test_durations b/test/.test_durations new file mode 100644 index 0000000..496f234 --- /dev/null +++ b/test/.test_durations @@ -0,0 +1,3 @@ +{ + "test/test_generator.py::test_generator": 246.233993379 +} \ No newline at end of file