From d878a782d527d60d0e140c6f698cb03486619f95 Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Wed, 19 Apr 2023 13:39:59 +0200 Subject: [PATCH 1/3] fix: use old callback for cost calculation --- src/apis/gpt.py | 35 ++++++++++++++++++++++++++++++----- src/constants.py | 6 ++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index 7a86524..31b5801 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -8,9 +8,11 @@ from langchain import PromptTemplate from langchain.callbacks import CallbackManager from langchain.chat_models import ChatOpenAI from openai.error import RateLimitError -from langchain.schema import AIMessage, HumanMessage, SystemMessage, BaseMessage +from langchain.schema import HumanMessage, SystemMessage, BaseMessage from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from src.constants import PRICING_GPT4_PROMPT, PRICING_GPT4_GENERATION, PRICING_GPT3_5_TURBO_PROMPT, \ + PRICING_GPT3_5_TURBO_GENERATION from src.options.generate.templates_system import template_system_message_base, executor_example, docarray_example, client_example from src.utils.string_tools import print_colored @@ -20,7 +22,18 @@ class GPTSession: self.task_description = task_description self.test_description = test_description self.configure_openai_api_key() - self.model_name = 'gpt-4' if model == 'gpt-4' and self.is_gpt4_available() else 'gpt-3.5-turbo' + if model == 'gpt-4' and self.is_gpt4_available(): + self.pricing_prompt = PRICING_GPT4_PROMPT + self.pricing_generation = PRICING_GPT4_GENERATION + else: + if model == 'gpt-4': + print_colored('GPT version info', 'GPT-4 is not available. Using GPT-3.5-turbo instead.', 'yellow') + model = 'gpt-3.5-turbo' + self.pricing_prompt = PRICING_GPT3_5_TURBO_PROMPT + self.pricing_generation = PRICING_GPT3_5_TURBO_GENERATION + self.model_name = model + self.chars_prompt_so_far = 0 + self.chars_generation_so_far = 0 @staticmethod def configure_openai_api_key(): @@ -50,11 +63,21 @@ If you have updated it already, please restart your terminal. continue return True except openai.error.InvalidRequestError: - print_colored('GPT version info', 'GPT-4 is not available. Using GPT-3.5-turbo instead.', 'yellow') return False + def cost_callback(self, chars_prompt, chars_generation): + self.chars_prompt_so_far += chars_prompt + self.chars_generation_so_far += chars_generation + print('\n') + money_prompt = self.calculate_money_spent(self.chars_prompt_so_far, self.pricing_prompt) + money_generation = self.calculate_money_spent(self.chars_generation_so_far, self.pricing_generation) + print('Total money spent so far on openai.com:', f'${money_prompt + money_generation}') + print('\n') + def get_conversation(self, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): - return _GPTConversation(self.model_name, self.task_description, self.test_description, system_definition_examples) + return _GPTConversation( + self.model_name, self.cost_callback, self.task_description, self.test_description, system_definition_examples + ) class AssistantStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler): @@ -64,7 +87,7 @@ class AssistantStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler): class _GPTConversation: - def __init__(self, model: str, task_description, test_description, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): + def __init__(self, model: str, cost_callback, task_description, test_description, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): self._chat = ChatOpenAI( model_name=model, streaming=True, @@ -72,6 +95,7 @@ class _GPTConversation: verbose=True, temperature=0, ) + self.cost_callback = cost_callback self.messages: List[BaseMessage] = [] self.system_message = self._create_system_message(task_description, test_description, system_definition_examples) if os.environ['VERBOSE'].lower() == 'true': @@ -86,6 +110,7 @@ class _GPTConversation: response = self._chat([self.system_message] + self.messages) if os.environ['VERBOSE'].lower() == 'true': print() + self.cost_callback(sum([m.content for m in self.messages]), len(response.content)) self.messages.append(response) return response.content diff --git a/src/constants.py b/src/constants.py index c1b3687..0d0dae0 100644 --- a/src/constants.py +++ b/src/constants.py @@ -23,6 +23,12 @@ FILE_AND_TAG_PAIRS = [ FLOW_URL_PLACEHOLDER = 'jcloud.jina.ai' +PRICING_GPT4_PROMPT = 0.03 +PRICING_GPT4_GENERATION = 0.06 +PRICING_GPT3_5_TURBO_PROMPT = 0.002 +PRICING_GPT3_5_TURBO_GENERATION = 0.002 + +CHARS_PER_TOKEN = 3.4 NUM_IMPLEMENTATION_STRATEGIES = 5 MAX_DEBUGGING_ITERATIONS = 10 From 3d1e23c173269bf2cda5049d316189487c40cabc Mon Sep 17 00:00:00 2001 From: Joschka Braun Date: Wed, 19 Apr 2023 13:43:47 +0200 Subject: [PATCH 2/3] fix: fix typo --- src/apis/gpt.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/apis/gpt.py b/src/apis/gpt.py index 31b5801..d80dd16 100644 --- a/src/apis/gpt.py +++ b/src/apis/gpt.py @@ -12,7 +12,7 @@ from langchain.schema import HumanMessage, SystemMessage, BaseMessage from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from src.constants import PRICING_GPT4_PROMPT, PRICING_GPT4_GENERATION, PRICING_GPT3_5_TURBO_PROMPT, \ - PRICING_GPT3_5_TURBO_GENERATION + PRICING_GPT3_5_TURBO_GENERATION, CHARS_PER_TOKEN from src.options.generate.templates_system import template_system_message_base, executor_example, docarray_example, client_example from src.utils.string_tools import print_colored @@ -35,6 +35,11 @@ class GPTSession: self.chars_prompt_so_far = 0 self.chars_generation_so_far = 0 + def get_conversation(self, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): + return _GPTConversation( + self.model_name, self.cost_callback, self.task_description, self.test_description, system_definition_examples + ) + @staticmethod def configure_openai_api_key(): if 'OPENAI_API_KEY' not in os.environ: @@ -69,15 +74,14 @@ If you have updated it already, please restart your terminal. self.chars_prompt_so_far += chars_prompt self.chars_generation_so_far += chars_generation print('\n') - money_prompt = self.calculate_money_spent(self.chars_prompt_so_far, self.pricing_prompt) - money_generation = self.calculate_money_spent(self.chars_generation_so_far, self.pricing_generation) + money_prompt = self._calculate_money_spent(self.chars_prompt_so_far, self.pricing_prompt) + money_generation = self._calculate_money_spent(self.chars_generation_so_far, self.pricing_generation) print('Total money spent so far on openai.com:', f'${money_prompt + money_generation}') print('\n') - def get_conversation(self, system_definition_examples: List[str] = ['executor', 'docarray', 'client']): - return _GPTConversation( - self.model_name, self.cost_callback, self.task_description, self.test_description, system_definition_examples - ) + @staticmethod + def _calculate_money_spent(num_chars, price): + return round(num_chars / CHARS_PER_TOKEN * price / 1000, 3) class AssistantStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler): @@ -110,7 +114,7 @@ class _GPTConversation: response = self._chat([self.system_message] + self.messages) if os.environ['VERBOSE'].lower() == 'true': print() - self.cost_callback(sum([m.content for m in self.messages]), len(response.content)) + self.cost_callback(sum([len(m.content) for m in self.messages]), len(response.content)) self.messages.append(response) return response.content From a336907a21955d7c707dc13427487b61a707322e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20H=C3=B6nicke?= Date: Wed, 19 Apr 2023 14:11:21 +0200 Subject: [PATCH 3/3] feat: http instead of grpc (#18) * feat: http * feat: http * fix: git ignore * fix: better formatting * docs: add missing print * fix: add protocol * fix: bump version --- .gitignore | 2 +- setup.py | 2 +- src/__init__.py | 2 +- src/apis/jina_cloud.py | 3 ++- src/options/generate/generator.py | 7 ++++--- src/options/generate/templates_system.py | 2 +- src/options/generate/templates_user.py | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index cee1258..0723889 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/executor_level2/ +/microservice/ .env config.yml diff --git a/setup.py b/setup.py index 50cb498..a73c7ff 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ def read_requirements(): setup( name='gptdeploy', - version='0.18.18', + version='0.18.19', description='Use natural language interface to generate, deploy and update your microservice infrastructure.', long_description=open('README.md', 'r', encoding='utf-8').read(), long_description_content_type='text/markdown', diff --git a/src/__init__.py b/src/__init__.py index c82d85e..8c35b99 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,2 +1,2 @@ -__version__ = '0.18.18' +__version__ = '0.18.19' from src.cli import main \ No newline at end of file diff --git a/src/apis/jina_cloud.py b/src/apis/jina_cloud.py index 2e77f46..3674749 100644 --- a/src/apis/jina_cloud.py +++ b/src/apis/jina_cloud.py @@ -155,7 +155,7 @@ streamlit run {os.path.join(microservice_path, "app.py")} --server.port 8081 --s def run_streamlit_app(app_path): subprocess.run(['streamlit', 'run', app_path, 'server.address', '0.0.0.0', '--server.port', '8081', '--', '--host', - 'grpc://localhost:8080']) + 'http://localhost:8080']) def run_locally(executor_name, microservice_version_path): @@ -207,6 +207,7 @@ jtype: Flow with: name: nowapi port: 8080 + protocol: http jcloud: version: 3.14.2.dev18 labels: diff --git a/src/options/generate/generator.py b/src/options/generate/generator.py index 97e9132..0119146 100644 --- a/src/options/generate/generator.py +++ b/src/options/generate/generator.py @@ -54,7 +54,7 @@ metas: def generate_and_persist_file(self, section_title, template, destination_folder, file_name, **template_kwargs): - print_colored('', f'\n############# {section_title} #############', 'blue') + print_colored('', f'\n\n############# {section_title} #############', 'blue') conversation = self.gpt_session.get_conversation() template_kwargs = {k: v for k, v in template_kwargs.items() if k in template.input_variables} content_raw = conversation.chat( @@ -139,7 +139,7 @@ metas: print('\nFirst version of the microservice generated. Start iterating on it to make the tests pass...') def generate_playground(self, microservice_name, microservice_path): - print_colored('', '\n############# Playground #############', 'blue') + print_colored('', '\n\n############# Playground #############', 'blue') file_name_to_content = get_all_microservice_files_with_content(microservice_path) conversation = self.gpt_session.get_conversation([]) @@ -222,13 +222,14 @@ metas: return 'yes' in answer.lower() def generate_microservice_name(self, description): + print_colored('', '\n\n############# What should be the name of the Microservice? #############', 'blue') conversation = self.gpt_session.get_conversation() name_raw = conversation.chat(template_generate_microservice_name.format(description=description)) name = self.extract_content_from_result(name_raw, 'name.txt') return name def get_possible_packages(self): - print_colored('', '############# What packages to use? #############', 'blue') + print_colored('', '\n\n############# What packages to use? #############', 'blue') conversation = self.gpt_session.get_conversation() packages_raw = conversation.chat( template_generate_possible_packages.format(description=self.task_description) diff --git a/src/options/generate/templates_system.py b/src/options/generate/templates_system.py index b4fd136..5b2bc88 100644 --- a/src/options/generate/templates_system.py +++ b/src/options/generate/templates_system.py @@ -64,7 +64,7 @@ Here is an example of a client file: **client.py** ```python from jina import Client, Document, DocumentArray -client = Client(host='{FLOW_URL_PLACEHOLDER}') +client = Client(host='{FLOW_URL_PLACEHOLDER}', protocol='http') d = Document(uri='...') d.load_uri_to_blob() response = client.post('/', inputs=DocumentArray([d])) # the client must be called on '/' diff --git a/src/options/generate/templates_user.py b/src/options/generate/templates_user.py index 03d58fb..e9af8ba 100644 --- a/src/options/generate/templates_user.py +++ b/src/options/generate/templates_user.py @@ -274,7 +274,7 @@ print(response[0].text) # can also be blob in case of image/audio..., this shoul ``` Note that the response will always be in response[0].text You must provide the complete app.py file with the exact same syntax to wrap the code. -The playground (app.py) must read the host from sys.argv because it will be started with a custom host: streamlit run app.py -- --host grpc://... +The playground (app.py) must read the host from sys.argv because it will be started with a custom host: streamlit run app.py -- --host http(s)://... The playground (app.py) must not let the user configure the host on the ui. ''' )