🔎 feat: search api

README.md

@@ -54,7 +54,9 @@ Your imagination is the limit!
 
 </p>
 
-Welcome to Dev GPT, where we bring your ideas to life with the power of advanced artificial intelligence! Our automated development team is designed to create microservices tailored to your specific needs, making your software development process seamless and efficient. Comprised of a virtual Product Manager, Developer, and DevOps, our AI team ensures that every aspect of your project is covered, from concept to deployment.
+Welcome to Dev-GPT, where we bring your ideas to life with the power of advanced artificial intelligence! 
+Our automated development team is designed to create microservices tailored to your specific needs, making your software development process seamless and efficient. 
+Comprised of a virtual Product Manager, Developer, and DevOps, our AI team ensures that every aspect of your project is covered, from concept to deployment.
 
 ## Quickstart
 
@@ -65,8 +67,13 @@ dev-gpt generate
 
 ### Requirements
 - OpenAI key with access to gpt-3.5-turbo or gpt-4
+- if you want to enable your microservice to search for web content, 
+you need to set the GOOGLE_API_KEY and GOOGLE_CSE_ID environment variables.
+More information can be found [here](https://developers.google.com/custom-search/v1/overview).
 ```bash
-dev-gpt configure --key <your openai api key>
+dev-gpt configure --openai_api_key <your openai api key>
+dev-gpt configure --google_api_key <google api key> (optional if you want to use google search)
+dev-gpt configure --google_cse_id <google cse id> (optional if you want to use google search)
 ```
 
 If you set the environment variable `OPENAI_API_KEY`, the configuration step can be skipped.

dev_gpt/apis/gpt.py

@@ -24,7 +24,7 @@ def configure_openai_api_key():
     if 'OPENAI_API_KEY' not in os.environ:
         print_colored('You need to set OPENAI_API_KEY in your environment.', '''
 Run:
-dev-gpt configure --key <your_openai_api_key>
+dev-gpt configure --openai_api_key <your_openai_api_key>
 
 If you have updated it already, please restart your terminal.
 ''', 'red')

dev_gpt/apis/jina_cloud.py

@@ -98,7 +98,7 @@ def _push_executor(dir_path):
         'public': 'True',
         'private': 'False',
         'verbose': 'True',
-        'buildEnv': f'{{"OPENAI_API_KEY": "{os.environ["OPENAI_API_KEY"]}"}}',
+        'buildEnv': f'{{"OPENAI_API_KEY": "{os.environ["OPENAI_API_KEY"]}", "GOOGLE_API_KEY": "{os.environ.get("GOOGLE_API_KEY","")}", "GOOGLE_CSE_ID": "{os.environ.get("GOOGLE_CSE_ID","")}"}}',
         'md5sum': md5_digest,
     }
     with suppress_stdout():
@@ -251,7 +251,9 @@ executors:
     uses: {prefix}://{get_user_name(DEMO_TOKEN)}/{executor_name}:latest
     {"" if use_docker else "install-requirements: True"}
     env:
-      OPENAI_API_KEY: {os.environ['OPENAI_API_KEY']}
+      OPENAI_API_KEY: ${{{{ ENV.OPENAI_API_KEY }}}}
+      GOOGLE_API_KEY: ${{{{ ENV.GOOGLE_API_KEY }}}}
+      GOOGLE_CSE_ID: ${{{{ ENV.GOOGLE_CSE_ID }}}}
     jcloud:
       resources:
         instance: C2

dev_gpt/cli.py

@@ -92,9 +92,16 @@ def deploy(path):
     Deployer().deploy(path)
 
 @main.command()
-@click.option('--key', required=True, help='Your OpenAI API key.')
-def configure(key):
-    set_api_key(key)
+@click.option('--openai-api-key', default=None, help='Your OpenAI API key.')
+@click.option('--google-api-key', default=None, help='Your Google API key.')
+@click.option('--google-cse-id', default=None, help='Your Google CSE ID.')
+def configure(openai_api_key, google_api_key, google_cse_id):
+    if openai_api_key:
+        set_api_key('OPENAI_API_KEY', openai_api_key)
+    if google_api_key:
+        set_api_key('GOOGLE_API_KEY', google_api_key)
+    if google_cse_id:
+        set_api_key('GOOGLE_CSE_ID', google_cse_id)
 
 
 if __name__ == '__main__':

dev_gpt/constants.py

@@ -55,3 +55,7 @@ LANGUAGE_PACKAGES = [
     'vadersentiment'
 ]
 
+SEARCH_PACKAGES = [
+    'googlesearch-python', 'google', 'googlesearch', 'google-api-python-client', 'pygooglenews', 'google-cloud'
+]
+

dev_gpt/options/configure/key_handling.py

@@ -40,26 +40,26 @@ def get_shell():
         return None
 
 
-def get_shell_config(key):
+def get_shell_config(name, key):
     return {
-        "bash": {"config_file": "~/.bashrc", "export_line": f"export OPENAI_API_KEY={key}"},
-        "zsh": {"config_file": "~/.zshrc", "export_line": f"export OPENAI_API_KEY={key}"},
-        "sh": {"config_file": "~/.profile", "export_line": f"export OPENAI_API_KEY={key}"},
+        "bash": {"config_file": "~/.bashrc", "export_line": f"export {name}={key}"},
+        "zsh": {"config_file": "~/.zshrc", "export_line": f"export {name}={key}"},
+        "sh": {"config_file": "~/.profile", "export_line": f"export {name}={key}"},
         "fish": {
             "config_file": "~/.config/fish/config.fish",
-            "export_line": f"set -gx OPENAI_API_KEY {key}",
+            "export_line": f"set -gx {name} {key}",
         },
-        "csh": {"config_file": "~/.cshrc", "export_line": f"setenv OPENAI_API_KEY {key}"},
-        "tcsh": {"config_file": "~/.tcshrc", "export_line": f"setenv OPENAI_API_KEY {key}"},
-        "ksh": {"config_file": "~/.kshrc", "export_line": f"export OPENAI_API_KEY={key}"},
-        "dash": {"config_file": "~/.profile", "export_line": f"export OPENAI_API_KEY={key}"}
+        "csh": {"config_file": "~/.cshrc", "export_line": f"setenv {name} {key}"},
+        "tcsh": {"config_file": "~/.tcshrc", "export_line": f"setenv {name} {key}"},
+        "ksh": {"config_file": "~/.kshrc", "export_line": f"export {name}={key}"},
+        "dash": {"config_file": "~/.profile", "export_line": f"export {name}={key}"}
     }
 
 
-def set_env_variable(shell, key):
-    shell_config = get_shell_config(key)
+def set_env_variable(shell, name, key):
+    shell_config = get_shell_config(name, key)
     if shell not in shell_config:
-        click.echo("Sorry, your shell is not supported. Please add the key OPENAI_API_KEY manually.")
+        click.echo(f"Sorry, your shell is not supported. Please add the key {name} manually.")
         return
 
     config_file = os.path.expanduser(shell_config[shell]["config_file"])
@@ -71,8 +71,8 @@ def set_env_variable(shell, key):
         export_line = shell_config[shell]['export_line']
 
         # Update the existing API key if it exists, otherwise append it to the config file
-        if f"OPENAI_API_KEY" in content:
-            content = re.sub(r'OPENAI_API_KEY=.*', f'OPENAI_API_KEY={key}', content, flags=re.MULTILINE)
+        if f"{name}" in content:
+            content = re.sub(rf'{name}=.*', f'{name}={key}', content, flags=re.MULTILINE)
 
             with open(config_file, "w", encoding='utf-8') as file:
                 file.write(content)
@@ -81,7 +81,7 @@ def set_env_variable(shell, key):
                 file.write(f"\n{export_line}\n")
 
         click.echo(f'''
-✅ Success, OPENAI_API_KEY has been set in {config_file}.
+✅ Success, {name} has been set in {config_file}.
 Please restart your shell to apply the changes or run:
 source {config_file}
 '''
@@ -91,21 +91,21 @@ source {config_file}
         click.echo(f"Error: {config_file} not found. Please set the environment variable manually.")
 
 
-def set_api_key(key):
+def set_api_key(name, key):
     system_platform = platform.system().lower()
 
     if system_platform == "windows":
-        set_env_variable_command = f'setx OPENAI_API_KEY "{key}"'
+        set_env_variable_command = f'setx {name} "{key}"'
         subprocess.call(set_env_variable_command, shell=True)
-        click.echo('''
-✅ Success, OPENAI_API_KEY has been set.
+        click.echo(f'''
+✅ Success, {name} has been set.
 Please restart your Command Prompt to apply the changes.
 '''
                    )
 
     elif system_platform in ["linux", "darwin"]:
-        if "OPENAI_API_KEY" in os.environ or is_key_set_in_config_file(key):
-            if not click.confirm("OPENAI_API_KEY is already set. Do you want to overwrite it?"):
+        if f"{name}" in os.environ or is_key_set_in_config_file(key):
+            if not click.confirm(f"{name} is already set. Do you want to overwrite it?"):
                 click.echo("Aborted.")
                 return
 
@@ -115,24 +115,24 @@ Please restart your Command Prompt to apply the changes.
                 "Error: Unable to detect your shell or psutil is not available. Please set the environment variable manually.")
             return
 
-        set_env_variable(shell, key)
+        set_env_variable(shell, name, key)
     else:
         click.echo("Sorry, this platform is not supported.")
 
 
-def is_key_set_in_config_file(key):
+def is_key_set_in_config_file(name, key):
     shell = get_shell()
     if shell is None:
         return False
 
-    shell_config = get_shell_config(key)
+    shell_config = get_shell_config(name, key)
 
     config_file = os.path.expanduser(shell_config[shell]["config_file"])
 
     try:
         with open(config_file, "r", encoding='utf-8') as file:
             content = file.read()
-            if f"OPENAI_API_KEY" in content:
+            if f"{name}" in content:
                 return True
     except FileNotFoundError:
         pass

dev_gpt/options/generate/chains/auto_refine_description.py

@@ -3,7 +3,7 @@ import json
 from dev_gpt.apis.gpt import ask_gpt
 from dev_gpt.options.generate.parser import identity_parser
 from dev_gpt.options.generate.prompt_factory import context_to_string
-
+from dev_gpt.options.generate.tools.tools import get_available_tools
 
 
 def auto_refine_description(context):
@@ -36,7 +36,9 @@ def auto_refine_description(context):
 better_description_prompt = f'''{{context_string}}
 Update the description of the Microservice to make it more precise without adding or removing information.
 Note: the output must be a list of tasks the Microservice has to perform.
-Example for the description: "return the average temperature of the 5 days weather forecast for a given location."
+Note: you can uses two tools if necessary:
+{get_available_tools()}
+Example for the description: "return a description of the average temperature of the 5 days weather forecast for a given location."
 1. get the 5 days weather forcast from the https://openweathermap.org/ API
 2. extract the temperature from the response
 3. calculate the average temperature'''

dev_gpt/options/generate/chains/question_answering.py

@@ -1,25 +1,46 @@
 from dev_gpt.apis.gpt import ask_gpt
-from dev_gpt.options.generate.parser import boolean_parser
+from dev_gpt.options.generate.parser import boolean_parser, identity_parser
+
 
 def is_question_true(question):
     def fn(text):
         return answer_yes_no_question(text, question)
+
     return fn
 
+
 def is_question_false(question):
     return lambda context: not is_question_true(question)(context)
 
 
 def answer_yes_no_question(text, question):
-    prompt = question_prompt.format(
+    pros_and_cons = ask_gpt(
+        pros_and_cons_prompt.format(
             question=question,
-        text=text
+            text=text,
+        ),
+        identity_parser,
     )
-    return ask_gpt(prompt, boolean_parser)
+
+    return ask_gpt(
+        question_prompt.format(
+            text=text,
+            question=question,
+            pros_and_cons=pros_and_cons,
+        ),
+        boolean_parser)
+
+pros_and_cons_prompt = '''\
+# Context
+{text}
+# Question
+{question}
+Note: You must not answer the question. Instead, give up to 5 bullet points (10 words) arguing why the question should be answered with true or false.'''
 
 question_prompt = '''\
+# Context
 {text}
+# Question
 {question}
 Note: You must answer "yes" or "no".
 '''
-

dev_gpt/options/generate/generator.py

@@ -17,7 +17,7 @@ from dev_gpt.apis.pypi import is_package_on_pypi, clean_requirements_txt
 from dev_gpt.constants import FILE_AND_TAG_PAIRS, NUM_IMPLEMENTATION_STRATEGIES, MAX_DEBUGGING_ITERATIONS, \
     BLACKLISTED_PACKAGES, EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_NAME, TEST_EXECUTOR_FILE_TAG, \
     REQUIREMENTS_FILE_NAME, REQUIREMENTS_FILE_TAG, DOCKER_FILE_NAME, IMPLEMENTATION_FILE_NAME, \
-    IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES, DOCKER_BASE_IMAGE_VERSION
+    IMPLEMENTATION_FILE_TAG, LANGUAGE_PACKAGES, UNNECESSARY_PACKAGES, DOCKER_BASE_IMAGE_VERSION, SEARCH_PACKAGES
 from dev_gpt.options.generate.pm.pm import PM
 from dev_gpt.options.generate.templates_user import template_generate_microservice_name, \
     template_generate_possible_packages, \
@@ -500,7 +500,7 @@ pytest
             description=self.microservice_specification.task
         )['strategies.json']
         packages_list = [[pkg.strip().lower() for pkg in packages] for packages in json.loads(packages_json_string)]
-        packages_list = [[self.replace_with_gpt_3_5_turbo_if_possible(pkg) for pkg in packages] for packages in
+        packages_list = [[self.replace_with_tool_if_possible(pkg) for pkg in packages] for packages in
                          packages_list]
 
         packages_list = self.filter_packages_list(packages_list)
@@ -543,9 +543,11 @@ dev-gpt deploy --path {self.microservice_root_path}
 
 
     @staticmethod
-    def replace_with_gpt_3_5_turbo_if_possible(pkg):
+    def replace_with_tool_if_possible(pkg):
         if pkg in LANGUAGE_PACKAGES:
             return 'gpt_3_5_turbo'
+        if pkg in SEARCH_PACKAGES:
+            return 'google_custom_search'
         return pkg
 
     @staticmethod

dev_gpt/options/generate/pm/pm.py

@@ -60,7 +60,8 @@ Description of the microservice:
         microservice_description += self.user_input_extension_if_needed(
             context,
             microservice_description,
-            condition_question='Does the microservice send requests to an API?',
+            condition_question='''\
+Does the microservice send requests to an API beside the Google Custom Search API and gpt-3.5-turbo?''',
             question_gen='Generate a question that asks for the endpoint of the external API and an example of a request and response when interacting with the external API.',
             extension_name='Example of API usage',
             post_transformation_fn=translation(from_format='api instruction', to_format='python code snippet raw without formatting')

dev_gpt/options/generate/static_files/gateway/app-template

@@ -0,0 +1,53 @@
+import json
+import os
+import base64
+import streamlit as st
+from jina import Client, Document, DocumentArray
+
+st.set_page_config(
+    page_title="<page title here>",
+    page_icon="<page icon here>",
+    layout="<page layout here>",
+    initial_sidebar_state="<sidebar state here>",
+)
+
+st.title("<thematic emoji here> <header title here>")
+st.markdown(
+    "<10 word description here>"
+    "To deploy your own microservice, click [here](https://github.com/jina-ai/dev-gpt)."
+)
+
+st.header("<another thematic emoji here> Input Parameters") # only if input parameters are needed
+with st.form(key="input_form"):
+    <input parameter definition here>
+
+input_data = {
+    <input parameters here>
+}
+input_json = json.dumps(input_data)
+
+# Process input and call microservice
+if submit_button:
+    with st.spinner("Generating collage..."):
+
+
+        client = Client(host="http://localhost:8080")
+        d = Document(text=input_json)
+        response = client.post("/", inputs=DocumentArray([d]))
+
+        output_data = json.loads(response[0].text)
+        <visualization of results>
+
+# Display curl command
+deployment_id = os.environ.get("K8S_NAMESPACE_NAME", "")
+host = (
+    f"https://dev-gpt-{deployment_id.split('-')[1]}.wolf.jina.ai/post"
+    if deployment_id
+    else "http://localhost:8080/post"
+)
+with st.expander("See curl command"):
+    st.markdown("You can use the following curl command to send a request to the microservice from the command line:")
+    st.code(
+        f'curl -X "POST" "{host}" -H "accept: application/json" -H "Content-Type: application/json" -d \'{{"data": [{{"text": "{input_json}"}}]}}\'',
+        language="bash",
+    )

dev_gpt/options/generate/static_files/microservice/apis.py

@@ -21,3 +21,35 @@ class GPT_3_5_Turbo:
             }]
         )
         return response.choices[0]['message']['content']
+
+
+
+import os
+from typing import Optional
+
+import requests
+
+
+def google_search(search_term, search_type, top_n):
+    google_api_key: Optional[str] = os.environ['GOOGLE_API_KEY']
+    google_cse_id: Optional[str] = os.environ['GOOGLE_CSE_ID']
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {
+        'q': search_term,
+        'key': google_api_key,
+        'cx': google_cse_id,
+        'searchType': search_type,
+        'num': top_n
+    }
+    response = requests.get(url, params=params)
+    response.raise_for_status()
+    return response.json()
+
+def search_images(search_term, top_n):
+    response = google_search(search_term, search_type="image", top_n=top_n)
+    return [item["link"] for item in response["items"]]
+
+def search_web(search_term, top_n):
+    response = google_search(search_term, search_type="web", top_n=top_n)
+    return [item["snippet"] for item in response["items"]]
+

dev_gpt/options/generate/static_files/microservice/search.py

@@ -1,29 +0,0 @@
-import os
-from typing import Optional
-
-import requests
-
-
-def google_search(search_term, search_type, top_n):
-    google_api_key: Optional[str] = os.environ['GOOGLE_API_KEY']
-    google_cse_id: Optional[str] = os.environ['GOOGLE_CSE_ID']
-    url = "https://www.googleapis.com/customsearch/v1"
-    params = {
-        'q': search_term,
-        'key': google_api_key,
-        'cx': google_cse_id,
-        'searchType': search_type,
-        'num': top_n
-    }
-    response = requests.get(url, params=params)
-    response.raise_for_status()
-    return response.json()
-
-def search_image(search_term, top_n):
-    response = google_search(search_term, search_type="image", top_n=top_n)
-    return [item["link"] for item in response["items"]]
-
-def search_web(search_term, top_n):
-    response = google_search(search_term, search_type="web", top_n=top_n)
-    return [item["snippet"] for item in response["items"]]
-

dev_gpt/options/generate/templates_user.py

@@ -91,7 +91,7 @@ Note that you must obey the double asterisk and triple backtick syntax from like
 You must provide the complete file with the exact same syntax to wrap the code.'''
 
 
-gpt_35_turbo_usage_string = """If need to use gpt_3_5_turbo, then this is an example on how to use it:
+gpt_35_turbo_usage_string = """If you need to use gpt_3_5_turbo, then use it like shown in the following example:
 ```
 from .apis import GPT_3_5_Turbo
 
@@ -106,17 +106,35 @@ generated_string = gpt(prompt)  # fill-in the prompt (str); the output is a stri
 ```
 """
 
+google_custom_search_usage_string = """If you need to use google_custom_search, then use it like shown in the following example:
+a) when searching for text:
+```
+from .apis import search_web
+
+# input: search term (str), top_n (int)
+# output: list of strings
+string_list = search_web('<search term>', top_n=10)
+```
+b) when searching for images:
+```
+from .apis import search_images
+
+# input: search term (str), top_n (int)
+# output: list of image urls
+image_url_list = search_images('<search term>', top_n=10)
+```
+"""
 
 template_generate_function = PromptTemplate.from_template(
-    general_guidelines_string + '''
+    general_guidelines_string + f'''
 
 Write a python function which receives as \
 input json string (that can be parsed with the python function json.loads) and \
 outputs a json string (that can be parsed with the python function json.loads). \
 The function is called 'func'.
-The function must fulfill the following description: '{microservice_description}'.
-It will be tested with the following scenario: '{test_description}'.
-For the implementation use the following package(s): '{packages}'.
+The function must fulfill the following description: '{{microservice_description}}'.
+It will be tested with the following scenario: '{{test_description}}'.
+For the implementation use the following package(s): '{{packages}}'.
 
 The code must start with the following imports:
 ```
@@ -124,14 +142,16 @@ from .apis import GPT_3_5_Turbo
 import json
 ```
 Obey the following rules:
-''' + not_allowed_function_string + '''
+{not_allowed_function_string}
 
 Your approach:
 1. Identify the core challenge when implementing the function.
 2. Think about solutions for these challenges.
 3. Decide for one of the solutions.
 4. Write the code for the function. Don't write code for the test.
-''' + gpt_35_turbo_usage_string + '\n' + template_code_wrapping_string
+{gpt_35_turbo_usage_string}
+{google_custom_search_usage_string}
+{template_code_wrapping_string}'''
 )
 
 

dev_gpt/options/generate/tools/tools.py

@@ -0,0 +1,9 @@
+import os
+
+
+def get_available_tools():
+    tools = ['gpt-3.5-turbo (for any kind of text processing like summarization, paraphrasing, etc.)']
+    if os.environ.get('GOOGLE_API_KEY') and os.environ.get('GOOGLE_CSE_ID'):
+        tools.append('Google Custom Search API')
+    chars = 'abcdefghijklmnopqrstuvwxyz'
+    return '\n'.join([f'{char}) {tool}' for tool, char in zip(tools, chars)])

test/unit/test_search.py

@@ -1,4 +1,4 @@
-from dev_gpt.options.generate.static_files.microservice.search import search_web, search_image
+from dev_gpt.options.generate.static_files.microservice.search import search_web, search_images
 
 
 def test_web_search():
@@ -8,6 +8,6 @@ def test_web_search():
     assert not results[0].startswith("http")
 
 def test_image_search():
-    results = search_image("jina", 10)
+    results = search_images("jina", 10)
     assert len(results) == 10
     assert results[0].startswith("http")

test/unit/test_tools.py

@@ -0,0 +1,13 @@
+import os
+
+from dev_gpt.options.generate.tools.tools import get_available_tools
+
+
+def test_all_tools():
+    tool_lines = get_available_tools().split('\n')
+    assert len(tool_lines) == 2
+
+def test_no_search():
+    os.environ['GOOGLE_API_KEY'] = ''
+    tool_lines = get_available_tools().split('\n')
+    assert len(tool_lines) == 1