From f2ba7f21c510fabe226fac661df0e9112674708a Mon Sep 17 00:00:00 2001 From: russellocean Date: Mon, 3 Apr 2023 16:44:10 -0400 Subject: [PATCH 1/6] Added support for Google Custom Search API This pull request adds functionality to the project to allow for a choice between the original Google search method and the Google Custom Search API. The google_search method uses the original method of scraping the HTML from the search results page, using googlesearch-python, while the google_official_search method uses the Google Custom Search API to retrieve search results. How to test: To test the functionality, ensure that you have valid API keys and search engine IDs for both the Google search method and the Google Custom Search API. You can set these values in your environment variables as described in the README.md file. Additional Notes: This pull request only adds functionality and makes improvements to existing code. No new features or major changes have been introduced. --- .DS_Store | Bin 0 -> 8196 bytes .env.template | 4 --- README.md | 52 ++++++++++++++++++++++++++++++----- auto_gpt_workspace/.DS_Store | Bin 0 -> 6148 bytes requirements.txt | 1 + scripts/commands.py | 45 +++++++++++++++++++++++++++++- scripts/config.py | 11 ++++++-- 7 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 .DS_Store delete mode 100644 .env.template create mode 100644 auto_gpt_workspace/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..2287363ff8124dfd2cf4b68bc7738e40b201829b GIT binary patch literal 8196 zcmeHM&2G~`5T0#QoU{T(6-WhvWQl7U(m$k%OPVH!O2DB(Z~zqSB(zo>JH$=`iX!C> zyaKPlod@AvIKel&P82%{LLi~4-IaE~{h1kiznQf&4iSk`udze4MMP;-#_R@~1%=nS z&XtvDW)mo&PbaQlw_V@HI6Eh;Co}_^0nLDBKr^5jxCjj3oy{e^G8Q8oE0m88G%^JMrZ6oF`tqkg(BuhVF~YGz%%BLR3RJ2hN-;!~IS8eW^I6cy@LGoTrmXMl6}6y>Q&9`)n-yZDH;FF$3GiY#|V z4uuK19|<{QB1h-cBReilM>w=MW~{DHr5#xJ$fqtf6FOfLI-6v{atPTPbUNtOXdpFP z6ku#oJ!wPS`x_hfAm~J2G``s#*9pQx;k#H_O|4xsjEu2iysn=6gKB7ny`W@uU-Iag z@4NP-Z`seC=Af3ndF*?k<#^4mRB#$ih`f5?cnyD0@_Sw*kh)d1kjbPowd|eYaC>KW z?=WW`6!u3sb9lI0$eH_l4@aX+`qupik4|eXx9$0#q<2U}k4Cz##?$sQdVj=NzuNU& z-#ddm+ghhGIIU9)(b1*1@VuL3mAy&%8E@W(j0GRCN4oHl2ehDB!(0=yA^IMC7QpKt zu+k=JD)f}9REbLyA|rNv19!}k(UENsm*yCVNoVL3c8{>nC%`=J zrjgL#GcdcqyBwYL4%QvnYs-@pVn5s1jf*QYS04Jj(>_Ld^xiK#GqZNHOYJVym%PuO zO#^pOhYd%bhQz)lOTUr6Fj0Q+k5QgrAra4O(mNcR34UZ;%<==L>!KOBS5T32oG!&r+1@W}twQ8$Y5HGRTgI6PZP^nE@G#Im`N$sH&@&NimK8UZQ zGrL<+t2Ysq8JPVh^OI!14Z9ft5cNT)3Qz)o0xDt7#o-g7IO&w+tY-#M=zH`ah2$_& z?I4=UX2)-2fY#1|1lrJt5Ze{ zbv1}PiLUicz;z35v$Q-KtyVWScgnI>sg28Ww6jqu%dO4r@z^aauB`7KHE*KZSl#P4 z2Z3LymU)X4ctGO=3wM4$j#PYwkxA7gn~@n{2AF}r&Vb$53dO%Z5q^JWfEoC9255hf zsDz%y%%I*naL_FPVjkUE(59b8s*x5wi|~n xW=UtQM7=~Mp}frCYY7_qD8^hmikDEepkI=K=vmARq6dY41T+miFatlzzzfnIS;hbW literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index 5f51bdf5..e1a98ee7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ requests tiktoken==0.3.3 docker googlesearch-python +google-api-python-client # If using Google's Custom Search JSON API (https://developers.google.com/custom-search/v1/overview) Won't result in a 403 error # Googlesearch python seems to be a bit cursed, anyone good at fixing thigns like this? \ No newline at end of file diff --git a/scripts/commands.py b/scripts/commands.py index 2e332711..f94fdc41 100644 --- a/scripts/commands.py +++ b/scripts/commands.py @@ -10,6 +10,9 @@ from file_operations import read_file, write_to_file, append_to_file, delete_fil from execute_code import execute_python_file from json_parser import fix_and_parse_json from googlesearch import search +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + cfg = Config() @@ -44,7 +47,13 @@ def get_command(response): def execute_command(command_name, arguments): try: if command_name == "google": - return google_search(arguments["input"]) + print("Using Google search method") + # Check if the Google API key is set and use the official search method + # If the API key is not set or has only whitespaces, use the unofficial search method + if cfg.google_api_key and (cfg.google_api_key.strip() if cfg.google_api_key else None): + return google_official_search(arguments["input"]) + else: + return google_search(arguments["input"]) elif command_name == "memory_add": return commit_memory(arguments["string"]) elif command_name == "memory_del": @@ -108,6 +117,40 @@ def google_search(query, num_results=8): return json.dumps(search_results, ensure_ascii=False, indent=4) +def google_official_search(query, num_results=8): + from googleapiclient.discovery import build + from googleapiclient.errors import HttpError + import json + + try: + # Get the Google API key and Custom Search Engine ID from the config file + api_key = cfg.google_api_key + custom_search_engine_id = cfg.custom_search_engine_id + + # Initialize the Custom Search API service + service = build("customsearch", "v1", developerKey=api_key) + + # Send the search query and retrieve the results + result = service.cse().list(q=query, cx=custom_search_engine_id, num=num_results).execute() + + # Extract the search result items from the response + search_results = result.get("items", []) + + # Create a list of only the URLs from the search results + search_results_links = [item["link"] for item in search_results] + + except HttpError as e: + # Handle errors in the API call + error_details = json.loads(e.content.decode()) + + # Check if the error is related to an invalid or missing API key + if error_details.get("error", {}).get("code") == 403 and "invalid API key" in error_details.get("error", {}).get("message", ""): + return "Error: The provided Google API key is invalid or missing." + else: + return f"Error: {e}" + + # Return the list of search result URLs + return search_results_links def browse_website(url): summary = get_text_summary(url) diff --git a/scripts/config.py b/scripts/config.py index 44d99bff..51e11757 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -35,11 +35,13 @@ class Config(metaclass=Singleton): self.openai_api_key = os.getenv("OPENAI_API_KEY") self.elevenlabs_api_key = os.getenv("ELEVENLABS_API_KEY") + + self.google_api_key = os.getenv("GOOGLE_API_KEY") + self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID") # Initialize the OpenAI API client openai.api_key = self.openai_api_key - def set_continuous_mode(self, value: bool): self.continuous_mode = value @@ -63,6 +65,9 @@ class Config(metaclass=Singleton): def set_elevenlabs_api_key(self, value: str): self.elevenlabs_api_key = value - - + def set_google_api_key(self, value: str): + self.google_api_key = value + + def set_custom_search_engine_id(self, value: str): + self.custom_search_engine_id = value \ No newline at end of file From 30d07d9102b528ee7d0a324d860904f2211339b6 Mon Sep 17 00:00:00 2001 From: russellocean Date: Mon, 3 Apr 2023 16:49:05 -0400 Subject: [PATCH 2/6] Added google-api-python-client to requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e1a98ee7..063931a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ requests tiktoken==0.3.3 docker googlesearch-python -google-api-python-client # If using Google's Custom Search JSON API (https://developers.google.com/custom-search/v1/overview) Won't result in a 403 error +google-api-python-client #(https://developers.google.com/custom-search/v1/overview) # Googlesearch python seems to be a bit cursed, anyone good at fixing thigns like this? \ No newline at end of file From cb6e8ee665deca39db20fde78a976693776d9aa3 Mon Sep 17 00:00:00 2001 From: russellocean Date: Mon, 3 Apr 2023 16:58:55 -0400 Subject: [PATCH 3/6] Template update and Added .DS_Store to .gitignore --- .DS_Store | Bin 8196 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 2287363ff8124dfd2cf4b68bc7738e40b201829b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHM&2G~`5T0#QoU{T(6-WhvWQl7U(m$k%OPVH!O2DB(Z~zqSB(zo>JH$=`iX!C> zyaKPlod@AvIKel&P82%{LLi~4-IaE~{h1kiznQf&4iSk`udze4MMP;-#_R@~1%=nS z&XtvDW)mo&PbaQlw_V@HI6Eh;Co}_^0nLDBKr^5jxCjj3oy{e^G8Q8oE0m88G%^JMrZ6oF`tqkg(BuhVF~YGz%%BLR3RJ2hN-;!~IS8eW^I6cy@LGoTrmXMl6}6y>Q&9`)n-yZDH;FF$3GiY#|V z4uuK19|<{QB1h-cBReilM>w=MW~{DHr5#xJ$fqtf6FOfLI-6v{atPTPbUNtOXdpFP z6ku#oJ!wPS`x_hfAm~J2G``s#*9pQx;k#H_O|4xsjEu2iysn=6gKB7ny`W@uU-Iag z@4NP-Z`seC=Af3ndF*?k<#^4mRB#$ih`f5?cnyD0@_Sw*kh)d1kjbPowd|eYaC>KW z?=WW`6!u3sb9lI0$eH_l4@aX+`qupik4|eXx9$0#q<2U}k4Cz##?$sQdVj=NzuNU& z-#ddm+ghhGIIU9)(b1*1@VuL3mAy&%8E@W(j0GRCN4oHl2ehDB!(0=yA^IMC7QpKt zu+k=JD)f}9REbLyA|rNv19!}k(UENsm*yCVNoVL3c8{>nC%`=J zrjgL#GcdcqyBwYL4%QvnYs-@pVn5s1jf*QYS04Jj(>_Ld^xiK#GqZNHOYJVym%PuO zO#^pOhYd%bhQz)lOTUr6Fj0Q+k5QgrAra4O(mNcR34UZ;%<==L>!KOB Date: Mon, 3 Apr 2023 17:07:53 -0400 Subject: [PATCH 4/6] Delete .DS_Store --- auto_gpt_workspace/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 auto_gpt_workspace/.DS_Store diff --git a/auto_gpt_workspace/.DS_Store b/auto_gpt_workspace/.DS_Store deleted file mode 100644 index aa6e57de2aafac3a20e6c9f853a58e8eaf02c8e3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5T32oG!&r+1@W}twQ8$Y5HGRTgI6PZP^nE@G#Im`N$sH&@&NimK8UZQ zGrL<+t2Ysq8JPVh^OI!14Z9ft5cNT)3Qz)o0xDt7#o-g7IO&w+tY-#M=zH`ah2$_& z?I4=UX2)-2fY#1|1lrJt5Ze{ zbv1}PiLUicz;z35v$Q-KtyVWScgnI>sg28Ww6jqu%dO4r@z^aauB`7KHE*KZSl#P4 z2Z3LymU)X4ctGO=3wM4$j#PYwkxA7gn~@n{2AF}r&Vb$53dO%Z5q^JWfEoC9255hf zsDz%y%%I*naL_FPVjkUE(59b8s*x5wi|~n xW=UtQM7=~Mp}frCYY7_qD8^hmikDEepkI=K=vmARq6dY41T+miFatlzzzfnIS;hbW From 064a2af9b594de65107abd4f00eb91079e14d51e Mon Sep 17 00:00:00 2001 From: russellocean Date: Mon, 3 Apr 2023 17:29:55 -0400 Subject: [PATCH 5/6] Added back .env.template --- .env.template | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .env.template diff --git a/.env.template b/.env.template new file mode 100644 index 00000000..c64d8502 --- /dev/null +++ b/.env.template @@ -0,0 +1,6 @@ +OPENAI_API_KEY=your-openai-api-key +ELEVENLABS_API_KEY=your-elevenlabs-api-key +SMART_LLM_MODEL="gpt-4" +FAST_LLM_MODEL="gpt-3.5-turbo" +GOOGLE_API_KEY= +CUSTOM_SEARCH_ENGINE_ID= \ No newline at end of file From 7e529e19d9b97c2e99720bd6f1bc829f7c5e8597 Mon Sep 17 00:00:00 2001 From: Toran Bruce Richards Date: Tue, 4 Apr 2023 00:24:22 +0100 Subject: [PATCH 6/6] Removes print. --- scripts/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/commands.py b/scripts/commands.py index f94fdc41..8ad95336 100644 --- a/scripts/commands.py +++ b/scripts/commands.py @@ -47,7 +47,7 @@ def get_command(response): def execute_command(command_name, arguments): try: if command_name == "google": - print("Using Google search method") + # Check if the Google API key is set and use the official search method # If the API key is not set or has only whitespaces, use the unofficial search method if cfg.google_api_key and (cfg.google_api_key.strip() if cfg.google_api_key else None):