mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2025-12-17 22:14:28 +01:00
Forge/workshop (#5654)
* Added basic memory * Added action history * Deleted placeholder files * adding memstore * Added web search ability * Added web search and reading web pages * remove agent.py changes Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com> --------- Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com> Co-authored-by: SwiftyOS <craigswift13@gmail.com>
This commit is contained in:
@@ -7,7 +7,12 @@ from forge.sdk import (
|
||||
Task,
|
||||
TaskRequestBody,
|
||||
Workspace,
|
||||
PromptEngine,
|
||||
chat_completion_request,
|
||||
ChromaMemStore
|
||||
)
|
||||
import json
|
||||
import pprint
|
||||
|
||||
LOG = ForgeLogger(__name__)
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import os
|
||||
|
||||
from forge.agent import ForgeAgent
|
||||
from forge.sdk import AgentDB, LocalWorkspace
|
||||
from forge.sdk import LocalWorkspace
|
||||
from .db import ForgeDatabase
|
||||
|
||||
database_name = os.getenv("DATABASE_STRING")
|
||||
workspace = LocalWorkspace(os.getenv("AGENT_WORKSPACE"))
|
||||
database = AgentDB(database_name, debug_enabled=False)
|
||||
database = ForgeDatabase(database_name, debug_enabled=False)
|
||||
agent = ForgeAgent(database=database, workspace=workspace)
|
||||
|
||||
app = agent.get_agent_app()
|
||||
|
||||
145
autogpts/forge/forge/db.py
Normal file
145
autogpts/forge/forge/db.py
Normal file
@@ -0,0 +1,145 @@
|
||||
from .sdk import AgentDB, ForgeLogger, NotFoundError, Base
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
import datetime
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
DateTime,
|
||||
String,
|
||||
)
|
||||
import uuid
|
||||
|
||||
LOG = ForgeLogger(__name__)
|
||||
|
||||
class ChatModel(Base):
|
||||
__tablename__ = "chat"
|
||||
msg_id = Column(String, primary_key=True, index=True)
|
||||
task_id = Column(String)
|
||||
role = Column(String)
|
||||
content = Column(String)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
modified_at = Column(
|
||||
DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow
|
||||
)
|
||||
|
||||
class ActionModel(Base):
|
||||
__tablename__ = "action"
|
||||
action_id = Column(String, primary_key=True, index=True)
|
||||
task_id = Column(String)
|
||||
name = Column(String)
|
||||
args = Column(String)
|
||||
created_at = Column(DateTime, default=datetime.datetime.utcnow)
|
||||
modified_at = Column(
|
||||
DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow
|
||||
)
|
||||
|
||||
|
||||
class ForgeDatabase(AgentDB):
|
||||
|
||||
async def add_chat_history(self, task_id, messages):
|
||||
for message in messages:
|
||||
await self.add_chat_message(task_id, message['role'], message['content'])
|
||||
|
||||
async def add_chat_message(self, task_id, role, content):
|
||||
if self.debug_enabled:
|
||||
LOG.debug("Creating new task")
|
||||
try:
|
||||
with self.Session() as session:
|
||||
mew_msg = ChatModel(
|
||||
msg_id=str(uuid.uuid4()),
|
||||
task_id=task_id,
|
||||
role=role,
|
||||
content=content,
|
||||
)
|
||||
session.add(mew_msg)
|
||||
session.commit()
|
||||
session.refresh(mew_msg)
|
||||
if self.debug_enabled:
|
||||
LOG.debug(f"Created new Chat message with task_id: {mew_msg.msg_id}")
|
||||
return mew_msg
|
||||
except SQLAlchemyError as e:
|
||||
LOG.error(f"SQLAlchemy error while creating task: {e}")
|
||||
raise
|
||||
except NotFoundError as e:
|
||||
raise
|
||||
except Exception as e:
|
||||
LOG.error(f"Unexpected error while creating task: {e}")
|
||||
raise
|
||||
|
||||
async def get_chat_history(self, task_id):
|
||||
if self.debug_enabled:
|
||||
LOG.debug(f"Getting chat history with task_id: {task_id}")
|
||||
try:
|
||||
with self.Session() as session:
|
||||
if messages := (
|
||||
session.query(ChatModel)
|
||||
.filter(ChatModel.task_id == task_id)
|
||||
.order_by(ChatModel.created_at)
|
||||
.all()
|
||||
):
|
||||
return [{"role": m.role, "content": m.content} for m in messages]
|
||||
|
||||
else:
|
||||
LOG.error(
|
||||
f"Chat history not found with task_id: {task_id}"
|
||||
)
|
||||
raise NotFoundError("Chat history not found")
|
||||
except SQLAlchemyError as e:
|
||||
LOG.error(f"SQLAlchemy error while getting chat history: {e}")
|
||||
raise
|
||||
except NotFoundError as e:
|
||||
raise
|
||||
except Exception as e:
|
||||
LOG.error(f"Unexpected error while getting chat history: {e}")
|
||||
raise
|
||||
|
||||
async def create_action(self, task_id, name, args):
|
||||
try:
|
||||
with self.Session() as session:
|
||||
new_action = ActionModel(
|
||||
action_id=str(uuid.uuid4()),
|
||||
task_id=task_id,
|
||||
name=name,
|
||||
args=str(args),
|
||||
)
|
||||
session.add(new_action)
|
||||
session.commit()
|
||||
session.refresh(new_action)
|
||||
if self.debug_enabled:
|
||||
LOG.debug(f"Created new Action with task_id: {new_action.action_id}")
|
||||
return new_action
|
||||
except SQLAlchemyError as e:
|
||||
LOG.error(f"SQLAlchemy error while creating action: {e}")
|
||||
raise
|
||||
except NotFoundError as e:
|
||||
raise
|
||||
except Exception as e:
|
||||
LOG.error(f"Unexpected error while creating action: {e}")
|
||||
raise
|
||||
|
||||
async def get_action_history(self, task_id):
|
||||
if self.debug_enabled:
|
||||
LOG.debug(f"Getting action history with task_id: {task_id}")
|
||||
try:
|
||||
with self.Session() as session:
|
||||
if actions := (
|
||||
session.query(ActionModel)
|
||||
.filter(ActionModel.task_id == task_id)
|
||||
.order_by(ActionModel.created_at)
|
||||
.all()
|
||||
):
|
||||
return [{"name": a.name, "args": a.args} for a in actions]
|
||||
|
||||
else:
|
||||
LOG.error(
|
||||
f"Action history not found with task_id: {task_id}"
|
||||
)
|
||||
raise NotFoundError("Action history not found")
|
||||
except SQLAlchemyError as e:
|
||||
LOG.error(f"SQLAlchemy error while getting action history: {e}")
|
||||
raise
|
||||
except NotFoundError as e:
|
||||
raise
|
||||
except Exception as e:
|
||||
LOG.error(f"Unexpected error while getting action history: {e}")
|
||||
raise
|
||||
@@ -9,7 +9,7 @@ Reply only in json with the following format:
|
||||
\"speak\": \"thoughts summary to say to user\",
|
||||
},
|
||||
\"ability\": {
|
||||
\"name\": {\"type\": \"string\"},
|
||||
\"name\": \"ability name\",
|
||||
\"args\": {
|
||||
\"arg1\": \"value1", etc...
|
||||
}
|
||||
|
||||
@@ -40,4 +40,11 @@ You have access to the following abilities you can call:
|
||||
- {{ best_practice }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% if previous_actions %}
|
||||
## History of Abilities Used
|
||||
{% for action in previous_actions %}
|
||||
- {{ action }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -3,7 +3,7 @@ The Forge SDK. This is the core of the Forge. It contains the agent protocol, wh
|
||||
core of the Forge.
|
||||
"""
|
||||
from .agent import Agent
|
||||
from .db import AgentDB
|
||||
from .db import AgentDB, Base
|
||||
from .forge_log import ForgeLogger
|
||||
from .llm import chat_completion_request, create_embedding_request, transcribe_audio
|
||||
from .prompting import PromptEngine
|
||||
@@ -22,3 +22,6 @@ from .schema import (
|
||||
TaskStepsListResponse,
|
||||
)
|
||||
from .workspace import LocalWorkspace, Workspace
|
||||
from .errors import *
|
||||
from .memory.chroma_memstore import ChromaMemStore
|
||||
from .memory.memstore import MemStore
|
||||
@@ -2,7 +2,6 @@ from typing import List
|
||||
|
||||
from ..registry import ability
|
||||
|
||||
|
||||
@ability(
|
||||
name="list_files",
|
||||
description="List files in a directory",
|
||||
@@ -20,7 +19,7 @@ async def list_files(agent, task_id: str, path: str) -> List[str]:
|
||||
"""
|
||||
List files in a workspace directory
|
||||
"""
|
||||
return agent.workspace.list(task_id=task_id, path=path)
|
||||
return agent.workspace.list(task_id=task_id, path=str(path))
|
||||
|
||||
|
||||
@ability(
|
||||
@@ -42,7 +41,7 @@ async def list_files(agent, task_id: str, path: str) -> List[str]:
|
||||
],
|
||||
output_type="None",
|
||||
)
|
||||
async def write_file(agent, task_id: str, file_path: str, data: bytes) -> None:
|
||||
async def write_file(agent, task_id: str, file_path: str, data: bytes):
|
||||
"""
|
||||
Write data to a file
|
||||
"""
|
||||
@@ -50,7 +49,7 @@ async def write_file(agent, task_id: str, file_path: str, data: bytes) -> None:
|
||||
data = data.encode()
|
||||
|
||||
agent.workspace.write(task_id=task_id, path=file_path, data=data)
|
||||
await agent.db.create_artifact(
|
||||
return await agent.db.create_artifact(
|
||||
task_id=task_id,
|
||||
file_name=file_path.split("/")[-1],
|
||||
relative_path=file_path,
|
||||
|
||||
75
autogpts/forge/forge/sdk/abilities/web/web_search.py
Normal file
75
autogpts/forge/forge/sdk/abilities/web/web_search.py
Normal file
@@ -0,0 +1,75 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from itertools import islice
|
||||
|
||||
from duckduckgo_search import DDGS
|
||||
|
||||
from ..registry import ability
|
||||
|
||||
DUCKDUCKGO_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
@ability(
|
||||
name="web_search",
|
||||
description="Searches the web",
|
||||
parameters=[
|
||||
{
|
||||
"name": "query",
|
||||
"description": "The search query",
|
||||
"type": "string",
|
||||
"required": True,
|
||||
}
|
||||
],
|
||||
output_type="list[str]",
|
||||
)
|
||||
async def web_search(agent, task_id: str, query: str) -> str:
|
||||
"""Return the results of a Google search
|
||||
|
||||
Args:
|
||||
query (str): The search query.
|
||||
num_results (int): The number of results to return.
|
||||
|
||||
Returns:
|
||||
str: The results of the search.
|
||||
"""
|
||||
search_results = []
|
||||
attempts = 0
|
||||
num_results = 8
|
||||
|
||||
while attempts < DUCKDUCKGO_MAX_ATTEMPTS:
|
||||
if not query:
|
||||
return json.dumps(search_results)
|
||||
|
||||
results = DDGS().text(query)
|
||||
search_results = list(islice(results, num_results))
|
||||
|
||||
if search_results:
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
attempts += 1
|
||||
|
||||
results = json.dumps(search_results, ensure_ascii=False, indent=4)
|
||||
return safe_google_results(results)
|
||||
|
||||
|
||||
def safe_google_results(results: str | list) -> str:
|
||||
"""
|
||||
Return the results of a Google search in a safe format.
|
||||
|
||||
Args:
|
||||
results (str | list): The search results.
|
||||
|
||||
Returns:
|
||||
str: The results of the search.
|
||||
"""
|
||||
if isinstance(results, list):
|
||||
safe_message = json.dumps(
|
||||
[result.encode("utf-8", "ignore").decode("utf-8") for result in results]
|
||||
)
|
||||
else:
|
||||
safe_message = results.encode("utf-8", "ignore").decode("utf-8")
|
||||
return safe_message
|
||||
375
autogpts/forge/forge/sdk/abilities/web/web_selenium.py
Normal file
375
autogpts/forge/forge/sdk/abilities/web/web_selenium.py
Normal file
@@ -0,0 +1,375 @@
|
||||
"""Commands for browsing a website"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
COMMAND_CATEGORY = "web_browse"
|
||||
COMMAND_CATEGORY_TITLE = "Web Browsing"
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from sys import platform
|
||||
from typing import TYPE_CHECKING, Optional, Type, List, Tuple
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium.common.exceptions import WebDriverException
|
||||
from selenium.webdriver.chrome.options import Options as ChromeOptions
|
||||
from selenium.webdriver.chrome.service import Service as ChromeDriverService
|
||||
from selenium.webdriver.chrome.webdriver import WebDriver as ChromeDriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.options import ArgOptions as BrowserOptions
|
||||
from selenium.webdriver.edge.options import Options as EdgeOptions
|
||||
from selenium.webdriver.edge.service import Service as EdgeDriverService
|
||||
from selenium.webdriver.edge.webdriver import WebDriver as EdgeDriver
|
||||
from selenium.webdriver.firefox.options import Options as FirefoxOptions
|
||||
from selenium.webdriver.firefox.service import Service as GeckoDriverService
|
||||
from selenium.webdriver.firefox.webdriver import WebDriver as FirefoxDriver
|
||||
from selenium.webdriver.remote.webdriver import WebDriver
|
||||
from selenium.webdriver.safari.options import Options as SafariOptions
|
||||
from selenium.webdriver.safari.webdriver import WebDriver as SafariDriver
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.wait import WebDriverWait
|
||||
from webdriver_manager.chrome import ChromeDriverManager
|
||||
from webdriver_manager.firefox import GeckoDriverManager
|
||||
from webdriver_manager.microsoft import EdgeChromiumDriverManager as EdgeDriverManager
|
||||
|
||||
|
||||
from ..registry import ability
|
||||
from forge.sdk.errors import *
|
||||
import functools
|
||||
import re
|
||||
from typing import Any, Callable
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from requests.compat import urljoin
|
||||
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from requests.compat import urljoin
|
||||
|
||||
|
||||
def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> list[tuple[str, str]]:
|
||||
"""Extract hyperlinks from a BeautifulSoup object
|
||||
|
||||
Args:
|
||||
soup (BeautifulSoup): The BeautifulSoup object
|
||||
base_url (str): The base URL
|
||||
|
||||
Returns:
|
||||
List[Tuple[str, str]]: The extracted hyperlinks
|
||||
"""
|
||||
return [
|
||||
(link.text, urljoin(base_url, link["href"]))
|
||||
for link in soup.find_all("a", href=True)
|
||||
]
|
||||
|
||||
|
||||
def format_hyperlinks(hyperlinks: list[tuple[str, str]]) -> list[str]:
|
||||
"""Format hyperlinks to be displayed to the user
|
||||
|
||||
Args:
|
||||
hyperlinks (List[Tuple[str, str]]): The hyperlinks to format
|
||||
|
||||
Returns:
|
||||
List[str]: The formatted hyperlinks
|
||||
"""
|
||||
return [f"{link_text} ({link_url})" for link_text, link_url in hyperlinks]
|
||||
|
||||
|
||||
|
||||
def validate_url(func: Callable[..., Any]) -> Any:
|
||||
"""The method decorator validate_url is used to validate urls for any command that requires
|
||||
a url as an argument"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(url: str, *args, **kwargs) -> Any:
|
||||
"""Check if the URL is valid using a basic check, urllib check, and local file check
|
||||
|
||||
Args:
|
||||
url (str): The URL to check
|
||||
|
||||
Returns:
|
||||
the result of the wrapped function
|
||||
|
||||
Raises:
|
||||
ValueError if the url fails any of the validation tests
|
||||
"""
|
||||
# Most basic check if the URL is valid:
|
||||
if not re.match(r"^https?://", url):
|
||||
raise ValueError("Invalid URL format")
|
||||
if not is_valid_url(url):
|
||||
raise ValueError("Missing Scheme or Network location")
|
||||
# Restrict access to local files
|
||||
if check_local_file_access(url):
|
||||
raise ValueError("Access to local files is restricted")
|
||||
# Check URL length
|
||||
if len(url) > 2000:
|
||||
raise ValueError("URL is too long")
|
||||
|
||||
return func(sanitize_url(url), *args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def is_valid_url(url: str) -> bool:
|
||||
"""Check if the URL is valid
|
||||
|
||||
Args:
|
||||
url (str): The URL to check
|
||||
|
||||
Returns:
|
||||
bool: True if the URL is valid, False otherwise
|
||||
"""
|
||||
try:
|
||||
result = urlparse(url)
|
||||
return all([result.scheme, result.netloc])
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def sanitize_url(url: str) -> str:
|
||||
"""Sanitize the URL
|
||||
|
||||
Args:
|
||||
url (str): The URL to sanitize
|
||||
|
||||
Returns:
|
||||
str: The sanitized URL
|
||||
"""
|
||||
parsed_url = urlparse(url)
|
||||
reconstructed_url = f"{parsed_url.path}{parsed_url.params}?{parsed_url.query}"
|
||||
return urljoin(url, reconstructed_url)
|
||||
|
||||
|
||||
def check_local_file_access(url: str) -> bool:
|
||||
"""Check if the URL is a local file
|
||||
|
||||
Args:
|
||||
url (str): The URL to check
|
||||
|
||||
Returns:
|
||||
bool: True if the URL is a local file, False otherwise
|
||||
"""
|
||||
local_prefixes = [
|
||||
"file:///",
|
||||
"file://localhost/",
|
||||
"file://localhost",
|
||||
"http://localhost",
|
||||
"http://localhost/",
|
||||
"https://localhost",
|
||||
"https://localhost/",
|
||||
"http://2130706433",
|
||||
"http://2130706433/",
|
||||
"https://2130706433",
|
||||
"https://2130706433/",
|
||||
"http://127.0.0.1/",
|
||||
"http://127.0.0.1",
|
||||
"https://127.0.0.1/",
|
||||
"https://127.0.0.1",
|
||||
"https://0.0.0.0/",
|
||||
"https://0.0.0.0",
|
||||
"http://0.0.0.0/",
|
||||
"http://0.0.0.0",
|
||||
"http://0000",
|
||||
"http://0000/",
|
||||
"https://0000",
|
||||
"https://0000/",
|
||||
]
|
||||
return any(url.startswith(prefix) for prefix in local_prefixes)
|
||||
|
||||
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FILE_DIR = Path(__file__).parent.parent
|
||||
TOKENS_TO_TRIGGER_SUMMARY = 50
|
||||
LINKS_TO_RETURN = 20
|
||||
|
||||
|
||||
class BrowsingError(CommandExecutionError):
|
||||
"""An error occurred while trying to browse the page"""
|
||||
|
||||
|
||||
@ability(
|
||||
name="read_webpage",
|
||||
description="Read a webpage, and extract specific information from it if a question is specified. If you are looking to extract specific information from the webpage, you should specify a question.",
|
||||
parameters=[
|
||||
{
|
||||
"name": "url",
|
||||
"description": "The URL to visit",
|
||||
"type": "string",
|
||||
"required": True,
|
||||
},
|
||||
{
|
||||
"name": "question",
|
||||
"description": "A question that you want to answer using the content of the webpage.",
|
||||
"type": "string",
|
||||
"required": False,
|
||||
}
|
||||
],
|
||||
output_type="string",
|
||||
)
|
||||
@validate_url
|
||||
async def read_webpage(agent, task_id: str, url: str, question: str = "") -> Tuple(str, List[str]):
|
||||
"""Browse a website and return the answer and links to the user
|
||||
|
||||
Args:
|
||||
url (str): The url of the website to browse
|
||||
question (str): The question to answer using the content of the webpage
|
||||
|
||||
Returns:
|
||||
str: The answer and links to the user and the webdriver
|
||||
"""
|
||||
driver = None
|
||||
try:
|
||||
driver = open_page_in_browser(url)
|
||||
|
||||
text = scrape_text_with_selenium(driver)
|
||||
links = scrape_links_with_selenium(driver, url)
|
||||
|
||||
if not text:
|
||||
return f"Website did not contain any text.\n\nLinks: {links}"
|
||||
|
||||
# Limit links to LINKS_TO_RETURN
|
||||
if len(links) > LINKS_TO_RETURN:
|
||||
links = links[:LINKS_TO_RETURN]
|
||||
return (text, links)
|
||||
|
||||
except WebDriverException as e:
|
||||
# These errors are often quite long and include lots of context.
|
||||
# Just grab the first line.
|
||||
msg = e.msg.split("\n")[0]
|
||||
if "net::" in msg:
|
||||
raise BrowsingError(
|
||||
f"A networking error occurred while trying to load the page: "
|
||||
+ re.sub(r"^unknown error: ", "", msg)
|
||||
)
|
||||
raise CommandExecutionError(msg)
|
||||
finally:
|
||||
if driver:
|
||||
close_browser(driver)
|
||||
|
||||
|
||||
def scrape_text_with_selenium(driver: WebDriver) -> str:
|
||||
"""Scrape text from a browser window using selenium
|
||||
|
||||
Args:
|
||||
driver (WebDriver): A driver object representing the browser window to scrape
|
||||
|
||||
Returns:
|
||||
str: the text scraped from the website
|
||||
"""
|
||||
|
||||
# Get the HTML content directly from the browser's DOM
|
||||
page_source = driver.execute_script("return document.body.outerHTML;")
|
||||
soup = BeautifulSoup(page_source, "html.parser")
|
||||
|
||||
for script in soup(["script", "style"]):
|
||||
script.extract()
|
||||
|
||||
text = soup.get_text()
|
||||
lines = (line.strip() for line in text.splitlines())
|
||||
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
||||
text = "\n".join(chunk for chunk in chunks if chunk)
|
||||
return text
|
||||
|
||||
|
||||
def scrape_links_with_selenium(driver: WebDriver, base_url: str) -> list[str]:
|
||||
"""Scrape links from a website using selenium
|
||||
|
||||
Args:
|
||||
driver (WebDriver): A driver object representing the browser window to scrape
|
||||
base_url (str): The base URL to use for resolving relative links
|
||||
|
||||
Returns:
|
||||
List[str]: The links scraped from the website
|
||||
"""
|
||||
page_source = driver.page_source
|
||||
soup = BeautifulSoup(page_source, "html.parser")
|
||||
|
||||
for script in soup(["script", "style"]):
|
||||
script.extract()
|
||||
|
||||
hyperlinks = extract_hyperlinks(soup, base_url)
|
||||
|
||||
return format_hyperlinks(hyperlinks)
|
||||
|
||||
|
||||
def open_page_in_browser(url: str) -> WebDriver:
|
||||
"""Open a browser window and load a web page using Selenium
|
||||
|
||||
Params:
|
||||
url (str): The URL of the page to load
|
||||
|
||||
Returns:
|
||||
driver (WebDriver): A driver object representing the browser window to scrape
|
||||
"""
|
||||
logging.getLogger("selenium").setLevel(logging.CRITICAL)
|
||||
selenium_web_browser = "chrome"
|
||||
selenium_headless = True
|
||||
options_available: dict[str, Type[BrowserOptions]] = {
|
||||
"chrome": ChromeOptions,
|
||||
"edge": EdgeOptions,
|
||||
"firefox": FirefoxOptions,
|
||||
"safari": SafariOptions,
|
||||
}
|
||||
|
||||
options: BrowserOptions = options_available[selenium_web_browser]()
|
||||
options.add_argument(
|
||||
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36"
|
||||
)
|
||||
|
||||
if selenium_web_browser == "firefox":
|
||||
if selenium_headless:
|
||||
options.headless = True
|
||||
options.add_argument("--disable-gpu")
|
||||
driver = FirefoxDriver(
|
||||
service=GeckoDriverService(GeckoDriverManager().install()), options=options
|
||||
)
|
||||
elif selenium_web_browser == "edge":
|
||||
driver = EdgeDriver(
|
||||
service=EdgeDriverService(EdgeDriverManager().install()), options=options
|
||||
)
|
||||
elif selenium_web_browser == "safari":
|
||||
# Requires a bit more setup on the users end
|
||||
# See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari
|
||||
driver = SafariDriver(options=options)
|
||||
else:
|
||||
if platform == "linux" or platform == "linux2":
|
||||
options.add_argument("--disable-dev-shm-usage")
|
||||
options.add_argument("--remote-debugging-port=9222")
|
||||
|
||||
options.add_argument("--no-sandbox")
|
||||
if selenium_headless:
|
||||
options.add_argument("--headless=new")
|
||||
options.add_argument("--disable-gpu")
|
||||
|
||||
chromium_driver_path = Path("/usr/bin/chromedriver")
|
||||
|
||||
driver = ChromeDriver(
|
||||
service=ChromeDriverService(str(chromium_driver_path))
|
||||
if chromium_driver_path.exists()
|
||||
else ChromeDriverService(ChromeDriverManager().install()),
|
||||
options=options,
|
||||
)
|
||||
driver.get(url)
|
||||
|
||||
WebDriverWait(driver, 10).until(
|
||||
EC.presence_of_element_located((By.TAG_NAME, "body"))
|
||||
)
|
||||
|
||||
return driver
|
||||
|
||||
|
||||
def close_browser(driver: WebDriver) -> None:
|
||||
"""Close the browser
|
||||
|
||||
Args:
|
||||
driver (WebDriver): The webdriver to close
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
driver.quit()
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
"""
|
||||
PROFILE CONCEPT:
|
||||
|
||||
The profile generator is used to intiliase and configure an ai agent.
|
||||
It came from the obsivation that if an llm is provided with a profile such as:
|
||||
```
|
||||
Expert:
|
||||
|
||||
```
|
||||
Then it's performance at a task can impove. Here we use the profile to generate
|
||||
a system prompt for the agent to use. However, it can be used to configure other
|
||||
aspects of the agent such as memory, planning, and actions available.
|
||||
|
||||
The possibilities are limited just by your imagination.
|
||||
"""
|
||||
|
||||
from forge.sdk import PromptEngine
|
||||
|
||||
|
||||
class ProfileGenerator:
|
||||
def __init__(self, task: str, PromptEngine: PromptEngine):
|
||||
"""
|
||||
Initialize the profile generator with the task to be performed.
|
||||
"""
|
||||
self.task = task
|
||||
@@ -37,6 +37,7 @@ class LocalWorkspace(Workspace):
|
||||
self.base_path = Path(base_path).resolve()
|
||||
|
||||
def _resolve_path(self, task_id: str, path: str) -> Path:
|
||||
path = str(path)
|
||||
path = path if not path.startswith("/") else path[1:]
|
||||
abs_path = (self.base_path / task_id / path).resolve()
|
||||
if not str(abs_path).startswith(str(self.base_path)):
|
||||
@@ -77,4 +78,6 @@ class LocalWorkspace(Workspace):
|
||||
def list(self, task_id: str, path: str) -> typing.List[str]:
|
||||
path = self.base_path / task_id / path
|
||||
base = self._resolve_path(task_id, path)
|
||||
if not base.exists() or not base.is_dir():
|
||||
return []
|
||||
return [str(p.relative_to(self.base_path / task_id)) for p in base.iterdir()]
|
||||
|
||||
1050
autogpts/forge/poetry.lock
generated
1050
autogpts/forge/poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,9 @@ toml = "^0.10.2"
|
||||
jinja2 = "^3.1.2"
|
||||
uvicorn = "^0.23.2"
|
||||
litellm = "^0.1.821"
|
||||
duckduckgo-search = "^3.9.3"
|
||||
selenium = "^4.13.0"
|
||||
bs4 = "^0.0.1"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
isort = "^5.12.0"
|
||||
|
||||
16
cli.py
16
cli.py
@@ -301,14 +301,22 @@ def stop():
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
pid = int(subprocess.check_output(["lsof", "-t", "-i", ":8000"]))
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
pids = subprocess.check_output(["lsof", "-t", "-i", ":8000"]).split()
|
||||
if isinstance(pids, int):
|
||||
os.kill(int(pids), signal.SIGTERM)
|
||||
else:
|
||||
for pid in pids:
|
||||
os.kill(int(pid), signal.SIGTERM)
|
||||
except subprocess.CalledProcessError:
|
||||
click.echo("No process is running on port 8000")
|
||||
|
||||
try:
|
||||
pid = int(subprocess.check_output(["lsof", "-t", "-i", ":8080"]))
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
pids = int(subprocess.check_output(["lsof", "-t", "-i", ":8080"]))
|
||||
if isinstance(pids, int):
|
||||
os.kill(int(pids), signal.SIGTERM)
|
||||
else:
|
||||
for pid in pids:
|
||||
os.kill(int(pid), signal.SIGTERM)
|
||||
except subprocess.CalledProcessError:
|
||||
click.echo("No process is running on port 8080")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user