feat: add vision support for Google (#141)

This commit is contained in:
Drew Hintz
2024-10-13 17:40:03 -05:00
committed by GitHub
parent 6ea6d1448d
commit 9679f07d09
2 changed files with 16 additions and 4 deletions

View File

@@ -7,7 +7,7 @@ from exchange import Message, Tool
from exchange.content import Text, ToolResult, ToolUse
from exchange.providers.base import Provider, Usage
from tenacity import retry, wait_fixed, stop_after_attempt
from exchange.providers.utils import raise_for_status, retry_if_status
from exchange.providers.utils import raise_for_status, retry_if_status, encode_image
GOOGLE_HOST = "https://generativelanguage.googleapis.com/v1beta"
@@ -111,9 +111,20 @@ class GoogleProvider(Provider):
elif isinstance(content, ToolUse):
converted["parts"].append({"functionCall": {"name": content.name, "args": content.parameters}})
elif isinstance(content, ToolResult):
converted["parts"].append(
{"functionResponse": {"name": content.tool_use_id, "response": {"content": content.output}}}
)
if content.output.startswith('"image:'):
image_path = content.output.replace('"image:', "").replace('"', "")
converted["parts"].append(
{
"inline_data": {
"mime_type": "image/png",
"data": f"{encode_image(image_path)}",
}
}
)
else:
converted["parts"].append(
{"functionResponse": {"name": content.tool_use_id, "response": {"content": content.output}}}
)
messages_spec.append(converted)
if not messages_spec:

View File

@@ -9,6 +9,7 @@ from exchange.providers import get_provider
cases = [
(get_provider("openai"), os.getenv("OPENAI_MODEL", "gpt-4o-mini")),
(get_provider("google"), os.getenv("GOOGLE_MODEL", "gemini-1.5-flash")),
]