bridge: add multimodal image decoding support for AGY provider
This commit is contained in:
+34
-1
@@ -48,9 +48,31 @@ class AGYBridgeHandler(BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
system_instructions = []
|
system_instructions = []
|
||||||
user_prompts = []
|
user_prompts = []
|
||||||
|
temp_image_files = []
|
||||||
|
|
||||||
for msg in messages:
|
for msg in messages:
|
||||||
role = msg.get("role", "user")
|
role = msg.get("role", "user")
|
||||||
content = msg.get("content", "")
|
content = msg.get("content", "")
|
||||||
|
if isinstance(content, list):
|
||||||
|
for part in content:
|
||||||
|
if isinstance(part, dict):
|
||||||
|
p_type = part.get("type", "")
|
||||||
|
if p_type == "text":
|
||||||
|
txt = part.get("text", "")
|
||||||
|
if role == "system":
|
||||||
|
system_instructions.append(txt)
|
||||||
|
else:
|
||||||
|
user_prompts.append(txt)
|
||||||
|
elif p_type == "image_url":
|
||||||
|
url_data = part.get("image_url", {}).get("url", "")
|
||||||
|
if url_data.startswith("data:image/"):
|
||||||
|
import tempfile, base64
|
||||||
|
header, b64_str = url_data.split(",", 1)
|
||||||
|
ext = ".png" if "png" in header else ".jpg"
|
||||||
|
with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_f:
|
||||||
|
tmp_f.write(base64.b64decode(b64_str))
|
||||||
|
temp_image_files.append(tmp_f.name)
|
||||||
|
elif isinstance(content, str):
|
||||||
if role == "system":
|
if role == "system":
|
||||||
system_instructions.append(content)
|
system_instructions.append(content)
|
||||||
else:
|
else:
|
||||||
@@ -61,11 +83,15 @@ class AGYBridgeHandler(BaseHTTPRequestHandler):
|
|||||||
full_prompt += f"System Instruction:\n" + "\n".join(system_instructions) + "\n\n"
|
full_prompt += f"System Instruction:\n" + "\n".join(system_instructions) + "\n\n"
|
||||||
full_prompt += "User Prompt:\n" + "\n".join(user_prompts)
|
full_prompt += "User Prompt:\n" + "\n".join(user_prompts)
|
||||||
|
|
||||||
|
if temp_image_files:
|
||||||
|
full_prompt += "\n\nAttached Images to analyze:\n" + "\n".join([f"- {f}" for f in temp_image_files])
|
||||||
|
|
||||||
cmd = [AGY_BIN, "-p", full_prompt, "--output-format", "json"]
|
cmd = [AGY_BIN, "-p", full_prompt, "--output-format", "json"]
|
||||||
if effort in ("low", "medium", "high"):
|
if effort in ("low", "medium", "high"):
|
||||||
cmd.extend(["--effort", effort])
|
cmd.extend(["--effort", effort])
|
||||||
|
|
||||||
logger.info(f"Executing agy for prompt length: {len(full_prompt)} chars")
|
logger.info(f"Executing agy for prompt length: {len(full_prompt)} chars (images: {len(temp_image_files)})")
|
||||||
|
try:
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
cmd,
|
cmd,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
@@ -73,6 +99,13 @@ class AGYBridgeHandler(BaseHTTPRequestHandler):
|
|||||||
timeout=120,
|
timeout=120,
|
||||||
env=os.environ.copy()
|
env=os.environ.copy()
|
||||||
)
|
)
|
||||||
|
finally:
|
||||||
|
for img_f in temp_image_files:
|
||||||
|
if os.path.exists(img_f):
|
||||||
|
try:
|
||||||
|
os.remove(img_f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
logger.error(f"agy error (code {proc.returncode}): {proc.stderr}")
|
logger.error(f"agy error (code {proc.returncode}): {proc.stderr}")
|
||||||
|
|||||||
Reference in New Issue
Block a user