466 lines
19 KiB
Python
466 lines
19 KiB
Python
"""Image-part handling for ``AIAgent`` API messages.
|
|
|
|
Vision capability probes, non-vision text fallbacks (cached ``vision_analyze`` descriptions), tool-result
|
|
image stripping, and provider quirks (Anthropic dot preservation, Qwen portal message shaping).
|
|
Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged.
|
|
"""
|
|
import logging
|
|
import asyncio
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any, List, Optional
|
|
|
|
from agent.lazy_forward import forward_static as _forward_static
|
|
from agent.tool_dispatch_helpers import _is_multimodal_tool_result, _multimodal_text_summary
|
|
from utils import base_url_host_matches, base_url_hostname
|
|
|
|
# Same logger name as the origin module so log records / caplog filters are unchanged.
|
|
logger = logging.getLogger("run_agent")
|
|
|
|
|
|
class VisionMessagePrepMixin:
|
|
"""Vision probes + image-part fallbacks for outgoing messages (see module docstring)."""
|
|
|
|
@staticmethod
|
|
def _content_has_image_parts(content: Any) -> bool:
|
|
if not isinstance(content, list):
|
|
return False
|
|
for part in content:
|
|
if isinstance(part, dict) and part.get("type") in {"image_url", "input_image"}:
|
|
return True
|
|
return False
|
|
|
|
# 20 MB base64 ≈ 15 MB decoded — prevents OOM from an oversized data: URL in a shared gateway process.
|
|
_MAX_DATA_URL_BASE64_BYTES = 20 * 1024 * 1024
|
|
|
|
@staticmethod
|
|
def _materialize_data_url_for_vision(image_url: str) -> tuple[str, Optional[Path]]:
|
|
header, _, data = str(image_url or "").partition(",")
|
|
if len(data) > VisionMessagePrepMixin._MAX_DATA_URL_BASE64_BYTES:
|
|
logger.warning(
|
|
"data-URL payload too large (%d bytes), skipping", len(data)
|
|
)
|
|
return "", None
|
|
mime = "image/jpeg"
|
|
if header.startswith("data:"):
|
|
mime_part = header[len("data:"):].split(";", 1)[0].strip()
|
|
if mime_part.startswith("image/"):
|
|
mime = mime_part
|
|
suffix = {
|
|
"image/png": ".png",
|
|
"image/gif": ".gif",
|
|
"image/webp": ".webp",
|
|
"image/jpeg": ".jpg",
|
|
"image/jpg": ".jpg",
|
|
}.get(mime, ".jpg")
|
|
tmp = tempfile.NamedTemporaryFile(prefix="anthropic_image_", suffix=suffix, delete=False)
|
|
try:
|
|
with tmp:
|
|
tmp.write(base64.b64decode(data))
|
|
except Exception:
|
|
# delete=False means a corrupt/unsupported data URL would otherwise
|
|
# leak a zero-byte temp file on every failed materialization.
|
|
try:
|
|
os.unlink(tmp.name)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
path = Path(tmp.name)
|
|
return str(path), path
|
|
|
|
def _describe_image_for_anthropic_fallback(self, image_url: str, role: str) -> str:
|
|
cache_key = hashlib.sha256(str(image_url or "").encode("utf-8")).hexdigest()
|
|
cached = self._anthropic_image_fallback_cache.get(cache_key)
|
|
if cached:
|
|
return cached
|
|
|
|
role_label = {
|
|
"assistant": "assistant",
|
|
"tool": "tool result",
|
|
}.get(role, "user")
|
|
analysis_prompt = (
|
|
"Describe everything visible in this image in thorough detail. "
|
|
"Include any text, code, UI, data, objects, people, layout, colors, "
|
|
"and any other notable visual information."
|
|
)
|
|
|
|
vision_source = str(image_url or "")
|
|
cleanup_path: Optional[Path] = None
|
|
if vision_source.startswith("data:"):
|
|
vision_source, cleanup_path = self._materialize_data_url_for_vision(vision_source)
|
|
|
|
description = ""
|
|
try:
|
|
from tools.vision_tools import vision_analyze_tool
|
|
|
|
result_json = asyncio.run(
|
|
vision_analyze_tool(image_url=vision_source, user_prompt=analysis_prompt)
|
|
)
|
|
result = json.loads(result_json) if isinstance(result_json, str) else {}
|
|
description = (result.get("analysis") or "").strip()
|
|
except Exception as e:
|
|
description = f"Image analysis failed: {e}"
|
|
finally:
|
|
if cleanup_path and cleanup_path.exists():
|
|
try:
|
|
cleanup_path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
if not description:
|
|
description = "Image analysis failed."
|
|
|
|
note = f"[The {role_label} attached an image. Here's what it contains:\n{description}]"
|
|
if vision_source and not str(image_url or "").startswith("data:"):
|
|
note += (
|
|
f"\n[If you need a closer look, use vision_analyze with image_url: {vision_source}]"
|
|
)
|
|
|
|
self._anthropic_image_fallback_cache[cache_key] = note
|
|
return note
|
|
|
|
def _model_supports_vision(self) -> bool:
|
|
"""Return True if the active provider+model reports native vision.
|
|
|
|
Resolution: ``model.supports_vision`` > ``providers.<p>.models.<m>.supports_vision`` > models.dev
|
|
lookup (see ``image_routing._supports_vision_override``). Custom/local models absent from models.dev
|
|
would otherwise be misclassified and have their images stripped.
|
|
"""
|
|
try:
|
|
from hermes_cli.config import load_config
|
|
from agent.image_routing import _lookup_supports_vision
|
|
cfg = load_config()
|
|
provider = (getattr(self, "provider", "") or "").strip()
|
|
model = (getattr(self, "model", "") or "").strip()
|
|
return _lookup_supports_vision(provider, model, cfg) is True
|
|
except Exception:
|
|
return False
|
|
|
|
def _provider_supports_vision_tool_messages(self) -> bool:
|
|
"""Return True if the active provider accepts list-type tool content.
|
|
|
|
Some providers (Xiaomi MiMo) accept multimodal user messages but 400 on list-type tool content;
|
|
reads the provider profile's ``supports_vision_tool_messages``.
|
|
"""
|
|
try:
|
|
from providers import get_provider_profile
|
|
provider = (getattr(self, "provider", "") or "").strip()
|
|
profile = get_provider_profile(provider)
|
|
if profile is not None:
|
|
return getattr(profile, "supports_vision_tool_messages", True)
|
|
except Exception:
|
|
pass
|
|
return True # default: assume compatible
|
|
|
|
def _preprocess_anthropic_content(self, content: Any, role: str) -> Any:
|
|
if not self._content_has_image_parts(content):
|
|
return content
|
|
|
|
text_parts: List[str] = []
|
|
image_notes: List[str] = []
|
|
for part in content:
|
|
if isinstance(part, str):
|
|
if part.strip():
|
|
text_parts.append(part.strip())
|
|
continue
|
|
if not isinstance(part, dict):
|
|
continue
|
|
|
|
ptype = part.get("type")
|
|
if ptype in {"text", "input_text"}:
|
|
text = str(part.get("text", "") or "").strip()
|
|
if text:
|
|
text_parts.append(text)
|
|
continue
|
|
|
|
if ptype in {"image_url", "input_image"}:
|
|
image_data = part.get("image_url", {})
|
|
image_url = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "")
|
|
if image_url:
|
|
image_notes.append(self._describe_image_for_anthropic_fallback(image_url, role))
|
|
else:
|
|
image_notes.append("[An image was attached but no image source was available.]")
|
|
continue
|
|
|
|
text = str(part.get("text", "") or "").strip()
|
|
if text:
|
|
text_parts.append(text)
|
|
|
|
prefix = "\n\n".join(note for note in image_notes if note).strip()
|
|
suffix = "\n".join(text for text in text_parts if text).strip()
|
|
if prefix and suffix:
|
|
return f"{prefix}\n\n{suffix}"
|
|
if prefix:
|
|
return prefix
|
|
if suffix:
|
|
return suffix
|
|
return "[A multimodal message was converted to text for Anthropic compatibility.]"
|
|
|
|
def _get_transport(self, api_mode: str = None):
|
|
"""Return the cached transport for the given (or current) api_mode (lazy; None if unregistered)."""
|
|
mode = api_mode or self.api_mode
|
|
cache = getattr(self, "_transport_cache", None)
|
|
if cache is None:
|
|
cache = {}
|
|
self._transport_cache = cache
|
|
t = cache.get(mode)
|
|
if t is None:
|
|
from agent.transports import get_transport
|
|
t = get_transport(mode)
|
|
cache[mode] = t
|
|
return t
|
|
|
|
def _prepare_messages_for_non_vision_model(self, api_messages: list) -> list:
|
|
"""Replace native image parts with cached vision_analyze text when the active model lacks vision.
|
|
|
|
Vision-capable models pass through unchanged (the provider adapter — including the Anthropic one —
|
|
handles image parts natively). The text fallback is the historically Anthropic-named preprocessor.
|
|
"""
|
|
if not any(
|
|
isinstance(msg, dict) and self._content_has_image_parts(msg.get("content"))
|
|
for msg in api_messages
|
|
):
|
|
return api_messages
|
|
|
|
if self._model_supports_vision():
|
|
return api_messages
|
|
|
|
transformed = copy.deepcopy(api_messages)
|
|
for msg in transformed:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
msg["content"] = self._preprocess_anthropic_content(
|
|
msg.get("content"),
|
|
str(msg.get("role", "user") or "user"),
|
|
)
|
|
return transformed
|
|
|
|
# Same transform for the Anthropic route (callers/tests patch this name independently).
|
|
_prepare_anthropic_messages_for_api = _prepare_messages_for_non_vision_model
|
|
|
|
def _tool_result_content_for_active_model(self, tool_name: str, result: Any) -> Any:
|
|
"""Return the tool message content that is safe for the active model.
|
|
|
|
Text-only providers must not receive image parts: a rejected tool result becomes canonical history
|
|
and can make the next user turn fail before the agent can recover.
|
|
"""
|
|
if not _is_multimodal_tool_result(result):
|
|
return result
|
|
|
|
content = result.get("content") or []
|
|
if not self._content_has_image_parts(content):
|
|
return content
|
|
|
|
if self._model_supports_vision():
|
|
# Vision on paper, but the provider rejects list-type tool content (or we already learned that
|
|
# in-session): short-circuit to a text summary.
|
|
if not self._provider_supports_vision_tool_messages():
|
|
logger.debug(
|
|
"Tool %s: provider %s does not accept list-type tool "
|
|
"content — sending text summary",
|
|
tool_name, getattr(self, "provider", ""),
|
|
)
|
|
return _multimodal_text_summary(result)
|
|
key = (
|
|
(getattr(self, "provider", "") or "").strip().lower(),
|
|
(getattr(self, "model", "") or "").strip(),
|
|
)
|
|
no_list = getattr(self, "_no_list_tool_content_models", None)
|
|
if no_list and key in no_list:
|
|
logger.debug(
|
|
"Tool %s: model %s/%s known to reject list-type tool "
|
|
"content this session — sending text summary",
|
|
tool_name, key[0], key[1],
|
|
)
|
|
return _multimodal_text_summary(result)
|
|
return content
|
|
|
|
summary = _multimodal_text_summary(result)
|
|
if tool_name == "computer_use":
|
|
return json.dumps({
|
|
"error": (
|
|
"computer_use returned screenshot/image content, but the active "
|
|
"model/provider does not support image input. Switch to a "
|
|
"vision-capable model for desktop computer use, or use browser "
|
|
"tools for browser tasks."
|
|
),
|
|
"text_summary": summary,
|
|
})
|
|
|
|
logger.warning(
|
|
"Tool %s returned image content for non-vision model %s/%s; "
|
|
"falling back to text summary",
|
|
tool_name,
|
|
self.provider,
|
|
self.model,
|
|
)
|
|
return summary
|
|
|
|
_try_shrink_image_parts_in_messages = _forward_static("agent.conversation_compression", "try_shrink_image_parts_in_messages")
|
|
|
|
def _try_strip_image_parts_from_tool_messages(
|
|
self,
|
|
api_messages: list,
|
|
*,
|
|
remember_model: bool = True,
|
|
) -> bool:
|
|
"""Downgrade list-type tool messages to text summaries in place; returns True if any were downgraded.
|
|
|
|
Recovery for providers that 400 on list-type tool content (e.g. MiMo "text is not set"). By default
|
|
records the (provider, model) in ``_no_list_tool_content_models`` so later results downgrade without a
|
|
round-trip; 413 recovery passes ``remember_model=False`` (body too large ≠ provider rejects lists).
|
|
"""
|
|
if not isinstance(api_messages, list):
|
|
return False
|
|
|
|
if remember_model:
|
|
# Record (provider, model) so we don't relearn this lesson.
|
|
key = (
|
|
(getattr(self, "provider", "") or "").strip().lower(),
|
|
(getattr(self, "model", "") or "").strip(),
|
|
)
|
|
if not hasattr(self, "_no_list_tool_content_models"):
|
|
self._no_list_tool_content_models = set()
|
|
if key[1]: # only record when we actually have a model id
|
|
self._no_list_tool_content_models.add(key)
|
|
|
|
changed = False
|
|
for msg in api_messages:
|
|
if not isinstance(msg, dict) or msg.get("role") != "tool":
|
|
continue
|
|
content = msg.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
|
|
# Salvage any text parts so the model still sees some signal.
|
|
text_parts: List[str] = []
|
|
had_image = False
|
|
for part in content:
|
|
if not isinstance(part, dict):
|
|
if isinstance(part, str) and part.strip():
|
|
text_parts.append(part.strip())
|
|
continue
|
|
ptype = part.get("type")
|
|
if ptype == "image_url" or ptype == "input_image":
|
|
had_image = True
|
|
continue
|
|
if ptype in {"text", "input_text"}:
|
|
text = str(part.get("text") or "").strip()
|
|
if text:
|
|
text_parts.append(text)
|
|
|
|
if not had_image:
|
|
# List content without image parts — leave alone; stripping wouldn't reduce ambiguity.
|
|
continue
|
|
|
|
if text_parts:
|
|
msg["content"] = "\n\n".join(text_parts)
|
|
else:
|
|
msg["content"] = (
|
|
"[image content removed — provider does not accept "
|
|
"list-type tool message content]"
|
|
)
|
|
changed = True
|
|
|
|
return changed
|
|
|
|
def _anthropic_preserve_dots(self) -> bool:
|
|
"""True when using an anthropic-compatible endpoint that preserves dots in model names.
|
|
|
|
DashScope, MiniMax, Xiaomi MiMo, OpenCode Go/Zen (non-Claude), ZAI/Zhipu keep dots; AWS Bedrock uses
|
|
dotted inference-profile IDs and rejects the hyphenated form with HTTP 400.
|
|
"""
|
|
if (getattr(self, "provider", "") or "").lower() in {
|
|
"alibaba", "minimax", "minimax-cn",
|
|
"opencode-go", "opencode-zen",
|
|
"zai", "bedrock",
|
|
"xiaomi", "vertex",
|
|
}:
|
|
return True
|
|
base = (getattr(self, "base_url", "") or "").lower()
|
|
host = base_url_hostname(base)
|
|
return (
|
|
"dashscope" in host
|
|
or base_url_host_matches(base, "aliyuncs.com")
|
|
or "minimax" in host
|
|
or (base_url_host_matches(base, "opencode.ai") and "/zen/" in base)
|
|
or base_url_host_matches(base, "bigmodel.cn")
|
|
or base_url_host_matches(base, "xiaomimimo.com")
|
|
# Vertex AI OpenAI-compat endpoint — Gemini model ids keep dots
|
|
# (e.g. google/gemini-3.5-flash); the hyphenated form is wrong.
|
|
or base_url_host_matches(base, "aiplatform.googleapis.com")
|
|
# AWS Bedrock runtime endpoints — defense-in-depth when
|
|
# ``provider`` is unset but ``base_url`` still names Bedrock.
|
|
or host.startswith("bedrock-runtime.")
|
|
)
|
|
|
|
def _is_qwen_portal(self) -> bool:
|
|
"""Return True when the base URL targets Qwen Portal."""
|
|
return base_url_host_matches(self._base_url_lower, "portal.qwen.ai")
|
|
|
|
def _qwen_prepare_chat_messages(self, api_messages: list) -> list:
|
|
prepared = copy.deepcopy(api_messages)
|
|
if not prepared:
|
|
return prepared
|
|
|
|
for msg in prepared:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
content = msg.get("content")
|
|
if isinstance(content, str):
|
|
msg["content"] = [{"type": "text", "text": content}]
|
|
elif isinstance(content, list):
|
|
# Normalize: convert bare strings to text dicts, keep dicts as-is.
|
|
# deepcopy already created independent copies, no need for dict().
|
|
normalized_parts = []
|
|
for part in content:
|
|
if isinstance(part, str):
|
|
normalized_parts.append({"type": "text", "text": part})
|
|
elif isinstance(part, dict):
|
|
normalized_parts.append(part)
|
|
if normalized_parts:
|
|
msg["content"] = normalized_parts
|
|
|
|
# Inject cache_control on the last part of the system message.
|
|
for msg in prepared:
|
|
if isinstance(msg, dict) and msg.get("role") == "system":
|
|
content = msg.get("content")
|
|
if isinstance(content, list) and content and isinstance(content[-1], dict):
|
|
content[-1]["cache_control"] = {"type": "ephemeral"}
|
|
break
|
|
|
|
return prepared
|
|
|
|
def _qwen_prepare_chat_messages_inplace(self, messages: list) -> None:
|
|
"""In-place variant — mutates an already-copied message list."""
|
|
if not messages:
|
|
return
|
|
|
|
for msg in messages:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
content = msg.get("content")
|
|
if isinstance(content, str):
|
|
msg["content"] = [{"type": "text", "text": content}]
|
|
elif isinstance(content, list):
|
|
normalized_parts = []
|
|
for part in content:
|
|
if isinstance(part, str):
|
|
normalized_parts.append({"type": "text", "text": part})
|
|
elif isinstance(part, dict):
|
|
normalized_parts.append(part)
|
|
if normalized_parts:
|
|
msg["content"] = normalized_parts
|
|
|
|
for msg in messages:
|
|
if isinstance(msg, dict) and msg.get("role") == "system":
|
|
content = msg.get("content")
|
|
if isinstance(content, list) and content and isinstance(content[-1], dict):
|
|
content[-1]["cache_control"] = {"type": "ephemeral"}
|
|
break
|