29422f0572
After the shared _accepts_tool_result_images gate the capture route still demanded _lookup_supports_vision(...) is True, so a whitelisted provider with a catalog-unknown model (proxy alias) went native in vision_analyze and aux in computer_use. The route is now the negation of the shared gate; the redundant catalog lookup helpers are gone.
77 lines
4.9 KiB
Python
77 lines
4.9 KiB
Python
"""Vision-routing decisions for ``computer_use`` capture results. ``capture`` (mode som|vision) returns a
|
|
``_multimodal`` screenshot envelope as the tool result. A text-only main model, or a provider that rejects multimodal
|
|
tool results, turns that into a hard 400/404 — even with a working ``auxiliary.vision`` model in config. This module
|
|
decides: multimodal envelope, or pre-analyse via aux vision so the main model only ever sees text?
|
|
|
|
Decision order (mirrors ``vision_analyze``):
|
|
1. ``auxiliary.vision`` explicitly configured (provider not ""/"auto", or model / base_url set) → aux routing; users
|
|
who pay for a vision model want it used.
|
|
2. User-declared ``supports_vision`` for the active route (escape hatch for custom/local VLMs absent from models.dev)
|
|
→ honour it (True → multimodal).
|
|
3. The shared ``vision_analyze`` gate (profile veto, then provider tool-result media OR catalog vision) says yes →
|
|
multimodal — the same predicate, so the lane never depends on which tool asked.
|
|
4. Everything else (non-vision model, provider rejecting multimodal tool results, lookup failure) → aux routing.
|
|
|
|
Fails *closed* toward aux routing when metadata is missing or ambiguous: a screenshot sent to a model that cannot read
|
|
it is a hard failure, while aux routing costs one extra LLM call and yields a usable description.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from typing import Any, Dict, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
|
|
"""True when ``auxiliary.vision`` carries a non-default user override; mirrors ``agent.image_routing`` so the capture
|
|
and user-attached-image paths agree. ``provider: "auto"``, blanks or a missing block are *not* explicit."""
|
|
aux = cfg.get("auxiliary") if isinstance(cfg, dict) else None
|
|
vision = aux.get("vision") if isinstance(aux, dict) else None
|
|
if not isinstance(vision, dict):
|
|
return False
|
|
provider = str(vision.get("provider") or "").strip().lower()
|
|
return provider not in ("", "auto") or any(str(vision.get(k) or "").strip() for k in ("model", "base_url"))
|
|
|
|
def _lookup_user_declared_supports_vision(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> Optional[bool]:
|
|
"""Config-declared ``supports_vision`` for the active route (None on failure)."""
|
|
try:
|
|
from agent.image_routing import _supports_vision_override
|
|
return _supports_vision_override(cfg, provider, model)
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
logger.debug("computer_use vision_routing: config override lookup failed: %s", exc)
|
|
return None
|
|
|
|
def _provider_accepts_multimodal_tool_result(provider: str, model: str, cfg: Optional[Dict[str, Any]] = None) -> Optional[bool]:
|
|
"""Whether *provider*+*model* may carry images inside tool-result messages — the SAME predicate the
|
|
``vision_analyze`` fast path uses (#115248: the two gates disagreed for deepseek/deepseek-flash, so the route
|
|
depended on which tool asked). None on import failure so callers fall back to aux, not guess."""
|
|
if not provider:
|
|
return None
|
|
try:
|
|
from tools.vision_tools import _accepts_tool_result_images
|
|
except Exception as exc: # pragma: no cover - defensive
|
|
logger.debug("computer_use vision_routing: tool-result support lookup failed: %s", exc)
|
|
return None
|
|
return bool(_accepts_tool_result_images(provider, model, cfg))
|
|
|
|
def should_route_capture_to_aux_vision(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> bool:
|
|
"""True iff the screenshot should be pre-analysed via aux vision; False keeps the multimodal envelope. *provider* is
|
|
the lower-case canonical id, *model* the slug sent to the provider, *cfg* the loaded ``config.yaml`` dict (or None).
|
|
Steps follow the module docstring's decision order."""
|
|
# auto: an explicitly configured auxiliary.vision backend is the DE-FACTO choice — the user named a
|
|
# dedicated vision model, so that's what they want images to go through, even when the main model has
|
|
# native vision (maintainer decision, 2026-08-28, reversing #29135's fallback-only posture: config that
|
|
# only takes effect when the main model gets worse is a trap, not a setting). Native vision remains the
|
|
# default for unconfigured installs, and the fallback when the aux backend is unset.
|
|
if _explicit_aux_vision_override(cfg):
|
|
return True
|
|
user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
|
|
if isinstance(user_declared, bool): # True → multimodal, False → aux
|
|
return not user_declared
|
|
# The shared gate already folds the capability lookup in; demanding a second `is True` here made
|
|
# the two lanes disagree for whitelisted providers whose model the catalog does not know.
|
|
return not _provider_accepts_multimodal_tool_result(provider, model, cfg)
|
|
|
|
__all__ = ["should_route_capture_to_aux_vision"]
|