diff --git a/agent/image_routing.py b/agent/image_routing.py index 8cb8d597a7..d133f6d4b1 100644 --- a/agent/image_routing.py +++ b/agent/image_routing.py @@ -310,12 +310,25 @@ def _probe_ollama(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> O return query_ollama_supports_vision(model, base_url, api_key=api_key) +def _probe_provider_profile(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> Optional[bool]: + """Registered ``ProviderProfile.supports_vision`` — the same declaration the tool-result media + path (``tools.vision_tools._supports_media_in_tool_results``) already trusts, so a plugin is not + vision-capable on one surface and text-only on another. Provider-wide and opt-in: only an + explicit ``True`` is a verdict (the field defaults to False, which means "not declared"). + Runs after the per-model catalogs so a catalog that knows the exact model still wins.""" + from providers import get_provider_profile + + profile = get_provider_profile(provider) + return True if profile is not None and profile.supports_vision is True else None + + # Capability probes after the config override, in priority order; each returns # True/False or None (unknown → next probe). Exceptions are logged and treated as None. _VISION_PROBES: Tuple[Tuple[str, Callable[..., Optional[bool]]], ...] = ( ("managed-runtime caps lookup", _probe_managed_runtime), ("caps lookup", _probe_models_dev), ("ollama vision probe", _probe_ollama), + ("provider profile declaration", _probe_provider_profile), ) @@ -329,7 +342,8 @@ def _lookup_supports_vision( """Return True/False if vision capability can be resolved, None if unknown. Order: config ``supports_vision`` override → :data:`_VISION_PROBES` - (managed local runtime → models.dev catalog → Ollama probe). + (managed local runtime → models.dev catalog → Ollama probe → registered + ``ProviderProfile.supports_vision`` declaration). """ # Named custom providers are canonicalized to ``provider="custom"``; the # original name lives in the context-local main runtime. Borrow it only on an diff --git a/website/docs/developer-guide/model-provider-plugin.md b/website/docs/developer-guide/model-provider-plugin.md index d012739f20..0d47fa4c6c 100644 --- a/website/docs/developer-guide/model-provider-plugin.md +++ b/website/docs/developer-guide/model-provider-plugin.md @@ -100,10 +100,16 @@ Full definition in `providers/base.py`. The most useful ones: | `base_url` | str | Default inference endpoint | | `models_url` | str | Explicit catalog URL (falls back to `{base_url}/models`) | | `auth_type` | str | `api_key` \| `oauth_device_code` \| `oauth_external` \| `copilot` \| `aws_sdk` \| `external_process` | +<<<<<<< HEAD | `auth_handler` | `Callable \| None` | Provider-owned `hermes auth add/status/logout/refresh ` — see [Provider-owned auth](#provider-owned-auth-auth_handler-refresh_credential) | | `refresh_credential` | `Callable \| None` | Provider-owned rotation of a pooled OAuth row — same section | | `fallback_models` | `tuple[str, ...]` | Curated list shown when live catalog fetch fails | | `default_headers` | `dict[str, str]` | Sent on every request (e.g. Copilot's `Editor-Version`) | +======= +| `fallback_models` | `tuple[str, ...]` | Curated list shown when live catalog fetch fails — in the `/model` picker AND the first-time `hermes setup` / `hermes model` API-key flow, which resolve the catalog the same way (`fetch_models()` merged curated-first with `fallback_models`; `fallback_models` alone when the fetch returns `None` or raises) | +| `supports_vision` | bool | Declares the endpoint accepts image input. Consumed by the tool-result media path and by the per-turn image-routing probe (`agent/image_routing.decide_image_input_mode`): an image attached to a model of a `supports_vision=True` plugin goes native instead of through `vision_analyze` text, unless a config override or a models.dev per-model entry says otherwise | +| `default_headers` | `dict[str, str]` | Sent on every request (e.g. Copilot's `Editor-Version`); also forwarded by the default `fetch_models()` catalog request | +>>>>>>> cac616e8159 (fix(vision): image routing honours ProviderProfile.supports_vision) | `fixed_temperature` | Any | `None` = use caller's value; `OMIT_TEMPERATURE` sentinel = don't send temperature at all (Kimi) | | `default_max_tokens` | `int \| None` | Provider-level max_tokens cap (Nvidia: 16384) | | `unsupported_response_formats` | `tuple` | `response_format` types the API rejects outright; auxiliary requests omit them instead of paying a guaranteed 400 (DeepSeek: `("json_schema",)`) |