From 08feaa98b5b85dac9b348b8ca6f8b6d64679965b Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:37:43 -0700 Subject: [PATCH] fix(image-routing): drop the provider-wide supports_vision probe; mirror registry via public types in setup test ProviderProfile.supports_vision is documented as "the API accepts image content inside tool-result messages" -- a provider-wide wire capability, not a per-model user-image verdict. Treating it as the latter flipped every model on the bundled `router` relay (and models.dev-unknown meta-ai/xiaomi models) from text to native image parts. Per-model vision now comes from ProviderProfile.model_capabilities (#116570) through the existing models.dev probe. The setup-catalog test mirrored the plugin through auth._register_plugin_provider, which #116553 renames; build the ProviderConfig from public types instead so the test passes on both trees. --- agent/image_routing.py | 13 ----------- tests/agent/test_image_routing.py | 22 ------------------- .../hermes_cli/test_setup_provider_catalog.py | 8 ++++++- .../developer-guide/model-provider-plugin.md | 2 +- 4 files changed, 8 insertions(+), 37 deletions(-) diff --git a/agent/image_routing.py b/agent/image_routing.py index d133f6d4b1..31c7516500 100644 --- a/agent/image_routing.py +++ b/agent/image_routing.py @@ -310,25 +310,12 @@ def _probe_ollama(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> O return query_ollama_supports_vision(model, base_url, api_key=api_key) -def _probe_provider_profile(provider: str, model: str, cfg: Optional[Dict[str, Any]]) -> Optional[bool]: - """Registered ``ProviderProfile.supports_vision`` — the same declaration the tool-result media - path (``tools.vision_tools._supports_media_in_tool_results``) already trusts, so a plugin is not - vision-capable on one surface and text-only on another. Provider-wide and opt-in: only an - explicit ``True`` is a verdict (the field defaults to False, which means "not declared"). - Runs after the per-model catalogs so a catalog that knows the exact model still wins.""" - from providers import get_provider_profile - - profile = get_provider_profile(provider) - return True if profile is not None and profile.supports_vision is True else None - - # Capability probes after the config override, in priority order; each returns # True/False or None (unknown → next probe). Exceptions are logged and treated as None. _VISION_PROBES: Tuple[Tuple[str, Callable[..., Optional[bool]]], ...] = ( ("managed-runtime caps lookup", _probe_managed_runtime), ("caps lookup", _probe_models_dev), ("ollama vision probe", _probe_ollama), - ("provider profile declaration", _probe_provider_profile), ) diff --git a/tests/agent/test_image_routing.py b/tests/agent/test_image_routing.py index 98de113be3..69eecf0761 100644 --- a/tests/agent/test_image_routing.py +++ b/tests/agent/test_image_routing.py @@ -693,25 +693,3 @@ class TestCodexContextVariantVisionLookup: # Ineligible alias: looked up verbatim, no capability gained. assert image_routing._probe_models_dev("openai-codex", "gpt-5.5-900k", {}) is None assert seen[-1] == "gpt-5.5-900k" - - -# ─── registered ProviderProfile.supports_vision ────────────────────────────── - - -class TestProviderProfileDeclaration: - - def test_profile_declared_vision_routes_native_only_when_declared(self, monkeypatch): - """The routing probe honours the same ``supports_vision`` the tool-result media path reads: - a plugin that declares it goes native; an undeclared plugin stays text (#116408).""" - import providers - from providers.base import ProviderProfile - - for name, declared in (("scout-vision", True), ("scout-novision", False)): - monkeypatch.setitem( - providers._REGISTRY, name, - ProviderProfile(name=name, auth_type="api_key", base_url="https://x.invalid/v1", - supports_vision=declared)) - with patch("agent.image_routing._probe_models_dev", return_value=None), \ - patch("agent.image_routing._probe_managed_runtime", return_value=None): - assert decide_image_input_mode("scout-vision", "any-model", {}) == "native" - assert decide_image_input_mode("scout-novision", "any-model", {}) == "text" diff --git a/tests/hermes_cli/test_setup_provider_catalog.py b/tests/hermes_cli/test_setup_provider_catalog.py index db87d617ed..491bc61a6b 100644 --- a/tests/hermes_cli/test_setup_provider_catalog.py +++ b/tests/hermes_cli/test_setup_provider_catalog.py @@ -24,7 +24,13 @@ def test_setup_offers_registered_provider_catalog(monkeypatch, live): ) monkeypatch.setitem(providers._REGISTRY, profile.name, profile) monkeypatch.setattr(auth, "PROVIDER_REGISTRY", dict(auth.PROVIDER_REGISTRY)) - auth._register_plugin_provider(profile) + # Mirror into the auth registry the way plugin discovery does; built from public types so the + # helper is independent of the auth module's private mirroring function. + pconfig = auth.ProviderConfig( + profile.name, profile.display_name or profile.name, profile.auth_type, inference_base_url=profile.base_url) + if profile.auth_type == "api_key" and profile.env_vars: + pconfig = auth._api_key_provider(profile.name, profile.display_name or profile.name, profile.base_url, tuple(profile.env_vars), "") + auth.PROVIDER_REGISTRY[profile.name] = pconfig monkeypatch.setattr(flows, "_ensure_flow_api_key", lambda *_: (None, "synthetic-test-key", False)) monkeypatch.setattr(flows, "_env_base_url", lambda *_: "") monkeypatch.setattr(flows, "_prompt_base_url_override", lambda value, *_args, **_kwargs: value) diff --git a/website/docs/developer-guide/model-provider-plugin.md b/website/docs/developer-guide/model-provider-plugin.md index 0d47fa4c6c..a01c4f8d19 100644 --- a/website/docs/developer-guide/model-provider-plugin.md +++ b/website/docs/developer-guide/model-provider-plugin.md @@ -107,7 +107,7 @@ Full definition in `providers/base.py`. The most useful ones: | `default_headers` | `dict[str, str]` | Sent on every request (e.g. Copilot's `Editor-Version`) | ======= | `fallback_models` | `tuple[str, ...]` | Curated list shown when live catalog fetch fails — in the `/model` picker AND the first-time `hermes setup` / `hermes model` API-key flow, which resolve the catalog the same way (`fetch_models()` merged curated-first with `fallback_models`; `fallback_models` alone when the fetch returns `None` or raises) | -| `supports_vision` | bool | Declares the endpoint accepts image input. Consumed by the tool-result media path and by the per-turn image-routing probe (`agent/image_routing.decide_image_input_mode`): an image attached to a model of a `supports_vision=True` plugin goes native instead of through `vision_analyze` text, unless a config override or a models.dev per-model entry says otherwise | +| `supports_vision` | bool | Declares the provider's API accepts image content inside **tool-result** messages (a provider-wide wire capability). Per-model user-image routing comes from `model_capabilities` / models.dev, not from this flag | | `default_headers` | `dict[str, str]` | Sent on every request (e.g. Copilot's `Editor-Version`); also forwarded by the default `fetch_models()` catalog request | >>>>>>> cac616e8159 (fix(vision): image routing honours ProviderProfile.supports_vision) | `fixed_temperature` | Any | `None` = use caller's value; `OMIT_TEMPERATURE` sentinel = don't send temperature at all (Kimi) |