53183d5016
check_vision_requirements only asked the auxiliary resolver, so a vision-capable main model on a provider the resolver cannot serve (minimax-oauth, local vLLM, anything uncatalogued) lost vision_analyze and browser_vision from the tool list even though both handlers already route to the native fast path and work when called. The image gate now accepts the native fast path OR an aux client; the aux-only probe becomes check_video_requirements and stays on video_analyze, whose handler has no native path. Fixes #47149.
283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""Regression tests for issue #31179.
|
|
|
|
Before the fix:
|
|
- ``auxiliary.vision.provider: openai`` silently failed to resolve because
|
|
``openai`` is not a first-class provider in PROVIDER_REGISTRY (only
|
|
``openai-codex`` for OAuth and ``custom`` for OPENAI_BASE_URL).
|
|
- The vision branch of ``call_llm`` then silently fell back to ``auto``
|
|
which happily picked the user's main provider (e.g. DeepSeek), sending
|
|
image content to a text-only endpoint and producing cryptic
|
|
``unknown variant 'image_url', expected 'text'`` errors.
|
|
- ``check_vision_requirements`` used the explicit-only path, so
|
|
``vision_analyze`` disappeared from the tool list while ``browser_vision``
|
|
stayed (its check_fn only validated the browser).
|
|
|
|
The three fixes covered here:
|
|
1. ``provider: openai`` in auxiliary task config resolves to
|
|
``custom`` + ``https://api.openai.com/v1``.
|
|
2. The vision auto-detect chain skips the user's main provider when it
|
|
reports ``supports_vision=False`` instead of routing image content to
|
|
a text-only endpoint.
|
|
3. ``check_vision_requirements`` mirrors the runtime fallback chain so
|
|
``vision_analyze`` shows up whenever the auto chain can serve vision,
|
|
and ``browser_vision`` gates on vision availability as well.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
import pytest
|
|
from tools import browser_tool_install as bt_install
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Test infrastructure
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.fixture
|
|
def isolated_home(monkeypatch):
|
|
"""Temp HERMES_HOME with config + clean credential env vars."""
|
|
test_home = tempfile.mkdtemp(prefix="hermes_test_31179_")
|
|
hermes_home = os.path.join(test_home, ".hermes")
|
|
os.makedirs(hermes_home)
|
|
monkeypatch.setenv("HERMES_HOME", hermes_home)
|
|
|
|
# Strip all credential-shaped env vars so each scenario starts hermetic.
|
|
for k in list(os.environ.keys()):
|
|
if k.endswith("_API_KEY") or k.endswith("_TOKEN"):
|
|
monkeypatch.delenv(k, raising=False)
|
|
|
|
yield hermes_home
|
|
shutil.rmtree(test_home, ignore_errors=True)
|
|
|
|
|
|
def _write_config(home: str, text: str) -> None:
|
|
with open(os.path.join(home, "config.yaml"), "w") as fp:
|
|
fp.write(text)
|
|
|
|
|
|
_RELOAD_PREFIXES = ("agent.auxiliary_client", "agent.image_routing",
|
|
"tools.vision_tools", "tools.browser_tool",
|
|
"hermes_cli.config")
|
|
|
|
|
|
def _drop_reload_targets():
|
|
for mod in list(sys.modules.keys()):
|
|
if mod.startswith(_RELOAD_PREFIXES):
|
|
del sys.modules[mod]
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _module_isolation():
|
|
"""Save/restore sys.modules entries this file reloads.
|
|
|
|
Without this, reloaded copies of agent.image_routing & friends leak
|
|
into sys.modules after the test, splitting module identity for any
|
|
later test that patches ``agent.image_routing.*`` while holding
|
|
function refs from the original module (issue #61597).
|
|
"""
|
|
saved = {name: mod for name, mod in sys.modules.items()
|
|
if name.startswith(_RELOAD_PREFIXES)}
|
|
yield
|
|
_drop_reload_targets()
|
|
sys.modules.update(saved)
|
|
|
|
|
|
def _fresh_modules():
|
|
"""Drop cached hermes modules so each test reloads against current env."""
|
|
_drop_reload_targets()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 1: provider=openai → custom + api.openai.com/v1
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestOpenAiAliasForAuxiliary:
|
|
"""``auxiliary.<task>.provider: openai`` should produce a working client."""
|
|
|
|
def test_provider_openai_routes_to_openai_dot_com(self, isolated_home, monkeypatch):
|
|
_write_config(isolated_home, """
|
|
auxiliary:
|
|
vision:
|
|
provider: openai
|
|
model: gpt-4o-mini
|
|
""")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
|
_fresh_modules()
|
|
|
|
from agent.auxiliary_client import _resolve_task_provider_model
|
|
provider, model, base_url, _key, _mode = _resolve_task_provider_model("vision")
|
|
assert provider == "custom"
|
|
assert model == "gpt-4o-mini"
|
|
assert base_url == "https://api.openai.com/v1"
|
|
|
|
def test_provider_openai_with_explicit_base_url_preserves_user_endpoint(
|
|
self, isolated_home, monkeypatch
|
|
):
|
|
"""User-supplied base_url wins; alias still normalizes provider name
|
|
to ``custom`` so resolution doesn't hit the unknown-provider path."""
|
|
_write_config(isolated_home, """
|
|
auxiliary:
|
|
vision:
|
|
provider: openai
|
|
model: gpt-4o-mini
|
|
base_url: https://my-proxy.example.com/v1
|
|
""")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
|
_fresh_modules()
|
|
|
|
from agent.auxiliary_client import _resolve_task_provider_model
|
|
provider, _model, base_url, _key, _mode = _resolve_task_provider_model("vision")
|
|
assert provider == "custom"
|
|
assert base_url == "https://my-proxy.example.com/v1"
|
|
|
|
def test_provider_openai_resolves_to_working_client(self, isolated_home, monkeypatch):
|
|
"""End-to-end: the resolved client points at api.openai.com."""
|
|
_write_config(isolated_home, """
|
|
auxiliary:
|
|
vision:
|
|
provider: openai
|
|
model: gpt-4o-mini
|
|
""")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
|
_fresh_modules()
|
|
|
|
from agent.auxiliary_client import resolve_vision_provider_client
|
|
from urllib.parse import urlparse
|
|
provider, client, model = resolve_vision_provider_client()
|
|
assert client is not None, "openai alias should produce a usable client"
|
|
# Exact hostname comparison (not substring) — defends against URLs
|
|
# like ``api.openai.com.evil.example`` and keeps CodeQL happy.
|
|
host = urlparse(str(getattr(client, "base_url", ""))).hostname or ""
|
|
assert host == "api.openai.com", f"expected api.openai.com host, got {host!r}"
|
|
assert model == "gpt-4o-mini"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 2: auto chain skips text-only main providers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTextOnlyMainSkippedForVision:
|
|
"""Vision auto-detect must not return a text-only main-provider client."""
|
|
|
|
def test_text_only_main_skipped_when_no_aggregator(self, isolated_home, monkeypatch):
|
|
"""DeepSeek main + no aggregator credentials → no client built.
|
|
|
|
Pre-fix this silently returned the deepseek client with model
|
|
substitution, producing ``unknown variant 'image_url'`` at call time.
|
|
"""
|
|
_write_config(isolated_home, """
|
|
model:
|
|
provider: deepseek
|
|
default: deepseek-v4-pro
|
|
""")
|
|
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-test")
|
|
_fresh_modules()
|
|
|
|
from agent.auxiliary_client import resolve_vision_provider_client
|
|
provider, client, _model = resolve_vision_provider_client(provider="auto")
|
|
assert client is None, (
|
|
f"Vision auto-detect must skip text-only main {provider!r} when "
|
|
"no vision-capable aggregator is available, not return a client "
|
|
"that will fail at API time"
|
|
)
|
|
|
|
def test_vision_capable_main_used(self, isolated_home, monkeypatch):
|
|
"""Vision-capable main provider should be returned by auto chain."""
|
|
_write_config(isolated_home, """
|
|
model:
|
|
provider: anthropic
|
|
default: claude-sonnet-4-6
|
|
""")
|
|
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-test")
|
|
_fresh_modules()
|
|
|
|
from agent.auxiliary_client import resolve_vision_provider_client
|
|
provider, client, _model = resolve_vision_provider_client(provider="auto")
|
|
assert client is not None
|
|
assert provider == "anthropic"
|
|
|
|
def test_unknown_capability_does_not_block(self, isolated_home, monkeypatch):
|
|
"""When models.dev has no entry, fall back to permissive (attempt the call).
|
|
|
|
This keeps new/custom providers working — only providers we have
|
|
cataloged as text-only are skipped.
|
|
"""
|
|
_fresh_modules()
|
|
from agent.auxiliary_client import _main_model_supports_vision
|
|
# Bogus provider/model — capability lookup returns None → permissive.
|
|
assert _main_model_supports_vision("nonexistent-provider", "nonexistent-model") is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 3: check_vision_requirements + check_browser_vision_requirements parity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestVisionToolGating:
|
|
"""Tool visibility must match runtime capability."""
|
|
|
|
def test_check_vision_succeeds_for_aliased_openai(self, isolated_home, monkeypatch):
|
|
"""The user's exact reported scenario: provider=openai unhides
|
|
vision_analyze instead of silently dropping it."""
|
|
_write_config(isolated_home, """
|
|
auxiliary:
|
|
vision:
|
|
provider: openai
|
|
model: gpt-4o-mini
|
|
""")
|
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
|
|
_fresh_modules()
|
|
|
|
from tools.vision_tools import check_vision_requirements
|
|
assert check_vision_requirements() is True
|
|
|
|
|
|
|
|
|
|
|
|
def test_browser_vision_true_when_both_available(self, isolated_home, monkeypatch):
|
|
from unittest.mock import patch
|
|
|
|
_write_config(isolated_home, """
|
|
model:
|
|
provider: openrouter
|
|
default: anthropic/claude-sonnet-4
|
|
""")
|
|
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test")
|
|
_fresh_modules()
|
|
|
|
import tools.browser_tool
|
|
with patch.object(bt_install, "check_browser_requirements", return_value=True):
|
|
assert tools.browser_tool_install.check_browser_vision_requirements() is True
|
|
|
|
def test_native_vision_main_advertises_image_tools_but_not_video(self, isolated_home, monkeypatch):
|
|
"""A vision-capable main model on a provider the aux resolver cannot serve (OAuth, local vLLM)
|
|
still gets vision_analyze and browser_vision — both handlers attach pixels natively — while
|
|
video_analyze, whose handler has no native path, stays hidden (#47149)."""
|
|
from unittest.mock import patch
|
|
|
|
_write_config(isolated_home, """
|
|
model:
|
|
provider: minimax-oauth
|
|
default: MiniMax-M3
|
|
supports_vision: true
|
|
""")
|
|
_fresh_modules()
|
|
|
|
import tools.browser_tool_install as bt_install
|
|
from tools import vision_tools
|
|
|
|
with patch.object(vision_tools, "_should_use_native_vision_fast_path", return_value=True), \
|
|
patch.object(bt_install, "check_browser_requirements", return_value=True):
|
|
assert vision_tools.check_video_requirements() is False # no aux client resolves
|
|
assert vision_tools.check_vision_requirements() is True
|
|
assert bt_install.check_browser_vision_requirements() is True
|