62509bb87f
Greptile on 0ce63424a06: - Only the first four images of a result were prepared; a damaged or already-in-context image among them hid a valid fifth. Images now prepare in waves sized to the free slots (bounded at 16 candidates), so a skip hands its slot to the next image. - The attach check compared against the 20 MB hard cap, not `vision.embed_target_bytes`; a best-effort resize that bottomed out above a low budget rode history anyway. It now checks the configured target (clamped to the hard cap). - `tools/computer_use/vision_routing.py` had become a one-line negation of `_native_tool_result_images`. The capture boundary calls the shared gate directly; the module and its tautological agreement test are gone, and the route-decision cases move into the capture-routing tests where they drive `_should_route_through_aux_vision` itself.
289 lines
12 KiB
Python
289 lines
12 KiB
Python
"""End-to-end regression for #24015 — capture routing via auxiliary.vision.
|
||
|
||
When ``computer_use(action='capture', mode='som'|'vision')`` returns a
|
||
screenshot, ``_capture_response`` previously always returned a
|
||
``_multimodal`` envelope. For non-vision main models, or when the user
|
||
explicitly configured ``auxiliary.vision`` in ``config.yaml``, that
|
||
envelope tripped HTTP 404 / 400 at the provider boundary even though a
|
||
perfectly good vision backend was sitting in config waiting to be used.
|
||
|
||
This file exercises the integrated ``_capture_response`` flow with
|
||
deterministic stubs for:
|
||
|
||
* ``_native_tool_result_images`` (the shared policy decision)
|
||
* ``_run_async`` (sync->async bridge)
|
||
* ``vision_analyze_tool`` (the aux LLM call)
|
||
* ``hermes_constants.get_hermes_dir`` (cache path)
|
||
|
||
…so the full code path is covered without a live cua-driver, a real
|
||
auxiliary client, or network access.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import base64
|
||
import json
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Fixtures / helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# 8×8 PNG (transparent) — minimal provider-acceptable bytes that decode cleanly.
|
||
_PNG_B64 = (
|
||
"iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAYAAADED76LAAAADUlEQVR4nG"
|
||
"NgGAUgAAABCAABgukLHQAAAABJRU5ErkJggg=="
|
||
)
|
||
|
||
# 1×1 JPEG — used to verify mime detection works for either stream type.
|
||
_JPEG_B64 = (
|
||
"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB"
|
||
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/"
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def tmp_cache_dir(tmp_path):
|
||
"""Override get_hermes_dir so cache writes land under tmp_path."""
|
||
cache_dir = tmp_path / "cache_vision"
|
||
cache_dir.mkdir()
|
||
|
||
def _fake_get(*_args, **_kw):
|
||
return cache_dir
|
||
|
||
with patch("hermes_constants.get_hermes_dir", _fake_get):
|
||
yield cache_dir
|
||
|
||
|
||
def _make_capture(
|
||
*,
|
||
png_b64: str = _PNG_B64,
|
||
mode: str = "som",
|
||
elements=None,
|
||
app: str = "Safari",
|
||
window_title: str = "GitHub – Issue #24015",
|
||
width: int = 1280,
|
||
height: int = 800,
|
||
):
|
||
from tools.computer_use.backend import CaptureResult, UIElement
|
||
|
||
elements = list(elements or [
|
||
UIElement(index=0, role="AXButton", label="Sign in",
|
||
bounds=(10, 20, 80, 30)),
|
||
UIElement(index=1, role="AXTextField", label="username",
|
||
bounds=(10, 60, 200, 24)),
|
||
])
|
||
raw = base64.b64decode(png_b64, validate=False)
|
||
return CaptureResult(
|
||
mode=mode,
|
||
width=width,
|
||
height=height,
|
||
png_b64=png_b64,
|
||
elements=elements,
|
||
app=app,
|
||
window_title=window_title,
|
||
png_bytes_len=len(raw),
|
||
)
|
||
|
||
|
||
def _stub_aux_analysis(text: str):
|
||
"""Return a fake vision_analyze_tool coroutine result (JSON envelope)."""
|
||
return json.dumps({"success": True, "analysis": text})
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _capture_response: routing OFF (current/native behaviour)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestCaptureResponseDefaultPath:
|
||
"""When routing helper says 'native', the existing multimodal envelope wins."""
|
||
|
||
def test_som_capture_returns_multimodal_envelope_when_native(self):
|
||
from tools.computer_use import tool as cu_tool
|
||
|
||
cap = _make_capture(png_b64=_PNG_B64, mode="som")
|
||
with patch.object(cu_tool, "_should_route_through_aux_vision",
|
||
return_value=False):
|
||
resp = cu_tool._capture_response(cap)
|
||
|
||
assert isinstance(resp, dict)
|
||
assert resp.get("_multimodal") is True
|
||
# Image part must use image/png MIME for a PNG payload.
|
||
image_part = next(
|
||
p for p in resp["content"] if p.get("type") == "image_url"
|
||
)
|
||
url = image_part["image_url"]["url"]
|
||
assert url.startswith("data:image/png;base64,")
|
||
assert "vision_analysis" not in resp
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _capture_response: routing ON (the #24015 fix)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestCaptureResponseRoutedToAuxVision:
|
||
"""When routing helper says 'aux', the PNG is pre-analysed and a text
|
||
response is returned with no image_url parts at all."""
|
||
|
||
def test_som_capture_returns_text_with_vision_analysis(
|
||
self, tmp_cache_dir,
|
||
):
|
||
from tools.computer_use import tool as cu_tool
|
||
|
||
cap = _make_capture(mode="som")
|
||
|
||
captured_calls = {}
|
||
|
||
def _fake_run_async(coro):
|
||
captured_calls["called"] = True
|
||
return _stub_aux_analysis(
|
||
"A Safari window showing a GitHub issue page with a 'Sign "
|
||
"in' button and a 'username' text field."
|
||
)
|
||
|
||
# vision_analyze_tool is async; force a sync MagicMock so we can
|
||
# assert positional args without dealing with awaitables.
|
||
fake_vat = MagicMock(return_value="<coro>")
|
||
|
||
with patch.object(cu_tool, "_should_route_through_aux_vision",
|
||
return_value=True), \
|
||
patch("model_tools._run_async", side_effect=_fake_run_async), \
|
||
patch("tools.vision_tools.vision_analyze_tool",
|
||
new_callable=lambda: fake_vat):
|
||
resp = cu_tool._capture_response(cap)
|
||
|
||
# Must be a JSON string, NOT a multimodal envelope. This is exactly
|
||
# the contract that prevents #24015's HTTP 404 from firing on the
|
||
# next agent turn.
|
||
assert isinstance(resp, str)
|
||
assert "data:image" not in resp and "image_url" not in resp
|
||
body = json.loads(resp)
|
||
assert body["mode"] == "som"
|
||
assert body["app"] == "Safari"
|
||
assert "Sign in" in body["vision_analysis"]
|
||
assert body["vision_analysis_routed_via"] == "auxiliary.vision"
|
||
# The original AX-only metadata (window title, element index, app)
|
||
# is preserved alongside the new vision analysis so the agent loses
|
||
# no context vs the multimodal path.
|
||
assert body["window_title"] == "GitHub – Issue #24015"
|
||
assert len(body["elements"]) == 2
|
||
|
||
assert captured_calls.get("called") is True
|
||
# vision_analyze_tool was invoked with a path under the patched cache
|
||
# and a non-empty prompt.
|
||
args, _kwargs = fake_vat.call_args
|
||
path_arg, prompt_arg = args[0], args[1]
|
||
assert str(tmp_cache_dir) in path_arg
|
||
# AX summary is included so the aux model can ground its description
|
||
# against the same set-of-mark index the agent will see.
|
||
assert "Sign in" in prompt_arg
|
||
|
||
|
||
def test_invalid_aux_response_degrades_to_text_payload(self, tmp_cache_dir):
|
||
from tools.computer_use import tool as cu_tool
|
||
|
||
cap = _make_capture(mode="som")
|
||
|
||
def _fake_run_async(_coro):
|
||
return 1234 # not a string at all
|
||
|
||
fake_vat = MagicMock(return_value="<coro>")
|
||
|
||
with patch.object(cu_tool, "_should_route_through_aux_vision",
|
||
return_value=True), \
|
||
patch("model_tools._run_async", side_effect=_fake_run_async), \
|
||
patch("tools.vision_tools.vision_analyze_tool",
|
||
new_callable=lambda: fake_vat):
|
||
resp = cu_tool._capture_response(cap)
|
||
|
||
assert isinstance(resp, str)
|
||
body = json.loads(resp)
|
||
assert body.get("vision_unavailable") is True
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# _should_route_through_aux_vision: end-to-end with real config plumbing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class TestRoutingDecisionWiring:
|
||
"""Verify _should_route_through_aux_vision wires the right config + helper."""
|
||
|
||
def test_explicit_aux_vision_in_config_routes_to_aux(self):
|
||
from tools.computer_use import tool as cu_tool
|
||
|
||
cfg = {
|
||
"model": {"default": "tencent/hy3-preview", "provider": "openrouter"},
|
||
"auxiliary": {
|
||
"vision": {
|
||
"provider": "openrouter",
|
||
"model": "google/gemini-2.5-flash",
|
||
}
|
||
},
|
||
}
|
||
with patch("agent.auxiliary_client._read_main_provider",
|
||
return_value="openrouter"), \
|
||
patch("agent.auxiliary_client._read_main_model",
|
||
return_value="tencent/hy3-preview"), \
|
||
patch("hermes_cli.config.load_config_readonly", return_value=cfg):
|
||
assert cu_tool._should_route_through_aux_vision() is True
|
||
|
||
|
||
def test_helper_decision_exception_is_swallowed(self):
|
||
from tools.computer_use import tool as cu_tool
|
||
|
||
with patch("agent.auxiliary_client._read_main_provider",
|
||
return_value="openrouter"), \
|
||
patch("agent.auxiliary_client._read_main_model",
|
||
return_value="x"), \
|
||
patch("hermes_cli.config.load_config_readonly", return_value={}), \
|
||
patch("tools.vision_tools._native_tool_result_images",
|
||
side_effect=ValueError("policy bug")):
|
||
assert cu_tool._should_route_through_aux_vision() is False
|
||
|
||
|
||
def _route(provider, model, cfg, *, catalog, provider_takes_media, veto=False):
|
||
"""The capture route through the real shared gate; only the capability lookups underneath are patched."""
|
||
from tools.computer_use import tool as cu_tool
|
||
with patch("agent.auxiliary_client._read_main_provider", return_value=provider), \
|
||
patch("agent.auxiliary_client._read_main_model", return_value=model), \
|
||
patch("hermes_cli.config.load_config_readonly", return_value=cfg), \
|
||
patch("agent.image_routing._lookup_supports_vision", return_value=catalog), \
|
||
patch("tools.vision_tools._supports_media_in_tool_results", return_value=provider_takes_media), \
|
||
patch("tools.vision_tools._profile_rejects_tool_media", return_value=veto):
|
||
return cu_tool._should_route_through_aux_vision()
|
||
|
||
|
||
class TestRouteDecision:
|
||
"""True = pre-analyse via auxiliary.vision, False = the multimodal envelope (#115248)."""
|
||
|
||
def test_explicit_aux_backend_wins_over_a_vision_main_model(self):
|
||
"""#24015: a configured auxiliary.vision backend is the de-facto image route in auto mode."""
|
||
cfg = {"auxiliary": {"vision": {"provider": "openrouter", "model": "google/gemini-2.5-flash"}}}
|
||
assert _route("anthropic", "claude-opus-4-5", cfg, catalog=True, provider_takes_media=True) is True
|
||
|
||
def test_vision_main_model_without_aux_backend_stays_native(self):
|
||
assert _route("anthropic", "claude-opus-4-5", {}, catalog=True, provider_takes_media=True) is False
|
||
|
||
def test_catalog_text_only_model_on_a_media_provider_goes_to_aux(self):
|
||
"""deepseek-v3.2 on OpenRouter: the provider carries tool-result media, the model cannot see. Routing it
|
||
native made the agent backstop swap the screenshot for a 'switch to a vision model' error."""
|
||
assert _route("openrouter", "deepseek/deepseek-v3.2", {}, catalog=False, provider_takes_media=True) is True
|
||
|
||
def test_unknown_provider_and_model_fail_closed_to_aux(self):
|
||
assert _route("exotic-provider", "exotic-model", {}, catalog=None, provider_takes_media=False) is True
|
||
|
||
def test_image_input_mode_text_keeps_pixels_out(self):
|
||
cfg = {"agent": {"image_input_mode": "text"}}
|
||
assert _route("anthropic", "claude-opus-4-5", cfg, catalog=True, provider_takes_media=True) is True
|
||
|
||
def test_image_input_mode_native_keeps_a_catalog_unknown_model_native(self):
|
||
"""A proxy alias the catalog does not know: native mode is the user's explicit word for it."""
|
||
cfg = {"agent": {"image_input_mode": "native"}}
|
||
assert _route("anthropic", "my-proxy-claude", cfg, catalog=None, provider_takes_media=True) is False
|
||
|
||
def test_profile_veto_routes_a_vision_model_to_aux(self):
|
||
assert _route("xiaomi", "mimo-v2.5", {}, catalog=True, provider_takes_media=False, veto=True) is True
|