From f37336522be2e5e7ac051100390b529ed35a43e6 Mon Sep 17 00:00:00 2001 From: Mohamad Kanso <91088196+MohamadKanso@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:09:58 -0700 Subject: [PATCH] feat(vision): vision.embed_target_bytes replaces the hardcoded 256 KB native embed budget The native vision_analyze fast path (and the browser screenshot twins) downscaled every embed to a fixed _EMBED_TARGET_BYTES = 256 KB. That is fine for photos but turns a 1080x2340 phone screenshot of a table into 540x1170, which the model then reads as "unreadable" and re-requests (#112095). The budget is now `vision.embed_target_bytes` in config.yaml (default unchanged: 256 KB, clamped to 64 KiB..4 MiB so one setting cannot make every later request a multi-megabyte resend), resolved in the new topical sibling tools/vision_tools_history_budget.py and read by vision_analyze, browser_vision and browser_exec screenshots alike. Ported from #112947 by @MohamadKanso (resolver + clamp), relocated out of the facade. --- hermes_cli/config_defaults.py | 7 +++ tests/tools/test_browser_console.py | 3 +- tests/tools/test_vision_history_budget.py | 63 +++++++++++++++++++++ tests/tools/test_vision_native_fast_path.py | 5 +- tools/browser_tool_vision.py | 5 +- tools/browser_use_cli.py | 6 +- tools/vision_tools.py | 14 +++-- tools/vision_tools_history_budget.py | 34 +++++++++++ 8 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 tests/tools/test_vision_history_budget.py create mode 100644 tools/vision_tools_history_budget.py diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index e7d02a75d9..d4b4fd4612 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1162,6 +1162,13 @@ DEFAULT_CONFIG = { # instead of going to the agent. [] disables. "stop_phrases": ["stop"], }, + # Native vision embeds (vision_analyze / browser screenshots on vision-capable main models) ride + # conversation history and are re-sent on every later API call. + "vision": { + # Byte budget for one embedded image (clamped 64 KiB..4 MiB). Raise it for dense phone + # screenshots of tables the model calls "unreadable" at 256 KB. + "embed_target_bytes": 256 * 1024, + }, # "Hey Hermes" hands-free wake word: always-on, on-device hotword detection that starts a fresh # voice session. Off by default; toggle with /wake. "wake_word": { diff --git a/tests/tools/test_browser_console.py b/tests/tools/test_browser_console.py index d8afb8664a..6f521f5547 100644 --- a/tests/tools/test_browser_console.py +++ b/tests/tools/test_browser_console.py @@ -333,7 +333,8 @@ class TestBrowserVisionConfig: from agent.auxiliary_client import clear_runtime_main, set_runtime_main from tools.browser_tool import browser_vision - from tools.vision_tools import _EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES + from tools.vision_tools import _EMBED_MAX_DIMENSION + from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES shots_dir = tmp_path / "browser_screenshots" shots_dir.mkdir() diff --git a/tests/tools/test_vision_history_budget.py b/tests/tools/test_vision_history_budget.py new file mode 100644 index 0000000000..2a75e0c2fb --- /dev/null +++ b/tests/tools/test_vision_history_budget.py @@ -0,0 +1,63 @@ +"""Invariants for the native-embed history budgets (#112095). + +A native ``vision_analyze`` result is re-sent on every later API call, so the per-embed byte budget +must follow ``vision.embed_target_bytes`` instead of a hardcoded 256 KB. +""" +from __future__ import annotations + +import asyncio +import random + +import pytest + +from hermes_cli.config import get_config_path +from tools import vision_tools_history_budget as budget +from tools.vision_tools import _vision_analyze_native + +PIL = pytest.importorskip("PIL.Image") + + +def _write_config(text: str) -> None: + path = get_config_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _png(path, size=(16, 16), noisy=False): + img = PIL.new("RGB", size, (200, 40, 40)) + if noisy: + rnd = random.Random(7) + img.putdata([(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256)) + for _ in range(size[0] * size[1])]) + img.save(path) + return str(path) + + +def _load(image, region=None): + return asyncio.get_event_loop().run_until_complete(_vision_analyze_native(image, "q", region=region)) + + +def _embed_len(result) -> int: + return len(next(p["image_url"]["url"] for p in result["content"] if p.get("type") == "image_url")) + + +class TestEmbedTargetBytes: + def test_native_embed_follows_configured_budget(self, tmp_path): + """A 400x400 noisy PNG (~160 KB base64) rides under the 256 KB default untouched, and is + shrunk once ``vision.embed_target_bytes`` drops to 64 KiB.""" + dense = _png(tmp_path / "dense.png", size=(400, 400), noisy=True) + default_len = _embed_len(_load(dense)) + assert 65536 < default_len <= budget._DEFAULT_EMBED_TARGET_BYTES + + _write_config("vision:\n embed_target_bytes: 65536\n") + assert _embed_len(_load(dense)) <= 65536 + + @pytest.mark.parametrize("raw, expected", [ + ("not-a-number", budget._DEFAULT_EMBED_TARGET_BYTES), + ("true", budget._DEFAULT_EMBED_TARGET_BYTES), + ("1", budget._MIN_EMBED_TARGET_BYTES), + (str(64 * 1024 * 1024), budget._MAX_EMBED_TARGET_BYTES), + ]) + def test_bad_or_extreme_values_are_clamped_to_the_safe_range(self, raw, expected): + _write_config(f"vision:\n embed_target_bytes: {raw}\n") + assert budget.resolve_embed_target_bytes() == expected diff --git a/tests/tools/test_vision_native_fast_path.py b/tests/tools/test_vision_native_fast_path.py index aa85600511..bd6e6e6890 100644 --- a/tests/tools/test_vision_native_fast_path.py +++ b/tests/tools/test_vision_native_fast_path.py @@ -340,7 +340,7 @@ class TestVisionAnalyzeNative: except ImportError: pytest.skip("Pillow not installed — proactive resize is a no-op") - from tools.vision_tools import _EMBED_TARGET_BYTES + from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES # Noisy PNG that base64-encodes to well over 5 MB (won't compress much). big = tmp_path / "big.png" @@ -364,7 +364,8 @@ class TestVisionAnalyzeNative: def test_embed_caps_are_sized_for_history_reuse(self): """Native embeds ride every later turn, so caps must stay well below the Anthropic 5 MB / 8000px reject limits (#92699).""" - from tools.vision_tools import _EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES + from tools.vision_tools import _EMBED_MAX_DIMENSION + from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES assert _EMBED_TARGET_BYTES <= 512 * 1024 assert _EMBED_MAX_DIMENSION <= 2048 diff --git a/tools/browser_tool_vision.py b/tools/browser_tool_vision.py index 561790c158..7d149b0292 100644 --- a/tools/browser_tool_vision.py +++ b/tools/browser_tool_vision.py @@ -61,12 +61,13 @@ def _native_vision_result( """ from tools.vision_tools import ( _EMBED_MAX_DIMENSION, - _EMBED_TARGET_BYTES, _build_native_vision_tool_result, _resize_image_for_vision, ) + from tools.vision_tools_history_budget import resolve_embed_target_bytes - data_url = _resize_image_for_vision(screenshot_path, mime_type="image/png", max_base64_bytes=_EMBED_TARGET_BYTES, + data_url = _resize_image_for_vision(screenshot_path, mime_type="image/png", + max_base64_bytes=resolve_embed_target_bytes(), max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True) native_result = _build_native_vision_tool_result(image_url=str(screenshot_path), question=question, image_data_url=data_url, diff --git a/tools/browser_use_cli.py b/tools/browser_use_cli.py index 1429ea4cee..9204b4ce9a 100644 --- a/tools/browser_use_cli.py +++ b/tools/browser_use_cli.py @@ -339,13 +339,15 @@ def _find_screenshot(stdout: str, since: float) -> Optional[str]: def _native_screenshot_result(result: Dict[str, Any], path: str) -> Optional[Dict[str, Any]]: """Build a multimodal tool result attaching path for vision models""" try: - from tools.vision_tools import (_EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES, + from tools.vision_tools import (_EMBED_MAX_DIMENSION, _resize_image_for_vision, _should_use_native_vision_fast_path) + from tools.vision_tools_history_budget import resolve_embed_target_bytes if not _should_use_native_vision_fast_path(): return None # History-reuse cap: this data URL bakes into the tool result and is re-sent every later turn — # same policy as the vision_analyze / browser_vision native embeds. - data_url = _resize_image_for_vision(Path(path), mime_type="image/png", max_base64_bytes=_EMBED_TARGET_BYTES, + data_url = _resize_image_for_vision(Path(path), mime_type="image/png", + max_base64_bytes=resolve_embed_target_bytes(), max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True) text = json.dumps(result, ensure_ascii=False) attached = text + "\n\nThe screenshot from this call is attached — inspect it with your native vision." diff --git a/tools/vision_tools.py b/tools/vision_tools.py index 506abe3362..06f419a720 100644 --- a/tools/vision_tools.py +++ b/tools/vision_tools.py @@ -36,6 +36,7 @@ def _load_auxiliary_client() -> None: from hermes_constants import get_hermes_dir from tools.debug_helpers import DebugSession from tools.website_policy import check_website_access +from tools.vision_tools_history_budget import resolve_embed_target_bytes as _resolve_embed_target_bytes from tools.vision_tools_image_prep import ( _VISION_MAX_VALIDATED_AGGREGATE_PIXELS, _VISION_MAX_VALIDATED_FRAME_COUNT, @@ -252,10 +253,10 @@ _MAX_BASE64_BYTES = 20 * 1024 * 1024 # downsamples to a 1568px long edge anyway, so pixels past that cost wire bytes for no fidelity. # The 20 MB hard ceiling / Anthropic 5 MB reject-cap still apply as safety nets; those are one-shot viewing # limits, not history-reuse sizes. A 4 MB / 7900px embed was observed at ~400K chars and ~100–260K billed -# tokens per image (#92699), so we size for model reading instead: 256 KB keeps a 1568px screenshot cheap -# enough to ride the session (PNGs that exceed it are downscaled further by the byte-budget ladder), well -# under every provider's per-image limit. -_EMBED_TARGET_BYTES = 256 * 1024 +# tokens per image (#92699), so we size for model reading instead: the byte budget is +# ``vision.embed_target_bytes`` (default 256 KB, see vision_tools_history_budget) — it keeps a 1568px +# screenshot cheap enough to ride the session (PNGs that exceed it are downscaled further by the +# byte-budget ladder), well under every provider's per-image limit. _EMBED_MAX_DIMENSION = 1568 # Target when auto-resizing after a provider size rejection (retry once). @@ -601,12 +602,13 @@ async def _vision_analyze_native( # Anthropic still rejects >5 MB / >8000px with a non-retryable 400, but those are one-shot viewing # limits — history embeds are sized smaller so repeated vision_analyze turns don't blow the context # (#92699). + embed_target_bytes = _resolve_embed_target_bytes() _over_dims = await _run_encode_on_cpu_executor( _image_exceeds_dimension, prepared.path, _EMBED_MAX_DIMENSION) - if len(image_data_url) > _EMBED_TARGET_BYTES or _over_dims: + if len(image_data_url) > embed_target_bytes or _over_dims: image_data_url = await _resize_prepared( prepared, _scale_info, - max_base64_bytes=_EMBED_TARGET_BYTES, max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True) + max_base64_bytes=embed_target_bytes, max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True) # Reject rather than embed a session-wedging payload. if len(image_data_url) > _MAX_BASE64_BYTES: return tool_error(_too_large_message(image_data_url), success=False) diff --git a/tools/vision_tools_history_budget.py b/tools/vision_tools_history_budget.py new file mode 100644 index 0000000000..d72a91fcdc --- /dev/null +++ b/tools/vision_tools_history_budget.py @@ -0,0 +1,34 @@ +"""History-reuse budgets for native vision embeds (config section ``vision``). + +A native ``vision_analyze`` result bakes the image into conversation history, where it is +re-sent on every later API call. ``vision.embed_target_bytes`` bounds that cost +(how large one embed may be); see #112095 for why 256 KB is a budget, not a constant. +""" +from __future__ import annotations + +# 256 KB keeps a 1568px screenshot cheap enough to ride the session (#92699); the clamp keeps one +# setting from turning every later request into a multi-megabyte resend or a useless thumbnail. +_DEFAULT_EMBED_TARGET_BYTES = 256 * 1024 +_MIN_EMBED_TARGET_BYTES = 64 * 1024 +_MAX_EMBED_TARGET_BYTES = 4 * 1024 * 1024 + + +def _cfg_vision(key: str, default=None): + """``vision.`` from config.yaml; ``default`` when config is unavailable.""" + try: + from hermes_cli.config import cfg_get, load_config + return cfg_get(load_config(), "vision", key, default=default) + except Exception: + return default + + +def resolve_embed_target_bytes() -> int: + """``vision.embed_target_bytes`` clamped to 64 KiB..4 MiB; the 256 KB default on a bad value.""" + raw = _cfg_vision("embed_target_bytes", default=_DEFAULT_EMBED_TARGET_BYTES) + try: + if isinstance(raw, bool): + raise ValueError("boolean is not a byte budget") + target = int(raw) + except (TypeError, ValueError, OverflowError): + return _DEFAULT_EMBED_TARGET_BYTES + return min(max(target, _MIN_EMBED_TARGET_BYTES), _MAX_EMBED_TARGET_BYTES)