hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

feat(vision): vision.embed_target_bytes replaces the hardcoded 256 KB native embed budget

Browse Source 
The native vision_analyze fast path (and the browser screenshot twins) downscaled every
embed to a fixed _EMBED_TARGET_BYTES = 256 KB. That is fine for photos but turns a
1080x2340 phone screenshot of a table into 540x1170, which the model then reads as
"unreadable" and re-requests (#112095).

The budget is now `vision.embed_target_bytes` in config.yaml (default unchanged: 256 KB,
clamped to 64 KiB..4 MiB so one setting cannot make every later request a multi-megabyte
resend), resolved in the new topical sibling tools/vision_tools_history_budget.py and read
by vision_analyze, browser_vision and browser_exec screenshots alike.

Ported from #112947 by @MohamadKanso (resolver + clamp), relocated out of the facade.
This commit is contained in:
Mohamad Kanso
2026-09-16 22:09:58 -07:00
committed by Teknium
parent 428056e7b9
commit f37336522b
8 changed files with 124 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hermes_cli/config_defaults.py
View File

@@ -1162,6 +1162,13 @@ DEFAULT_CONFIG = {
# instead of going to the agent. [] disables.
"stop_phrases": ["stop"],
},
# Native vision embeds (vision_analyze / browser screenshots on vision-capable main models) ride
# conversation history and are re-sent on every later API call.
"vision": {
# Byte budget for one embedded image (clamped 64 KiB..4 MiB). Raise it for dense phone
# screenshots of tables the model calls "unreadable" at 256 KB.
"embed_target_bytes": 256 * 1024,
},
# "Hey Hermes" hands-free wake word: always-on, on-device hotword detection that starts a fresh
# voice session. Off by default; toggle with /wake.
"wake_word": {

3
tests/tools/test_browser_console.py
View File

@@ -333,7 +333,8 @@ class TestBrowserVisionConfig:
from agent.auxiliary_client import clear_runtime_main, set_runtime_main
from tools.browser_tool import browser_vision
from tools.vision_tools import _EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES
from tools.vision_tools import _EMBED_MAX_DIMENSION
from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES
shots_dir = tmp_path / "browser_screenshots"
shots_dir.mkdir()

63
tests/tools/test_vision_history_budget.py Normal file
View File

@@ -0,0 +1,63 @@
"""Invariants for the native-embed history budgets (#112095).
A native ``vision_analyze`` result is re-sent on every later API call, so the per-embed byte budget
must follow ``vision.embed_target_bytes`` instead of a hardcoded 256 KB.
"""
from __future__ import annotations
import asyncio
import random
import pytest
from hermes_cli.config import get_config_path
from tools import vision_tools_history_budget as budget
from tools.vision_tools import _vision_analyze_native
PIL = pytest.importorskip("PIL.Image")
def _write_config(text: str) -> None:
path = get_config_path()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
def _png(path, size=(16, 16), noisy=False):
img = PIL.new("RGB", size, (200, 40, 40))
if noisy:
rnd = random.Random(7)
img.putdata([(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256))
for _ in range(size[0] * size[1])])
img.save(path)
return str(path)
def _load(image, region=None):
return asyncio.get_event_loop().run_until_complete(_vision_analyze_native(image, "q", region=region))
def _embed_len(result) -> int:
return len(next(p["image_url"]["url"] for p in result["content"] if p.get("type") == "image_url"))
class TestEmbedTargetBytes:
def test_native_embed_follows_configured_budget(self, tmp_path):
"""A 400x400 noisy PNG (~160 KB base64) rides under the 256 KB default untouched, and is
shrunk once ``vision.embed_target_bytes`` drops to 64 KiB."""
dense = _png(tmp_path / "dense.png", size=(400, 400), noisy=True)
default_len = _embed_len(_load(dense))
assert 65536 < default_len <= budget._DEFAULT_EMBED_TARGET_BYTES
_write_config("vision:\n embed_target_bytes: 65536\n")
assert _embed_len(_load(dense)) <= 65536
@pytest.mark.parametrize("raw, expected", [
("not-a-number", budget._DEFAULT_EMBED_TARGET_BYTES),
("true", budget._DEFAULT_EMBED_TARGET_BYTES),
("1", budget._MIN_EMBED_TARGET_BYTES),
(str(64 * 1024 * 1024), budget._MAX_EMBED_TARGET_BYTES),
])
def test_bad_or_extreme_values_are_clamped_to_the_safe_range(self, raw, expected):
_write_config(f"vision:\n embed_target_bytes: {raw}\n")
assert budget.resolve_embed_target_bytes() == expected

5
tests/tools/test_vision_native_fast_path.py
View File

@@ -340,7 +340,7 @@ class TestVisionAnalyzeNative:
except ImportError:
pytest.skip("Pillow not installed — proactive resize is a no-op")
from tools.vision_tools import _EMBED_TARGET_BYTES
from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES
# Noisy PNG that base64-encodes to well over 5 MB (won't compress much).
big = tmp_path / "big.png"
@@ -364,7 +364,8 @@ class TestVisionAnalyzeNative:
def test_embed_caps_are_sized_for_history_reuse(self):
"""Native embeds ride every later turn, so caps must stay well below
the Anthropic 5 MB / 8000px reject limits (#92699)."""
from tools.vision_tools import _EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES
from tools.vision_tools import _EMBED_MAX_DIMENSION
from tools.vision_tools_history_budget import _DEFAULT_EMBED_TARGET_BYTES as _EMBED_TARGET_BYTES
assert _EMBED_TARGET_BYTES <= 512 * 1024
assert _EMBED_MAX_DIMENSION <= 2048

5
tools/browser_tool_vision.py
View File

@@ -61,12 +61,13 @@ def _native_vision_result(
"""
from tools.vision_tools import (
_EMBED_MAX_DIMENSION,
_EMBED_TARGET_BYTES,
_build_native_vision_tool_result,
_resize_image_for_vision,
)
from tools.vision_tools_history_budget import resolve_embed_target_bytes
data_url = _resize_image_for_vision(screenshot_path, mime_type="image/png", max_base64_bytes=_EMBED_TARGET_BYTES,
data_url = _resize_image_for_vision(screenshot_path, mime_type="image/png",
max_base64_bytes=resolve_embed_target_bytes(),
max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
native_result = _build_native_vision_tool_result(image_url=str(screenshot_path), question=question,
image_data_url=data_url,

6
tools/browser_use_cli.py
View File

@@ -339,13 +339,15 @@ def _find_screenshot(stdout: str, since: float) -> Optional[str]:
def _native_screenshot_result(result: Dict[str, Any], path: str) -> Optional[Dict[str, Any]]:
"""Build a multimodal tool result attaching path for vision models"""
try:
from tools.vision_tools import (_EMBED_MAX_DIMENSION, _EMBED_TARGET_BYTES,
from tools.vision_tools import (_EMBED_MAX_DIMENSION,
_resize_image_for_vision, _should_use_native_vision_fast_path)
from tools.vision_tools_history_budget import resolve_embed_target_bytes
if not _should_use_native_vision_fast_path():
return None
# History-reuse cap: this data URL bakes into the tool result and is re-sent every later turn —
# same policy as the vision_analyze / browser_vision native embeds.
data_url = _resize_image_for_vision(Path(path), mime_type="image/png", max_base64_bytes=_EMBED_TARGET_BYTES,
data_url = _resize_image_for_vision(Path(path), mime_type="image/png",
max_base64_bytes=resolve_embed_target_bytes(),
max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
text = json.dumps(result, ensure_ascii=False)
attached = text + "\n\nThe screenshot from this call is attached — inspect it with your native vision."

14
tools/vision_tools.py
View File

@@ -36,6 +36,7 @@ def _load_auxiliary_client() -> None:
from hermes_constants import get_hermes_dir
from tools.debug_helpers import DebugSession
from tools.website_policy import check_website_access
from tools.vision_tools_history_budget import resolve_embed_target_bytes as _resolve_embed_target_bytes
from tools.vision_tools_image_prep import (
_VISION_MAX_VALIDATED_AGGREGATE_PIXELS,
_VISION_MAX_VALIDATED_FRAME_COUNT,
@@ -252,10 +253,10 @@ _MAX_BASE64_BYTES = 20 * 1024 * 1024
# downsamples to a 1568px long edge anyway, so pixels past that cost wire bytes for no fidelity.
# The 20 MB hard ceiling / Anthropic 5 MB reject-cap still apply as safety nets; those are one-shot viewing
# limits, not history-reuse sizes. A 4 MB / 7900px embed was observed at ~400K chars and ~100–260K billed
# tokens per image (#92699), so we size for model reading instead: 256 KB keeps a 1568px screenshot cheap
# enough to ride the session (PNGs that exceed it are downscaled further by the byte-budget ladder), well
# under every provider's per-image limit.
_EMBED_TARGET_BYTES = 256 * 1024
# tokens per image (#92699), so we size for model reading instead: the byte budget is
# ``vision.embed_target_bytes`` (default 256 KB, see vision_tools_history_budget) — it keeps a 1568px
# screenshot cheap enough to ride the session (PNGs that exceed it are downscaled further by the
# byte-budget ladder), well under every provider's per-image limit.
_EMBED_MAX_DIMENSION = 1568
# Target when auto-resizing after a provider size rejection (retry once).
@@ -601,12 +602,13 @@ async def _vision_analyze_native(
# Anthropic still rejects >5 MB / >8000px with a non-retryable 400, but those are one-shot viewing
# limits — history embeds are sized smaller so repeated vision_analyze turns don't blow the context
# (#92699).
embed_target_bytes = _resolve_embed_target_bytes()
_over_dims = await _run_encode_on_cpu_executor(
_image_exceeds_dimension, prepared.path, _EMBED_MAX_DIMENSION)
if len(image_data_url) > _EMBED_TARGET_BYTES or _over_dims:
if len(image_data_url) > embed_target_bytes or _over_dims:
image_data_url = await _resize_prepared(
prepared, _scale_info,
max_base64_bytes=_EMBED_TARGET_BYTES, max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
max_base64_bytes=embed_target_bytes, max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
# Reject rather than embed a session-wedging payload.
if len(image_data_url) > _MAX_BASE64_BYTES:
return tool_error(_too_large_message(image_data_url), success=False)

34
tools/vision_tools_history_budget.py Normal file
View File

@@ -0,0 +1,34 @@
"""History-reuse budgets for native vision embeds (config section ``vision``).
A native ``vision_analyze`` result bakes the image into conversation history, where it is
re-sent on every later API call. ``vision.embed_target_bytes`` bounds that cost
(how large one embed may be); see #112095 for why 256 KB is a budget, not a constant.
"""
from __future__ import annotations
# 256 KB keeps a 1568px screenshot cheap enough to ride the session (#92699); the clamp keeps one
# setting from turning every later request into a multi-megabyte resend or a useless thumbnail.
_DEFAULT_EMBED_TARGET_BYTES = 256 * 1024
_MIN_EMBED_TARGET_BYTES = 64 * 1024
_MAX_EMBED_TARGET_BYTES = 4 * 1024 * 1024
def _cfg_vision(key: str, default=None):
"""``vision.<key>`` from config.yaml; ``default`` when config is unavailable."""
try:
from hermes_cli.config import cfg_get, load_config
return cfg_get(load_config(), "vision", key, default=default)
except Exception:
return default
def resolve_embed_target_bytes() -> int:
"""``vision.embed_target_bytes`` clamped to 64 KiB..4 MiB; the 256 KB default on a bad value."""
raw = _cfg_vision("embed_target_bytes", default=_DEFAULT_EMBED_TARGET_BYTES)
try:
if isinstance(raw, bool):
raise ValueError("boolean is not a byte budget")
target = int(raw)
except (TypeError, ValueError, OverflowError):
return _DEFAULT_EMBED_TARGET_BYTES
return min(max(target, _MIN_EMBED_TARGET_BYTES), _MAX_EMBED_TARGET_BYTES)