hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

test: trim the context-cache guard tests to invariants; docs: name the surfaces that actually confirm

Browse Source 
Tests collapse 13 change-detectors into 7 invariants (silent below threshold /
without context, fires above, same-model re-select silent, config override and
0-disables, registry threading, agent context derivation). The legacy 5-arg
guard test goes with the TypeError fallback it covered: that fallback was
defence for a case nobody has (every in-tree guard and test double is
*args-tolerant) and would re-run a guard whose real TypeError it masked, so the
rebased port passes the context positionally like every other argument.

Docs no longer claim the confirm fires on the Telegram/Discord pickers or the
dashboard: those surfaces call combined_selection_warning() without a live
agent, so the context-cache guard is (correctly) silent there.
This commit is contained in:
teknium1
2026-09-13 19:55:18 -07:00
committed by Teknium
parent 8b6931393e
commit cb3447b139
3 changed files with 33 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hermes_cli/model_selection_guards.py
View File

@@ -83,7 +83,6 @@ def _data_policy_guard(
# Context-token threshold above which a mid-session switch asks for confirmation: providers key
# prompt caches per model, so the first call after a switch re-reads the whole context uncached.
# Mirrors deepagents' `warnings.model_switch_token_threshold` (langchain-ai/deepagents#5829).
DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD = 100_000

121
tests/hermes_cli/test_context_cache_switch_guard.py
View File

@@ -1,10 +1,8 @@
"""Tests for the context-cache model-switch guard.
"""Context-cache model-switch guard.
Ported from langchain-ai/deepagents#5829 ("confirm model switches with large
context"): a mid-session model switch abandons the provider prompt cache, so
the first call after the switch re-reads the whole conversation at full input
price. The guard asks for confirmation when the live session exceeds a
configurable token threshold.
A mid-session model switch abandons the provider prompt cache, so the first call after the switch
re-reads the whole conversation at full input price. The guard asks for confirmation only when the
live session exceeds a configurable token threshold.
"""
from unittest.mock import patch
@@ -22,121 +20,66 @@ def _no_config(*_a, **_k):
raise FileNotFoundError("no config in tests")
def _guard(model, ctx, provider="openrouter"):
with patch("hermes_cli.config.load_config", _no_config):
return _context_cache_guard(model, provider, None, None, None, ctx)
def _guard(model, ctx, cfg=_no_config):
with patch("hermes_cli.config.load_config", cfg):
return _context_cache_guard(model, "openrouter", None, None, None, ctx)
class TestContextCacheGuard:
def test_silent_without_selection_context(self):
def test_silent_without_context_or_below_threshold(self):
assert _guard("new/model", None) is None
def test_silent_below_threshold(self):
ctx = SelectionContext(context_tokens=5_000, current_model="old/model")
assert _guard("new/model", ctx) is None
assert _guard("new/model", SelectionContext(context_tokens=5_000, current_model="old/model")) is None
def test_fires_above_default_threshold(self):
ctx = SelectionContext(
context_tokens=DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD + 1,
current_model="old/model",
)
warning = _guard("new/model", ctx)
tokens = DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD + 1
warning = _guard("new/model", SelectionContext(context_tokens=tokens, current_model="old/model"))
assert warning is not None
assert warning.kind == "context_cache"
assert "uncached" in warning.message
assert f"{DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD + 1:,}" in warning.message
assert f"{tokens:,}" in warning.message
def test_same_model_reselect_stays_silent(self):
ctx = SelectionContext(
context_tokens=DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD * 2,
current_model="same/model",
)
ctx = SelectionContext(context_tokens=DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD * 2, current_model="same/model")
assert _guard("same/model", ctx) is None
def test_config_threshold_override(self):
def _cfg():
return {"model": {"switch_context_confirm_tokens": 10_000}}
def test_config_threshold_override_and_zero_disables(self):
ctx = SelectionContext(context_tokens=20_000, current_model="old/model")
with patch("hermes_cli.config.load_config", _cfg):
warning = _context_cache_guard(
"new/model", "openrouter", None, None, None, ctx
)
assert warning is not None
def test_config_zero_disables(self):
def _cfg():
return {"model": {"switch_context_confirm_tokens": 0}}
ctx = SelectionContext(context_tokens=10**9, current_model="old/model")
with patch("hermes_cli.config.load_config", _cfg):
assert (
_context_cache_guard("new/model", "openrouter", None, None, None, ctx)
is None
)
assert _guard("new/model", ctx, lambda: {"model": {"switch_context_confirm_tokens": 10_000}}) is not None
huge = SelectionContext(context_tokens=10**9, current_model="old/model")
assert _guard("new/model", huge, lambda: {"model": {"switch_context_confirm_tokens": 0}}) is None
def test_registry_threads_selection_context(self):
ctx = SelectionContext(
context_tokens=DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD + 1,
current_model="old/model",
)
ctx = SelectionContext(context_tokens=DEFAULT_CONTEXT_CACHE_SWITCH_THRESHOLD + 1, current_model="old/model")
with patch("hermes_cli.config.load_config", _no_config):
warnings = selection_warnings(
"new/model", provider="openrouter", selection_context=ctx
)
assert any(w.kind == "context_cache" for w in warnings)
def test_registry_silent_without_context(self):
with patch("hermes_cli.config.load_config", _no_config):
warnings = selection_warnings("new/model", provider="openrouter")
assert not any(w.kind == "context_cache" for w in warnings)
def test_legacy_five_arg_guard_still_supported(self):
# Externally patched guards with the pre-context 5-arg signature must
# not break the registry (back-compat TypeError fallback).
def _old_style(model, provider, base_url, api_key, model_info):
from hermes_cli.model_selection_guards import SelectionWarning
return SelectionWarning("cost", "t", model, provider or "", "OLD")
with patch(
"hermes_cli.model_selection_guards._GUARDS", (_old_style,)
):
warnings = selection_warnings("m", provider="p")
assert [w.message for w in warnings] == ["OLD"]
with_ctx = selection_warnings("new/model", provider="openrouter", selection_context=ctx)
without = selection_warnings("new/model", provider="openrouter")
assert any(w.kind == "context_cache" for w in with_ctx)
assert not any(w.kind == "context_cache" for w in without)
class TestSelectionContextForAgent:
def test_none_agent(self):
assert selection_context_for_agent(None) is None
def test_uses_compressor_measured_tokens(self):
def test_measured_tokens_then_session_counter_fallback(self):
class _CC:
last_prompt_tokens = 123_456
class _Agent:
class _Measured:
context_compressor = _CC()
model = "current/model"
ctx = selection_context_for_agent(_Agent())
assert ctx is not None
assert ctx.context_tokens == 123_456
assert ctx.current_model == "current/model"
def test_falls_back_to_session_prompt_tokens(self):
class _Agent:
class _Fallback:
context_compressor = None
session_prompt_tokens = 42_000
model = "current/model"
ctx = selection_context_for_agent(_Agent())
assert ctx is not None
assert ctx.context_tokens == 42_000
ctx = selection_context_for_agent(_Measured())
assert (ctx.context_tokens, ctx.current_model) == (123_456, "current/model")
assert selection_context_for_agent(_Fallback()).context_tokens == 42_000
def test_empty_session_returns_none(self):
class _Agent:
def test_no_agent_or_empty_session_returns_none(self):
class _Empty:
context_compressor = None
session_prompt_tokens = 0
model = "current/model"
assert selection_context_for_agent(_Agent()) is None
assert selection_context_for_agent(None) is None
assert selection_context_for_agent(_Empty()) is None

2
website/docs/user-guide/configuring-models.md
View File

@@ -55,7 +55,7 @@ When you switch models **inside an active session** (Herm TUI model picker, `her
Prompt caches are keyed to the model serving the request, so any mid-conversation model change — an explicit `/model` switch, an [automatic fallback](./features/fallback-providers.md), or a [credential-pool](./features/credential-pools.md) rotation onto a different account — means the next message re-reads the entire conversation at full input-token price instead of the cached (~75–90% discounted) rate. On a long session this one-time re-read can dwarf the per-token difference between the two models. Switch when you need to, but prefer doing it early in a conversation or right after starting a fresh session.
:::
Because of that one-time re-read cost, Hermes asks for **explicit confirmation** before applying a mid-session switch when the live session already holds a large context (default: **100,000 tokens**, measured from the latest provider-billed prompt size). The confirmation renders through the same selection-guard prompt as the expensive-model and data-training warnings on every surface (CLI/TUI picker, gateway `/model`, Telegram/Discord pickers, dashboard). Tune or disable it in `config.yaml`:
Because of that one-time re-read cost, Hermes asks for **explicit confirmation** before applying a mid-session switch when the live session already holds a large context (default: **100,000 tokens**, measured from the latest provider-billed prompt size). The confirmation renders through the same selection-guard prompt as the expensive-model and data-training warnings wherever a live session is switching: the CLI and TUI `/model` command and picker, and a typed gateway `/model` in a chat with an active agent. Tune or disable it in `config.yaml`:
```yaml
model: