9927e3cd06
Follow-up to #134847 (review findings): - _stash_pending_model_switch ran the pick-time guards with only the provider the user typed. A bare `/model o1-pro` mid-turn resolved against the live provider at turn start, where the cost guard fired and dropped the queued pick; the "pick it again" toast then looped. Guard on the live agent's provider when none was given. - The turn-start drop toast appended the full guard banner after "\n\n"; the Desktop toast collapses newlines and the TUI status bar truncates, so it rendered as a run-on blob. The toast is one sentence now, the re-pick confirm carries the banner, and the server log names the guard (first line). - context_switch_guard._estimate_tokens still fell back to the lifetime session_prompt_tokens sum (#126343) when no history or measurement existed. Removed the fallback.
219 lines
7.1 KiB
Python
219 lines
7.1 KiB
Python
"""Tests for hermes_cli.context_switch_guard."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
from hermes_cli.context_switch_guard import merge_preflight_compression_warning
|
|
from hermes_cli.model_switch import ModelSwitchResult
|
|
|
|
|
|
def _result(*, model: str = "small-model") -> ModelSwitchResult:
|
|
return ModelSwitchResult(
|
|
success=True,
|
|
new_model=model,
|
|
target_provider="openrouter",
|
|
provider_changed=False,
|
|
api_key="k",
|
|
base_url="https://example.com/v1",
|
|
api_mode="chat_completions",
|
|
provider_label="openrouter",
|
|
model_info={"context_length": 32_000},
|
|
)
|
|
|
|
|
|
def _compressor(
|
|
monkeypatch,
|
|
*,
|
|
context_length: int = 200_000,
|
|
threshold_tokens_cap: int | None = None,
|
|
):
|
|
from agent.context_compressor import ContextCompressor
|
|
|
|
monkeypatch.setattr(
|
|
"agent.context_compressor.get_model_context_length",
|
|
lambda *a, **k: context_length,
|
|
)
|
|
return ContextCompressor(
|
|
model="big-model",
|
|
threshold_percent=0.5,
|
|
protect_first_n=3,
|
|
protect_last_n=20,
|
|
quiet_mode=True,
|
|
config_context_length=context_length,
|
|
threshold_tokens_cap=threshold_tokens_cap,
|
|
)
|
|
|
|
|
|
|
|
|
|
def test_merge_appends_to_existing_warning(monkeypatch):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.context_switch_guard._estimate_tokens",
|
|
lambda *a, **k: 90_000,
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.context_switch_guard.resolve_display_context_length",
|
|
lambda *a, **k: 32_000,
|
|
)
|
|
cc = _compressor(monkeypatch)
|
|
agent = SimpleNamespace(
|
|
context_compressor=cc,
|
|
compression_enabled=True,
|
|
base_url="",
|
|
api_key="",
|
|
)
|
|
result = _result()
|
|
result.warning_message = "expensive"
|
|
merge_preflight_compression_warning(result, agent=agent)
|
|
assert "expensive" in result.warning_message
|
|
assert "preflight compression" in result.warning_message
|
|
|
|
|
|
def test_cap_lowers_the_switch_warning_threshold_below_the_ratio(monkeypatch):
|
|
"""The warning quotes the trigger the compressor will install: on a 1M target the ratio alone says
|
|
500K (no warning at 300K in-flight), the cap says less — the guard must warn with the capped number."""
|
|
cap = 256_000
|
|
monkeypatch.setattr(
|
|
"hermes_cli.context_switch_guard._estimate_tokens",
|
|
lambda *a, **k: 300_000,
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.context_switch_guard.resolve_display_context_length",
|
|
lambda *a, **k: 1_000_000,
|
|
)
|
|
cc = _compressor(
|
|
monkeypatch,
|
|
context_length=200_000,
|
|
threshold_tokens_cap=cap,
|
|
)
|
|
agent = SimpleNamespace(
|
|
context_compressor=cc,
|
|
compression_enabled=True,
|
|
base_url="",
|
|
api_key="",
|
|
)
|
|
|
|
result = _result(model="large-model")
|
|
merge_preflight_compression_warning(result, agent=agent)
|
|
|
|
assert "preflight compression" in result.warning_message
|
|
assert f"auto-compress at ~{cap:,}" in result.warning_message
|
|
|
|
|
|
def test_custom_provider_context_avoids_false_shrink_warning(monkeypatch):
|
|
"""Classic CLI used to omit custom_providers from the shrink warning.
|
|
|
|
Repro: switch onto a custom endpoint with models.<id>.context_length=1M
|
|
while session ~147k. Probe fails → hardcoded catalog match on "qwen"
|
|
(131072) → false "Context window shrinks (... → 131,072)" warning, even
|
|
though /model confirmation and the status bar correctly show 1M.
|
|
"""
|
|
custom_provs = [
|
|
{
|
|
"name": "qwen-token-plan",
|
|
"base_url": "https://token-plan.example/compatible-mode/v1",
|
|
"models": {
|
|
"qwen3.9-max-preview": {"context_length": 1_048_576},
|
|
},
|
|
}
|
|
]
|
|
# Force the probe-down path that hit the "qwen" → 131072 catalog match
|
|
# when custom_providers was not threaded through.
|
|
monkeypatch.setattr(
|
|
"agent.model_metadata._resolve_endpoint_context_length",
|
|
lambda *a, **k: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"agent.model_metadata._query_ollama_api_show",
|
|
lambda *a, **k: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.context_switch_guard._estimate_tokens",
|
|
lambda *a, **k: 147_053,
|
|
)
|
|
cc = _compressor(monkeypatch, context_length=1_000_000)
|
|
agent = SimpleNamespace(
|
|
model="MiniMax-M3",
|
|
provider="minimax",
|
|
context_compressor=cc,
|
|
compression_enabled=True,
|
|
conversation_history=[],
|
|
base_url="https://api.minimax.example/v1",
|
|
api_key="",
|
|
_custom_providers=custom_provs,
|
|
)
|
|
result = ModelSwitchResult(
|
|
success=True,
|
|
new_model="qwen3.9-max-preview",
|
|
target_provider="qwen-token-plan",
|
|
provider_changed=True,
|
|
api_key="k",
|
|
base_url="https://token-plan.example/compatible-mode/v1",
|
|
api_mode="chat_completions",
|
|
provider_label="qwen-token-plan",
|
|
model_info=None,
|
|
)
|
|
|
|
# Explicit custom_providers — no false shrink warning (1M > 147k*2).
|
|
merge_preflight_compression_warning(
|
|
result,
|
|
agent=agent,
|
|
custom_providers=custom_provs,
|
|
)
|
|
assert not result.warning_message
|
|
|
|
# Agent snapshot alone (classic CLI historically forgot to pass the kwarg).
|
|
result2 = ModelSwitchResult(
|
|
success=True,
|
|
new_model="qwen3.9-max-preview",
|
|
target_provider="qwen-token-plan",
|
|
provider_changed=True,
|
|
api_key="k",
|
|
base_url="https://token-plan.example/compatible-mode/v1",
|
|
api_mode="chat_completions",
|
|
provider_label="qwen-token-plan",
|
|
model_info=None,
|
|
)
|
|
merge_preflight_compression_warning(result2, agent=agent)
|
|
assert not result2.warning_message
|
|
|
|
# Without any custom_providers source, catalog match still warns (131k).
|
|
agent_no_cp = SimpleNamespace(
|
|
model="MiniMax-M3",
|
|
provider="minimax",
|
|
context_compressor=cc,
|
|
compression_enabled=True,
|
|
conversation_history=[],
|
|
base_url="https://api.minimax.example/v1",
|
|
api_key="",
|
|
_custom_providers=None,
|
|
)
|
|
result3 = ModelSwitchResult(
|
|
success=True,
|
|
new_model="qwen3.9-max-preview",
|
|
target_provider="qwen-token-plan",
|
|
provider_changed=True,
|
|
api_key="k",
|
|
base_url="https://token-plan.example/compatible-mode/v1",
|
|
api_mode="chat_completions",
|
|
provider_label="qwen-token-plan",
|
|
model_info=None,
|
|
)
|
|
merge_preflight_compression_warning(result3, agent=agent_no_cp)
|
|
assert result3.warning_message
|
|
assert "preflight compression" in result3.warning_message
|
|
assert "shrinks" in result3.warning_message
|
|
# Must not honor the unused 1M custom override when no providers were passed.
|
|
assert "1,048,576" not in result3.warning_message
|
|
|
|
|
|
def test_lifetime_prompt_counter_is_not_a_size_estimate():
|
|
"""``session_prompt_tokens`` sums every call; without a measured size the guard stays silent (#126343)."""
|
|
from hermes_cli.context_switch_guard import _estimate_tokens
|
|
|
|
agent = SimpleNamespace(context_compressor=SimpleNamespace(last_prompt_tokens=0),
|
|
session_prompt_tokens=16_282_033)
|
|
|
|
assert _estimate_tokens(agent, None) is None
|