0dbaf33f67
The channel-override baseline from #131295 skipped the session /model tier and normalized the expected model with the agent's current provider. Both disagree with how the agent was built, which turned three cases main handled into wrong verdicts: - a recorded Nous switch off a session /model choice was evicted (the baseline was the channel model, so the (alias, backing) record never matched); - a session-primary fallback that landed on the channel's model was kept; - an OpenRouter primary that fell back to native Anthropic was kept, because the channel slug normalized under the fallback provider equals the fallback model. Resolve the baseline in the same order as _resolve_session_agent_runtime (session /model, channel_overrides, global) and normalize it for the agent's primary provider, the same route the welcome-host pin already uses. A session override whose provider was unavailable this turn ran on the default route (the resolver drops it for that turn and retries next turn), so it is the baseline only when its provider is the agent's primary provider. A model-name comparison cannot see a fallback that serves the configured model on another endpoint; with a channel model that differs from the global one, #131295 turned that case from evicted into kept. A turn whose agent has _provider_fallback_active (set only by _try_activate_fallback, cleared by restore_primary_runtime) now evicts before any name comparison. The baseline lives in GatewayAgentCacheMixin._fallback_baseline_model beside _is_intentional_model_switch, which keeps gateway/run_turn.py under its code-health line cap (the channel lookup from #131295 had grown it past main). Session tier taken from #132238; both regressions diagnosed in review by @andrexibiza. Co-authored-by: salch-cred <141555468+salch-cred@users.noreply.github.com> Co-authored-by: andrexibiza <84248988+andrexibiza@users.noreply.github.com>
403 lines
18 KiB
Python
403 lines
18 KiB
Python
"""Tests that gateway /model switch persists across messages.
|
|
|
|
The gateway /model command stores session overrides in
|
|
``_session_model_overrides``. These must:
|
|
|
|
1. Be applied in ``run_sync()`` so the next agent uses the switched model.
|
|
2. Not be mistaken for fallback activation (which evicts the cached agent).
|
|
3. Survive across multiple messages until /reset clears them.
|
|
|
|
Tests exercise the real ``_apply_session_model_override()`` and
|
|
``_is_intentional_model_switch()`` methods on ``GatewayRunner``.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from gateway.config import ChannelOverride, GatewayConfig, Platform, PlatformConfig
|
|
from gateway.session import SessionEntry, SessionSource, build_session_key
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_source() -> SessionSource:
|
|
return SessionSource(
|
|
platform=Platform.TELEGRAM,
|
|
user_id="u1",
|
|
chat_id="c1",
|
|
user_name="tester",
|
|
chat_type="dm",
|
|
)
|
|
|
|
|
|
def _make_runner():
|
|
"""Create a minimal GatewayRunner with stubbed internals."""
|
|
from gateway.run import GatewayRunner
|
|
|
|
runner = object.__new__(GatewayRunner)
|
|
runner.config = GatewayConfig(
|
|
platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="tok")}
|
|
)
|
|
adapter = MagicMock()
|
|
adapter.send = AsyncMock()
|
|
runner.adapters = {Platform.TELEGRAM: adapter}
|
|
runner._voice_mode = {}
|
|
runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False)
|
|
runner._session_model_overrides = {}
|
|
runner._pending_one_turn_model_restores = {}
|
|
runner._pending_model_notes = {}
|
|
runner._background_tasks = set()
|
|
runner._running_agents = {}
|
|
runner._pending_messages = {}
|
|
runner._pending_approvals = {}
|
|
runner._session_db = None
|
|
runner._agent_cache = {}
|
|
runner._agent_cache_lock = None
|
|
runner._effective_model = None
|
|
runner._effective_provider = None
|
|
runner.session_store = MagicMock()
|
|
session_key = build_session_key(_make_source())
|
|
session_entry = SessionEntry(
|
|
session_key=session_key,
|
|
session_id="sess-1",
|
|
created_at=datetime.now(),
|
|
updated_at=datetime.now(),
|
|
platform=Platform.TELEGRAM,
|
|
chat_type="dm",
|
|
)
|
|
runner.session_store.get_or_create_session.return_value = session_entry
|
|
runner.session_store._entries = {session_key: session_entry}
|
|
return runner
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: _apply_session_model_override
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestApplySessionModelOverride:
|
|
"""Verify _apply_session_model_override replaces config defaults."""
|
|
|
|
def test_override_replaces_all_fields(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
|
|
runner._session_model_overrides[sk] = {
|
|
"model": "gpt-5.4-turbo",
|
|
"provider": "openrouter",
|
|
"api_key": "or-key-123",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"api_mode": "chat_completions",
|
|
}
|
|
|
|
model, rt = runner._apply_session_model_override(
|
|
sk,
|
|
"anthropic/claude-sonnet-4",
|
|
{"provider": "anthropic", "api_key": "ant-key", "base_url": "https://api.anthropic.com", "api_mode": "anthropic_messages"},
|
|
)
|
|
|
|
assert model == "gpt-5.4-turbo"
|
|
assert rt["provider"] == "openrouter"
|
|
assert rt["api_key"] == "or-key-123"
|
|
assert rt["base_url"] == "https://openrouter.ai/api/v1"
|
|
assert rt["api_mode"] == "chat_completions"
|
|
|
|
def test_no_override_returns_originals(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
|
|
orig_model = "anthropic/claude-sonnet-4"
|
|
orig_rt = {"provider": "anthropic", "api_key": "key", "base_url": "https://api.anthropic.com", "api_mode": "anthropic_messages"}
|
|
|
|
model, rt = runner._apply_session_model_override(sk, orig_model, dict(orig_rt))
|
|
|
|
assert model == orig_model
|
|
assert rt == orig_rt
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tests: _is_intentional_model_switch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestIsIntentionalModelSwitch:
|
|
"""The fallback-eviction check must not evict a session whose model differs from the config
|
|
default for a reason the system produced: a /model override, or the Nous gateway moving the
|
|
session off the ``nous/welcome`` alias the config still carries."""
|
|
|
|
def test_matches_override(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
|
|
runner._session_model_overrides[sk] = {
|
|
"model": "gpt-5.4",
|
|
"provider": "openai",
|
|
"api_key": "key",
|
|
"base_url": "",
|
|
"api_mode": "chat_completions",
|
|
}
|
|
|
|
agent = SimpleNamespace(model="gpt-5.4")
|
|
assert runner._is_intentional_model_switch(sk, agent, "openai/gpt-5") is True
|
|
|
|
def test_server_model_switch_off_the_welcome_alias_is_intentional(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
# apply_model_switch moved the session and recorded the move (alias -> backing).
|
|
agent = SimpleNamespace(model="z-ai/glm-5.3-flash", _nous_model_switch=("nous/welcome", "z-ai/glm-5.3-flash"))
|
|
assert runner._is_intentional_model_switch(sk, agent, "nous/welcome") is True
|
|
# A config that names something else is real drift, not the server's move.
|
|
assert runner._is_intentional_model_switch(sk, agent, "openai/gpt-5") is False
|
|
# A later fallback onto a third model is drift too, even with the config still on the alias.
|
|
agent.model = "fallback/model"
|
|
assert runner._is_intentional_model_switch(sk, agent, "nous/welcome") is False
|
|
|
|
def test_plain_drift_is_not_intentional(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
agent = SimpleNamespace(model="fallback/model")
|
|
assert runner._is_intentional_model_switch(sk, agent, "primary/model") is False
|
|
|
|
|
|
class TestFallbackEvictionHonorsChannelOverrides:
|
|
"""A ``channel_overrides`` model is the configured model for that chat, so an agent still running it
|
|
after a successful turn is not a fallback; comparing it with the global default evicted (and rebuilt)
|
|
the cached agent after every turn."""
|
|
|
|
@staticmethod
|
|
def _evicted_after_turn(
|
|
monkeypatch, source, agent_model, *, global_model="default/model", channel_model="chan/model",
|
|
session_model=None, session_provider="nous", **agent_attrs,
|
|
):
|
|
from gateway.turn_context import TurnContext
|
|
|
|
monkeypatch.setattr("gateway.run._resolve_gateway_model", lambda *a, **k: global_model)
|
|
runner = _make_runner()
|
|
runner.config.platforms[Platform.TELEGRAM].channel_overrides = {"c1": ChannelOverride(model=channel_model)}
|
|
if session_model:
|
|
runner._session_model_overrides[build_session_key(source)] = {
|
|
"model": session_model, "provider": session_provider,
|
|
}
|
|
runner._evict_cached_agent = MagicMock()
|
|
ctx = TurnContext(source=source, session_key=build_session_key(source))
|
|
ctx.agent_holder[0] = SimpleNamespace(model=agent_model, **({"provider": "openrouter"} | agent_attrs))
|
|
ctx.result_holder[0] = {"final_response": "ok"}
|
|
runner._run_agent_evict_on_fallback(ctx)
|
|
return runner._evict_cached_agent.called
|
|
|
|
@pytest.mark.parametrize("thread_of_the_channel", [False, True])
|
|
def test_channel_override_model_keeps_the_cached_agent(self, monkeypatch, thread_of_the_channel):
|
|
source = _make_source()
|
|
if thread_of_the_channel:
|
|
source = SessionSource(platform=Platform.TELEGRAM, user_id="u1", chat_id="t9", parent_chat_id="c1",
|
|
user_name="tester", chat_type="thread")
|
|
assert self._evicted_after_turn(monkeypatch, source, "chan/model") is False
|
|
|
|
def test_fallback_off_the_channel_model_still_evicts(self, monkeypatch):
|
|
assert self._evicted_after_turn(monkeypatch, _make_source(), "fallback/model") is True
|
|
|
|
_WELCOME = "https://welcome-api.nousresearch.com/v1"
|
|
|
|
@pytest.mark.parametrize("global_model", ["nous/welcome", "default/model"])
|
|
def test_the_welcome_host_pins_its_model_over_the_channel_model(self, monkeypatch, global_model):
|
|
"""AIAgent.__init__ runs a welcome-host agent on ``nous/welcome`` whatever the chat configured."""
|
|
from hermes_cli.anon_auth import pin_model_for_route
|
|
|
|
agent_model = pin_model_for_route("nous", self._WELCOME, "chan/model")
|
|
assert agent_model == "nous/welcome"
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), agent_model, global_model=global_model,
|
|
provider="nous", base_url=self._WELCOME,
|
|
) is False
|
|
|
|
def test_the_recorded_welcome_alias_switch_keeps_the_cached_agent(self, monkeypatch):
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), "backing/model", provider="nous", base_url=self._WELCOME,
|
|
_nous_model_switch=("nous/welcome", "backing/model"),
|
|
) is False
|
|
|
|
def test_a_fallback_onto_the_welcome_host_still_evicts(self, monkeypatch):
|
|
"""The pin applies to the agent's primary route, not the fallback route it is on now."""
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), "nous/welcome", provider="nous", base_url=self._WELCOME,
|
|
_primary_runtime={"provider": "openrouter", "base_url": "https://openrouter.ai/api/v1"},
|
|
) is True
|
|
|
|
@pytest.mark.parametrize("agent_model, session_provider, switch, evicted", [
|
|
("backing/model", "nous", ("nous/welcome", "backing/model"), False), # the server's recorded move
|
|
("chan/model", "nous", None, True), # a fallback that happens to land on the channel's model
|
|
# The override's provider was unavailable, so this turn ran on the channel's model.
|
|
("chan/model", "openai-codex", None, False),
|
|
], ids=["recorded-switch-off-the-session-model", "fallback-onto-the-channel-model", "override-unavailable"])
|
|
def test_a_session_model_override_outranks_the_channel_model(
|
|
self, monkeypatch, agent_model, session_provider, switch, evicted,
|
|
):
|
|
"""The agent was built on the session's /model choice, so drift is measured from it."""
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), agent_model, global_model="backing/model", session_model="nous/welcome",
|
|
session_provider=session_provider, provider="nous", base_url="https://inference-api.nousresearch.com/v1",
|
|
**({"_nous_model_switch": switch} if switch else {}),
|
|
) is evicted
|
|
|
|
@pytest.mark.parametrize("session_model", [None, "chan/model"], ids=["channel-model", "session-model"])
|
|
def test_a_provider_fallback_serving_the_configured_model_still_evicts(self, monkeypatch, session_model):
|
|
"""Same model name, different endpoint: the model strings match, but the turn ran on a fallback."""
|
|
route = {"provider": "custom", "base_url": "https://primary.example/v1"}
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), "chan/model", session_model=session_model, session_provider="custom",
|
|
provider="custom", base_url="https://fallback.example/v1", _primary_runtime=route,
|
|
_provider_fallback_active=True,
|
|
) is True
|
|
|
|
@pytest.mark.parametrize("primary_provider, evicted", [
|
|
("anthropic", False), # native primary: the agent runs the prefix-stripped channel model
|
|
("openrouter", True), # OpenRouter primary that fell back to native Anthropic
|
|
])
|
|
def test_the_channel_model_is_normalized_for_the_primary_provider(self, monkeypatch, primary_provider, evicted):
|
|
assert self._evicted_after_turn(
|
|
monkeypatch, _make_source(), "claude-sonnet-4-6", channel_model="anthropic/claude-sonnet-4.6",
|
|
provider="anthropic", base_url="https://api.anthropic.com",
|
|
_primary_runtime={"provider": primary_provider},
|
|
) is evicted
|
|
|
|
|
|
class TestOneTurnModelOverrideRestore:
|
|
"""Verify gateway one-turn overrides restore previous session state."""
|
|
|
|
def test_restores_previous_override(self):
|
|
runner = _make_runner()
|
|
sk = build_session_key(_make_source())
|
|
previous = {
|
|
"model": "old/model",
|
|
"provider": "openrouter",
|
|
"api_key": "old-key",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"api_mode": "chat_completions",
|
|
}
|
|
runner._session_model_overrides[sk] = previous
|
|
|
|
snapshot = runner._snapshot_session_model_override(sk)
|
|
runner._session_model_overrides[sk] = {
|
|
"model": "temp/model",
|
|
"provider": "anthropic",
|
|
}
|
|
|
|
runner._restore_session_model_override(sk, snapshot)
|
|
|
|
assert runner._session_model_overrides[sk] == previous
|
|
|
|
|
|
class TestOneTurnNeverPersisted:
|
|
"""/model --once must never write through to the session store.
|
|
|
|
Regression guard for the #29923 review defect: the original
|
|
implementation wrote the once-override through set_model_override, so a
|
|
gateway restart before the finally-restore rehydrated a supposedly
|
|
one-turn model permanently. Drives the real _handle_model_command with
|
|
a mocked switch pipeline and asserts on the store boundary.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _runner_with_store(tmp_path, monkeypatch):
|
|
import hermes_yaml as _yaml
|
|
|
|
import gateway.run as gateway_run
|
|
from gateway.run import GatewayRunner
|
|
from hermes_cli.model_switch import ModelSwitchResult
|
|
|
|
hermes_home = tmp_path / ".hermes"
|
|
hermes_home.mkdir()
|
|
(hermes_home / "config.yaml").write_text(
|
|
_yaml.safe_dump(
|
|
{"model": {"default": "old-model", "provider": "openrouter"}}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setattr(gateway_run, "_hermes_home", hermes_home)
|
|
monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
|
|
monkeypatch.setattr(
|
|
"hermes_cli.model_switch.switch_model",
|
|
lambda **kw: ModelSwitchResult(
|
|
success=True,
|
|
new_model="gpt-5.5",
|
|
target_provider="openrouter",
|
|
provider_changed=False,
|
|
api_key="sk-test",
|
|
base_url="https://openrouter.ai/api/v1",
|
|
api_mode="chat_completions",
|
|
runtime_capabilities={"openai_native_compaction": True},
|
|
provider_label="OpenRouter",
|
|
),
|
|
)
|
|
monkeypatch.setattr("hermes_constants.get_hermes_home", lambda: hermes_home)
|
|
monkeypatch.setattr("hermes_cli.config.get_hermes_home", lambda: hermes_home)
|
|
|
|
runner = object.__new__(GatewayRunner)
|
|
runner.adapters = {}
|
|
runner._voice_mode = {}
|
|
runner._session_model_overrides = {}
|
|
runner._pending_one_turn_model_restores = {}
|
|
runner._running_agents = {}
|
|
# async_session_store is a property over session_store; install the
|
|
# mock behind the private cache attribute it reads.
|
|
_store = MagicMock()
|
|
_store.set_model_override = AsyncMock()
|
|
_store._store = None
|
|
runner.session_store = None
|
|
runner._async_session_store = _store
|
|
return runner
|
|
|
|
@staticmethod
|
|
def _event(text):
|
|
from gateway.platforms.event import MessageEvent, MessageType
|
|
|
|
return MessageEvent(
|
|
text=text,
|
|
message_type=MessageType.TEXT,
|
|
source=_make_source(),
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_once_skips_session_store_write_through(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
runner = self._runner_with_store(tmp_path, monkeypatch)
|
|
sk = build_session_key(_make_source())
|
|
|
|
result = await runner._handle_model_command(
|
|
self._event("/model gpt-5.5 --once")
|
|
)
|
|
|
|
assert result is not None and "gpt-5.5" in result
|
|
# In-memory override installed for the next turn + restore queued...
|
|
assert runner._session_model_overrides[sk]["model"] == "gpt-5.5"
|
|
assert runner._session_model_overrides[sk]["capabilities"] == {
|
|
"openai_native_compaction": True
|
|
}
|
|
assert sk in runner._pending_one_turn_model_restores
|
|
# ...but NEVER written through to the persistent session store.
|
|
runner.async_session_store.set_model_override.assert_not_awaited()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_repeated_once_keeps_the_earliest_restore_target(self, tmp_path, monkeypatch):
|
|
"""`/model X --once` then `/model Y --once` before any turn: the pending snapshot must still
|
|
be the user's standing override (none here), not X — otherwise slot cleanup would make the
|
|
first temporary model permanent."""
|
|
runner = self._runner_with_store(tmp_path, monkeypatch)
|
|
sk = build_session_key(_make_source())
|
|
|
|
await runner._handle_model_command(self._event("/model gpt-5.5 --once"))
|
|
assert runner._session_model_overrides[sk]["model"] == "gpt-5.5"
|
|
await runner._handle_model_command(self._event("/model gpt-5.5 --once"))
|
|
|
|
# The second producer call snapshotted the live gpt-5.5 override; the pending restore
|
|
# must still be the ORIGINAL "no override" state.
|
|
assert runner._pending_one_turn_model_restores[sk]["had_override"] is False
|
|
|