fix(models): keep generation models out of chat

Chat catalogs and the session switch treated image and video generation
models as chat. Exclude them by the capability type and name shape the
catalog already publishes, reject selecting one as the session model, and
do not restore a primary already known to be non-chat. Desktop shows the
fallback switch in the transcript.
This commit is contained in:
brooklyn!
2026-09-24 06:14:53 -05:00
parent d0cb567273
commit 296ec08dcf
10 changed files with 462 additions and 21 deletions

View File

@@ -1263,10 +1263,13 @@ def restore_primary_runtime(agent) -> bool:
primary_provider = str((rt or {}).get("provider") or "").strip().lower()
primary_model = str((rt or {}).get("model") or "").strip()
from agent.fallback_cooldown import _is_entitlement_rejected
if primary_model and _is_entitlement_rejected(agent, primary_provider, primary_model):
# The primary slug was rejected as unentitled for this account (#106475): restoring
# here would announce a recovery that was never verified and re-fail every turn.
# Stay on the fallback; the user sees the terminal entitlement error instead.
from hermes_cli.chat_catalog import is_known_non_chat_model
if primary_model and (
_is_entitlement_rejected(agent, primary_provider, primary_model)
or is_known_non_chat_model(primary_model)
):
# Unentitled (#106475) or already known non-chat: restoring would announce a recovery
# that was never verified and re-fail every turn. Stay on the fallback.
return False
primary_runtime_base_url = str((rt or {}).get("base_url") or "")

View File

@@ -0,0 +1,61 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { handleStatusEvent } from './status'
import type { GatewayEventContext } from './types'
function statusContext(text: string, kind = 'lifecycle') {
const updateSessionState = vi.fn((_id: string, updater: (state: { messages: unknown[] }) => unknown) =>
updater({ messages: [] })
)
const payload = { kind, text } as GatewayEventContext['payload']
const ctx: GatewayEventContext = {
deps: {
compactedTurnRef: { current: new Set<string>() },
failAssistantMessage: vi.fn(),
flushQueuedDeltas: vi.fn(),
hydrateFromStoredSession: vi.fn(),
queryClient: { invalidateQueries: vi.fn() },
sessionStateByRuntimeIdRef: { current: new Map() },
updateSessionState
} as unknown as GatewayEventContext['deps'],
event: { payload, session_id: 'sess-1', type: 'status.update' },
explicitSid: 'sess-1',
fromActiveSource: () => true,
isActiveEvent: true,
occurredAt: 1_700_000_100,
payload,
scheduleConfigRefresh: vi.fn(),
sessionId: 'sess-1'
}
return { ctx, updateSessionState }
}
function systemText(updateSessionState: ReturnType<typeof statusContext>['updateSessionState']): string {
const updater = updateSessionState.mock.calls[0]?.[1]
const next = updater?.({ messages: [] }) as { messages: Array<{ parts: Array<{ text?: string }>; role: string }> } | undefined
return next?.messages.find(message => message.role === 'system')?.parts[0]?.text ?? ''
}
afterEach(() => {
vi.restoreAllMocks()
})
describe('desktop fallback switch', () => {
it('shows a model fallback switch in the transcript', () => {
const notice = '⚠️ Model fallback: wan2.7-image-pro via alibaba-token-plan unavailable (bad request); using qwen3.8-max via alibaba-token-plan.'
const { ctx, updateSessionState } = statusContext(notice)
expect(handleStatusEvent(ctx)).toBe(true)
expect(systemText(updateSessionState)).toContain('qwen3.8-max')
expect(systemText(updateSessionState)).toMatch(/fallback/i)
})
it('does not turn unrelated lifecycle status into a transcript line', () => {
const { ctx, updateSessionState } = statusContext('⚠️ Rate limited — retrying')
expect(handleStatusEvent(ctx)).toBe(true)
expect(updateSessionState).not.toHaveBeenCalled()
})
})

View File

@@ -21,6 +21,15 @@ import { clearActiveSessionTodos } from '@/store/todos'
import type { GatewayEventContext } from './types'
/** Lifecycle text that announces the session left its selected model for a fallback. */
export function isFallbackSwitchStatus(kind: string | undefined, text: string): boolean {
if (kind === 'fallback') {
return Boolean(text.trim())
}
return /model fallback|provider fallback|switched to fallback|switching to fallback/i.test(text)
}
/** status.update / review.summary / notification.show / notification.clear /
* error — the status-and-notice tail of the dispatcher. */
export function handleStatusEvent(ctx: GatewayEventContext): boolean {
@@ -111,6 +120,25 @@ export function handleStatusEvent(ctx: GatewayEventContext): boolean {
void refreshBackgroundProcesses(sessionId)
} else if (sessionId && payload?.kind === 'goal') {
applyGoalStatusText(sessionId, coerceGatewayText(payload?.text))
} else if (sessionId && isFallbackSwitchStatus(payload?.kind, coerceGatewayText(payload?.text))) {
// A provider/model switch is durable: the TUI paints it on the status rail, but Desktop
// used to swallow every non-compaction status.update, so the reply came from a different
// model with no indication.
const text = coerceGatewayText(payload?.text).trim()
flushQueuedDeltas(sessionId)
updateSessionState(sessionId, state => ({
...state,
messages: [
...state.messages,
{
id: `fallback-switch-${occurredAt}`,
role: 'system',
parts: [textPart(text, occurredAt)],
timestamp: occurredAt
}
]
}))
}
return true

136
hermes_cli/chat_catalog.py Normal file
View File

@@ -0,0 +1,136 @@
"""Admit only chat models into shared chat catalogs.
Generation models are identified by the capability type or name shape the
catalog already publishes. Missing, malformed, or unknown metadata fails open.
No single model id is special-cased.
"""
from __future__ import annotations
import re
from typing import Any
# Structured catalog types that mean "this row is not a chat completion".
_GENERATION_TYPES = frozenset({
"image",
"image_generation",
"image-generation",
"image-gen",
"video",
"video_generation",
"video-generation",
"video-gen",
})
_GENERATION_OUTPUTS = frozenset({"image", "video"})
_CHAT_OUTPUTS = frozenset({"text", "chat"})
# Separator-qualified generation tokens, plus the generation id tokens the
# DeepInfra chat catalog already excludes. ``vision`` / ``vl`` are not tokens
# here — those remain chat models.
_GENERATION_ID_RE = re.compile(
r"(?i)(?:^|[/_.-])(?:"
r"image(?:[/_.-]gen(?:eration)?)?|"
r"video(?:[/_.-]gen(?:eration)?)?|"
r"text[/_.-]to[/_.-](?:image|video)|"
r"stable-diffusion|sdxl|flux"
r")(?=$|[/_.-])"
)
# Ids a live catalog item already proved are generation-only, even when the
# id itself has no generation token. Process-local: the proof came from this
# process's catalog parse.
_seen_generation_ids: set[str] = set()
def note_generation_model(model_id: Any) -> None:
key = str(model_id or "").strip().lower()
if key:
_seen_generation_ids.add(key)
def model_id_is_generation(model_id: Any) -> bool:
"""Whether *model_id* names an image/video generation surface."""
return bool(_GENERATION_ID_RE.search(str(model_id or "").strip()))
def _tag_list_is_generation(tags: Any) -> bool:
if not isinstance(tags, list):
return False
return any(str(tag or "").strip().lower() in _GENERATION_TYPES for tag in tags)
def _output_modalities_are_generation(outputs: Any) -> bool:
if not isinstance(outputs, list) or not outputs:
return False
kinds = {str(item or "").strip().lower() for item in outputs if str(item or "").strip()}
return bool(kinds & _GENERATION_OUTPUTS) and not bool(kinds & _CHAT_OUTPUTS)
def catalog_item_is_generation(item: Any) -> bool:
"""Fail open unless this catalog row's own type or name proves it is generation-only."""
if not isinstance(item, dict):
return False
if model_id_is_generation(item.get("id")):
return True
capabilities = item.get("capabilities")
if isinstance(capabilities, dict):
model_type = str(capabilities.get("type") or "").strip().lower()
if model_type in _GENERATION_TYPES:
return True
if _tag_list_is_generation(capabilities.get("tags")):
return True
top_type = item.get("type")
if isinstance(top_type, str) and top_type.strip().lower() in _GENERATION_TYPES:
return True
if _tag_list_is_generation(item.get("tags")):
return True
architecture = item.get("architecture")
if isinstance(architecture, dict) and _output_modalities_are_generation(architecture.get("output_modalities")):
return True
metadata = item.get("metadata")
if isinstance(metadata, dict) and _tag_list_is_generation(metadata.get("tags")):
return True
return False
def note_catalog_item(item: Any) -> bool:
"""Remember a generation row and report whether it must leave the chat catalog."""
if not catalog_item_is_generation(item):
return False
if isinstance(item, dict):
note_generation_model(item.get("id"))
return True
def is_known_non_chat_model(model_id: Any) -> bool:
"""True when the id's name shape, or a catalog item already parsed, says it is not chat."""
mid = str(model_id or "").strip()
return model_id_is_generation(mid) or mid.lower() in _seen_generation_ids
def without_generation_models(models):
"""Drop generation ids, preserving a list subclass (curated-fallback marker)."""
if not models:
return models
kept = [model for model in models if not is_known_non_chat_model(model)]
if len(kept) == len(list(models)):
return models
try:
return type(models)(kept)
except TypeError:
return kept
def chat_catalog_ids(items: Any) -> list[str]:
"""Model ids from a ``/models`` payload, minus generation rows."""
if not isinstance(items, list):
return []
ids: list[str] = []
for item in items:
if not isinstance(item, dict) or "id" not in item:
continue
if note_catalog_item(item):
continue
ids.append(item["id"])
return ids

View File

@@ -1610,6 +1610,15 @@ def _validate_switch(st: _Switch) -> Optional[ModelSwitchResult]:
st.new_model = _resolve_named_custom_model_id(st.new_model, st.target_provider, st.custom_providers)
st.new_model = normalize_model_for_provider(st.new_model, st.target_provider)
from hermes_cli.chat_catalog import is_known_non_chat_model
if is_known_non_chat_model(st.new_model):
return st.fail(
f"`{st.new_model}` is a generation model and cannot be used for chat. "
"Pick a chat model, or use image generation for image models.",
new_model=st.new_model, target_provider=st.target_provider,
provider_label=st.provider_label,
)
if st.target_provider.strip().lower() == "ollama":
headers = {} if st.suppress_ollama_headers else (st.validation_headers or _get_ollama_request_headers())
else:

View File

@@ -412,12 +412,14 @@ def _live_or_curated_ids(slug: str, curated: dict, *fallback_keys: str, merge_mo
``non_blocking`` (GUI read path) reads the disk cache only — a provider that is slow or down
contributes its curated list instead of stalling the whole picker (#114215)."""
from hermes_cli.models import _MODELS_DEV_PREFERRED, _merge_with_models_dev, cached_provider_model_ids
from hermes_cli.chat_catalog import without_generation_models
model_ids = cached_provider_model_ids(slug, non_blocking=non_blocking)
if not model_ids:
model_ids = _first_curated(curated, fallback_keys or (slug,))
if merge_models_dev and slug in _MODELS_DEV_PREFERRED:
model_ids = _merge_with_models_dev(slug, model_ids)
return model_ids
return without_generation_models(model_ids)
def _first_curated(curated: dict, keys) -> list:

View File

@@ -1596,6 +1596,12 @@ def _drop_delisted_opencode_models(normalized: str, rows: Optional[list[str]]) -
return rows
def _chat_catalog_rows(models):
"""Drop generation ids from a chat-catalog list, keeping its list subclass."""
from hermes_cli.chat_catalog import without_generation_models
return without_generation_models(models)
def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) -> list[str]:
"""Best known model catalog for a provider: per-provider live fetchers, then the generic profile
fetch, then the static list (merged with models.dev for ``_MODELS_DEV_PREFERRED`` providers)."""
@@ -1608,13 +1614,13 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
if fetcher is not None:
models = fetcher(normalized, force_refresh)
if models is not None:
return models
return _chat_catalog_rows(models)
try:
models = _profile_live_catalog(normalized)
except Exception:
models = None
if models is not None:
return models
return _chat_catalog_rows(models)
# Merge static curated list with live API results so models that the live endpoint omits (stale cache,
# partial rollout) still appear in the picker. Single providers (kimi, zai) use curated-first (commit
@@ -1628,10 +1634,10 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
# source is serving its authoritative catalog.
curated_static = (CuratedFallbackModels if fetcher is not None else list)(_PROVIDER_MODELS.get(normalized, []))
if normalized not in _MODELS_DEV_PREFERRED:
return _drop_delisted_opencode_models(normalized, curated_static)
return _chat_catalog_rows(_drop_delisted_opencode_models(normalized, curated_static))
# models.dev keeps listing retired Zen ids too: filter after the merge, not before.
merged = _drop_delisted_opencode_models(normalized, _merge_with_models_dev(normalized, curated_static))
return _xai_finalize_catalog(merged) if normalized in {"xai", "xai-oauth"} else merged
return _chat_catalog_rows(_xai_finalize_catalog(merged) if normalized in {"xai", "xai-oauth"} else merged)
# ---------------------------------------------------------------------------
@@ -1904,7 +1910,7 @@ def cached_provider_model_ids(
if tier is not None:
if tier == "stale":
_spawn_swr_refresh(normalized)
return list(entry["models"])
return _chat_catalog_rows(list(entry["models"]))
if non_blocking and not force_refresh:
# Read path: never touch the network in the caller's thread. A same-credentials row past the
@@ -1912,8 +1918,9 @@ def cached_provider_model_ids(
# warms the next open; a cold row returns [] and the caller falls back to its curated list.
_spawn_swr_refresh(normalized)
if _cache_entry_valid(entry, fp, allow_empty=is_ollama):
return [model for model in entry["models"]
if not _model_requires_account_discovery(normalized, model)]
return _chat_catalog_rows([
model for model in entry["models"]
if not _model_requires_account_discovery(normalized, model)])
return []
live = provider_model_ids(normalized, force_refresh=force_refresh)
@@ -1921,9 +1928,9 @@ def cached_provider_model_ids(
fresh = _live_result_entry(fp, live, entry, now)
if fresh is None:
# The live fetch degraded to the curated list; the account's real catalog is on disk.
return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)]
return _chat_catalog_rows([model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)])
_store_cache_entry(normalized, fresh, cache)
return list(live)
return _chat_catalog_rows(list(live))
if is_ollama:
if _ollama_native_probe_reachable():
@@ -1934,13 +1941,13 @@ def cached_provider_model_ids(
# the picker during a transient outage.
same_creds = isinstance(entry, dict) and entry.get("fp") == fp
if same_creds and isinstance(entry.get("models"), list) and entry["models"]:
return list(entry["models"])
return _chat_catalog_rows(list(entry["models"]))
return []
# Live returned nothing: a stale same-fingerprint entry beats an empty result — minus account-gated
# models, which only a successful discovery may advertise (the entry itself is untouched, so the
# next successful fetch restores them).
if _cache_entry_valid(entry, fp):
return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)]
return _chat_catalog_rows([model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)])
return []
@@ -2495,8 +2502,15 @@ def probe_api_models(
continue
if _neg_key is not None:
_probe_neg_cache.pop(_neg_key, None)
from hermes_cli.chat_catalog import note_catalog_item
probed = []
for item in data.get("data", []):
if isinstance(item, dict) and note_catalog_item(item):
continue
probed.append(item.get("id", ""))
return _probe_result(
[m.get("id", "") for m in data.get("data", [])], url, candidate_base.rstrip("/"),
probed, url, candidate_base.rstrip("/"),
alternate_base if alternate_base != candidate_base else normalized, is_fallback)
if _neg_key is not None and all_timed_out:
@@ -2713,7 +2727,8 @@ def cached_fetch_api_models(
from hermes_cli.model_switch_providers import _NativePickerModelList
def _catalog(entry):
return (_NativePickerModelList if entry.get("native_catalog") else list)(entry["models"])
rows = (_NativePickerModelList if entry.get("native_catalog") else list)(entry["models"])
return _chat_catalog_rows(rows)
def _entry(live, at=None):
return {**_cache_entry(fp, live, at), "native_catalog": isinstance(live, _NativePickerModelList)}
@@ -2726,7 +2741,7 @@ def cached_fetch_api_models(
normalized_url = str(base_url or "").strip().rstrip("/").lower()
if not normalized_url: # nothing to key the cache on
return None if cache_only else _live()
return None if cache_only else _chat_catalog_rows(_live())
# Key on URL AND credential fingerprint: N ``custom_providers`` rows can share one proxy URL
# with distinct keys (#106184). A URL-only key let the last probe overwrite its siblings'
@@ -2771,7 +2786,7 @@ def cached_fetch_api_models(
# (non-empty only: an empty native row is not worth resurrecting over the generic fallback).
if _cache_entry_valid(entry, fp):
return _catalog(entry)
return live
return _chat_catalog_rows(live)
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----

View File

@@ -409,7 +409,9 @@ class ProviderProfile:
with open_credentialed_url(req, timeout=timeout) as resp:
data = json.loads(resp.read().decode())
items = data if isinstance(data, list) else data.get("data", [])
return [m["id"] for m in items if isinstance(m, dict) and "id" in m]
from hermes_cli.chat_catalog import chat_catalog_ids
return chat_catalog_ids(items)
except Exception as exc:
logger.debug("fetch_models(%s): %s", self.name, exc)
return None

View File

@@ -0,0 +1,63 @@
"""A primary already known to be non-chat must not be restored on the next turn."""
from unittest.mock import MagicMock, patch
from run_agent import AIAgent
_TEST_KEY = "test-" + "key-12345678"
def _make_agent():
with (
patch("model_tools.get_tool_definitions", return_value=[]),
patch("model_tools.check_toolset_requirements", return_value={}),
patch("agent.process_bootstrap.OpenAI"),
patch("agent.context_compressor.get_model_context_length", return_value=200_000),
patch("agent.anthropic_adapter.build_anthropic_client", return_value=MagicMock()),
):
agent = AIAgent(
api_key=_TEST_KEY,
base_url="https://my-llm.example.com/v1",
provider="custom",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
fallback_model={"provider": "zai", "model": "glm-5.2"},
)
agent.client = MagicMock()
return agent
def _restore_after_fallback(primary_model):
agent = _make_agent()
agent._primary_runtime["model"] = primary_model
agent._primary_runtime["provider"] = "alibaba-token-plan"
fb_client = MagicMock()
fb_client.api_key = "fallback-" + "key-1234"
fb_client.base_url = "https://fallback.example.com/v1"
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(fb_client, None)):
assert agent._try_activate_fallback() is True
emitted = []
agent._emit_status = emitted.append
with patch("agent.process_bootstrap.OpenAI", return_value=MagicMock()):
restored = agent._restore_primary_runtime()
return agent, restored, emitted
def test_restore_skips_a_primary_already_known_to_be_non_chat():
agent, restored, emitted = _restore_after_fallback("wan2.7-image-pro")
assert restored is False
assert (agent.provider, agent.model, agent._fallback_activated) == ("zai", "glm-5.2", True)
assert not any("Primary model restored" in notice for notice in emitted)
other, other_restored, _emitted = _restore_after_fallback("acme/text-to-image")
assert other_restored is False
assert other.model == "glm-5.2"
def test_restore_still_returns_a_chat_primary():
_agent, restored, emitted = _restore_after_fallback("qwen3.7-plus")
assert restored is True
assert any("Primary model restored" in notice for notice in emitted)

View File

@@ -0,0 +1,122 @@
"""Chat catalogs must not offer generation models, and a direct switch must reject them.
Name shape and capability type are the catalog's own signals — not one model id.
"""
import json
from unittest.mock import MagicMock, patch
from providers.base import ProviderProfile
def _models_response(items):
response = MagicMock()
response.__enter__.return_value.read.return_value = json.dumps({"data": items}).encode()
return response
def test_live_catalog_drops_capability_typed_and_name_shaped_generation_models():
profile = ProviderProfile(
name="alibaba-token-plan",
base_url="https://token-plan.example.test/v1",
)
items = [
{"id": "wan2.7-image-pro"},
{"id": "acme/text-to-image"},
{"id": "gen-1", "capabilities": {"type": "image"}},
{"id": "clip-gen", "capabilities": {"type": "video-generation"}},
{"id": "router/flux-pro", "architecture": {"output_modalities": ["image"]}},
{"id": "qwen3.7-plus", "capabilities": {"type": "chat"}},
{"id": "qwen-vl-max"},
{"id": "missing-capabilities"},
{"id": "malformed-capabilities", "capabilities": "image"},
{"id": "unknown-kind", "capabilities": {"type": "future-kind"}},
]
with patch("hermes_cli.urllib_security.open_credentialed_url", return_value=_models_response(items)):
assert profile.fetch_models(api_key="test-key") == [
"qwen3.7-plus",
"qwen-vl-max",
"missing-capabilities",
"malformed-capabilities",
"unknown-kind",
]
def test_shared_chat_catalog_drops_cached_generation_ids():
"""A warm picker cache is the list model.options serves. Generation ids must not survive it."""
from hermes_cli.models import cached_provider_model_ids
entry = {
"fp": "fp",
"at": 10**10,
"models": ["qwen3.7-plus", "wan2.7-image-pro", "acme/text-to-video", "qwen-vl-max"],
}
with (
patch("hermes_cli.models._credential_fingerprint", return_value="fp"),
patch("hermes_cli.models._load_provider_models_cache", return_value={"alibaba-token-plan": entry}),
patch("hermes_cli.models._spawn_swr_refresh"),
):
assert cached_provider_model_ids("alibaba-token-plan") == ["qwen3.7-plus", "qwen-vl-max"]
def test_direct_switch_rejects_a_generation_model_and_keeps_a_vision_chat_model():
from hermes_cli.model_switch import _Switch, _validate_switch
accepted = {"accepted": True, "persist": True, "recognized": True, "message": ""}
def _state(model_id):
return _Switch(
raw_input=model_id,
current_provider="alibaba-token-plan",
current_model="qwen3.7-plus",
current_base_url="https://token-plan.example.test/v1",
current_api_key="",
is_global=False,
explicit_provider="",
user_providers=None,
custom_providers=None,
new_model=model_id,
target_provider="alibaba-token-plan",
provider_label="Alibaba Cloud (Token Plan)",
)
with patch("hermes_cli.models_validate.validate_requested_model", return_value=accepted):
rejected = _validate_switch(_state("vendor/image-pro"))
kept = _validate_switch(_state("qwen-vl-max"))
assert rejected is not None and rejected.success is False
assert "chat" in (rejected.error_message or "").lower()
assert kept is None
def test_capability_typed_id_is_rejected_once_the_catalog_has_seen_it():
"""An innocuous id is still non-chat when the catalog item says so."""
from hermes_cli.model_switch import _Switch, _validate_switch
profile = ProviderProfile(name="example", base_url="https://catalog.example.test/v1")
with patch(
"hermes_cli.urllib_security.open_credentialed_url",
return_value=_models_response([{"id": "gen-1", "capabilities": {"type": "image"}}]),
):
assert profile.fetch_models(api_key="test-key") == []
state = _Switch(
raw_input="gen-1",
current_provider="example",
current_model="qwen3.7-plus",
current_base_url="https://catalog.example.test/v1",
current_api_key="",
is_global=False,
explicit_provider="",
user_providers=None,
custom_providers=None,
new_model="gen-1",
target_provider="example",
provider_label="Example",
)
accepted = {"accepted": True, "persist": True, "recognized": True, "message": ""}
with patch("hermes_cli.models_validate.validate_requested_model", return_value=accepted):
rejected = _validate_switch(state)
assert rejected is not None and rejected.success is False