fix(models): keep generation models out of chat
Chat catalogs and the session switch treated image and video generation models as chat. Exclude them by the capability type and name shape the catalog already publishes, reject selecting one as the session model, and do not restore a primary already known to be non-chat. Desktop shows the fallback switch in the transcript.
This commit is contained in:
@@ -1263,10 +1263,13 @@ def restore_primary_runtime(agent) -> bool:
|
||||
primary_provider = str((rt or {}).get("provider") or "").strip().lower()
|
||||
primary_model = str((rt or {}).get("model") or "").strip()
|
||||
from agent.fallback_cooldown import _is_entitlement_rejected
|
||||
if primary_model and _is_entitlement_rejected(agent, primary_provider, primary_model):
|
||||
# The primary slug was rejected as unentitled for this account (#106475): restoring
|
||||
# here would announce a recovery that was never verified and re-fail every turn.
|
||||
# Stay on the fallback; the user sees the terminal entitlement error instead.
|
||||
from hermes_cli.chat_catalog import is_known_non_chat_model
|
||||
if primary_model and (
|
||||
_is_entitlement_rejected(agent, primary_provider, primary_model)
|
||||
or is_known_non_chat_model(primary_model)
|
||||
):
|
||||
# Unentitled (#106475) or already known non-chat: restoring would announce a recovery
|
||||
# that was never verified and re-fail every turn. Stay on the fallback.
|
||||
return False
|
||||
primary_runtime_base_url = str((rt or {}).get("base_url") or "")
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { handleStatusEvent } from './status'
|
||||
import type { GatewayEventContext } from './types'
|
||||
|
||||
function statusContext(text: string, kind = 'lifecycle') {
|
||||
const updateSessionState = vi.fn((_id: string, updater: (state: { messages: unknown[] }) => unknown) =>
|
||||
updater({ messages: [] })
|
||||
)
|
||||
const payload = { kind, text } as GatewayEventContext['payload']
|
||||
const ctx: GatewayEventContext = {
|
||||
deps: {
|
||||
compactedTurnRef: { current: new Set<string>() },
|
||||
failAssistantMessage: vi.fn(),
|
||||
flushQueuedDeltas: vi.fn(),
|
||||
hydrateFromStoredSession: vi.fn(),
|
||||
queryClient: { invalidateQueries: vi.fn() },
|
||||
sessionStateByRuntimeIdRef: { current: new Map() },
|
||||
updateSessionState
|
||||
} as unknown as GatewayEventContext['deps'],
|
||||
event: { payload, session_id: 'sess-1', type: 'status.update' },
|
||||
explicitSid: 'sess-1',
|
||||
fromActiveSource: () => true,
|
||||
isActiveEvent: true,
|
||||
occurredAt: 1_700_000_100,
|
||||
payload,
|
||||
scheduleConfigRefresh: vi.fn(),
|
||||
sessionId: 'sess-1'
|
||||
}
|
||||
|
||||
return { ctx, updateSessionState }
|
||||
}
|
||||
|
||||
function systemText(updateSessionState: ReturnType<typeof statusContext>['updateSessionState']): string {
|
||||
const updater = updateSessionState.mock.calls[0]?.[1]
|
||||
const next = updater?.({ messages: [] }) as { messages: Array<{ parts: Array<{ text?: string }>; role: string }> } | undefined
|
||||
|
||||
return next?.messages.find(message => message.role === 'system')?.parts[0]?.text ?? ''
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('desktop fallback switch', () => {
|
||||
it('shows a model fallback switch in the transcript', () => {
|
||||
const notice = '⚠️ Model fallback: wan2.7-image-pro via alibaba-token-plan unavailable (bad request); using qwen3.8-max via alibaba-token-plan.'
|
||||
const { ctx, updateSessionState } = statusContext(notice)
|
||||
|
||||
expect(handleStatusEvent(ctx)).toBe(true)
|
||||
expect(systemText(updateSessionState)).toContain('qwen3.8-max')
|
||||
expect(systemText(updateSessionState)).toMatch(/fallback/i)
|
||||
})
|
||||
|
||||
it('does not turn unrelated lifecycle status into a transcript line', () => {
|
||||
const { ctx, updateSessionState } = statusContext('⚠️ Rate limited — retrying')
|
||||
|
||||
expect(handleStatusEvent(ctx)).toBe(true)
|
||||
expect(updateSessionState).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -21,6 +21,15 @@ import { clearActiveSessionTodos } from '@/store/todos'
|
||||
|
||||
import type { GatewayEventContext } from './types'
|
||||
|
||||
/** Lifecycle text that announces the session left its selected model for a fallback. */
|
||||
export function isFallbackSwitchStatus(kind: string | undefined, text: string): boolean {
|
||||
if (kind === 'fallback') {
|
||||
return Boolean(text.trim())
|
||||
}
|
||||
|
||||
return /model fallback|provider fallback|switched to fallback|switching to fallback/i.test(text)
|
||||
}
|
||||
|
||||
/** status.update / review.summary / notification.show / notification.clear /
|
||||
* error — the status-and-notice tail of the dispatcher. */
|
||||
export function handleStatusEvent(ctx: GatewayEventContext): boolean {
|
||||
@@ -111,6 +120,25 @@ export function handleStatusEvent(ctx: GatewayEventContext): boolean {
|
||||
void refreshBackgroundProcesses(sessionId)
|
||||
} else if (sessionId && payload?.kind === 'goal') {
|
||||
applyGoalStatusText(sessionId, coerceGatewayText(payload?.text))
|
||||
} else if (sessionId && isFallbackSwitchStatus(payload?.kind, coerceGatewayText(payload?.text))) {
|
||||
// A provider/model switch is durable: the TUI paints it on the status rail, but Desktop
|
||||
// used to swallow every non-compaction status.update, so the reply came from a different
|
||||
// model with no indication.
|
||||
const text = coerceGatewayText(payload?.text).trim()
|
||||
|
||||
flushQueuedDeltas(sessionId)
|
||||
updateSessionState(sessionId, state => ({
|
||||
...state,
|
||||
messages: [
|
||||
...state.messages,
|
||||
{
|
||||
id: `fallback-switch-${occurredAt}`,
|
||||
role: 'system',
|
||||
parts: [textPart(text, occurredAt)],
|
||||
timestamp: occurredAt
|
||||
}
|
||||
]
|
||||
}))
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
136
hermes_cli/chat_catalog.py
Normal file
136
hermes_cli/chat_catalog.py
Normal file
@@ -0,0 +1,136 @@
|
||||
"""Admit only chat models into shared chat catalogs.
|
||||
|
||||
Generation models are identified by the capability type or name shape the
|
||||
catalog already publishes. Missing, malformed, or unknown metadata fails open.
|
||||
No single model id is special-cased.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# Structured catalog types that mean "this row is not a chat completion".
|
||||
_GENERATION_TYPES = frozenset({
|
||||
"image",
|
||||
"image_generation",
|
||||
"image-generation",
|
||||
"image-gen",
|
||||
"video",
|
||||
"video_generation",
|
||||
"video-generation",
|
||||
"video-gen",
|
||||
})
|
||||
|
||||
_GENERATION_OUTPUTS = frozenset({"image", "video"})
|
||||
_CHAT_OUTPUTS = frozenset({"text", "chat"})
|
||||
|
||||
# Separator-qualified generation tokens, plus the generation id tokens the
|
||||
# DeepInfra chat catalog already excludes. ``vision`` / ``vl`` are not tokens
|
||||
# here — those remain chat models.
|
||||
_GENERATION_ID_RE = re.compile(
|
||||
r"(?i)(?:^|[/_.-])(?:"
|
||||
r"image(?:[/_.-]gen(?:eration)?)?|"
|
||||
r"video(?:[/_.-]gen(?:eration)?)?|"
|
||||
r"text[/_.-]to[/_.-](?:image|video)|"
|
||||
r"stable-diffusion|sdxl|flux"
|
||||
r")(?=$|[/_.-])"
|
||||
)
|
||||
|
||||
# Ids a live catalog item already proved are generation-only, even when the
|
||||
# id itself has no generation token. Process-local: the proof came from this
|
||||
# process's catalog parse.
|
||||
_seen_generation_ids: set[str] = set()
|
||||
|
||||
|
||||
def note_generation_model(model_id: Any) -> None:
|
||||
key = str(model_id or "").strip().lower()
|
||||
if key:
|
||||
_seen_generation_ids.add(key)
|
||||
|
||||
|
||||
def model_id_is_generation(model_id: Any) -> bool:
|
||||
"""Whether *model_id* names an image/video generation surface."""
|
||||
return bool(_GENERATION_ID_RE.search(str(model_id or "").strip()))
|
||||
|
||||
|
||||
def _tag_list_is_generation(tags: Any) -> bool:
|
||||
if not isinstance(tags, list):
|
||||
return False
|
||||
return any(str(tag or "").strip().lower() in _GENERATION_TYPES for tag in tags)
|
||||
|
||||
|
||||
def _output_modalities_are_generation(outputs: Any) -> bool:
|
||||
if not isinstance(outputs, list) or not outputs:
|
||||
return False
|
||||
kinds = {str(item or "").strip().lower() for item in outputs if str(item or "").strip()}
|
||||
return bool(kinds & _GENERATION_OUTPUTS) and not bool(kinds & _CHAT_OUTPUTS)
|
||||
|
||||
|
||||
def catalog_item_is_generation(item: Any) -> bool:
|
||||
"""Fail open unless this catalog row's own type or name proves it is generation-only."""
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
if model_id_is_generation(item.get("id")):
|
||||
return True
|
||||
capabilities = item.get("capabilities")
|
||||
if isinstance(capabilities, dict):
|
||||
model_type = str(capabilities.get("type") or "").strip().lower()
|
||||
if model_type in _GENERATION_TYPES:
|
||||
return True
|
||||
if _tag_list_is_generation(capabilities.get("tags")):
|
||||
return True
|
||||
top_type = item.get("type")
|
||||
if isinstance(top_type, str) and top_type.strip().lower() in _GENERATION_TYPES:
|
||||
return True
|
||||
if _tag_list_is_generation(item.get("tags")):
|
||||
return True
|
||||
architecture = item.get("architecture")
|
||||
if isinstance(architecture, dict) and _output_modalities_are_generation(architecture.get("output_modalities")):
|
||||
return True
|
||||
metadata = item.get("metadata")
|
||||
if isinstance(metadata, dict) and _tag_list_is_generation(metadata.get("tags")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def note_catalog_item(item: Any) -> bool:
|
||||
"""Remember a generation row and report whether it must leave the chat catalog."""
|
||||
if not catalog_item_is_generation(item):
|
||||
return False
|
||||
if isinstance(item, dict):
|
||||
note_generation_model(item.get("id"))
|
||||
return True
|
||||
|
||||
|
||||
def is_known_non_chat_model(model_id: Any) -> bool:
|
||||
"""True when the id's name shape, or a catalog item already parsed, says it is not chat."""
|
||||
mid = str(model_id or "").strip()
|
||||
return model_id_is_generation(mid) or mid.lower() in _seen_generation_ids
|
||||
|
||||
|
||||
def without_generation_models(models):
|
||||
"""Drop generation ids, preserving a list subclass (curated-fallback marker)."""
|
||||
if not models:
|
||||
return models
|
||||
kept = [model for model in models if not is_known_non_chat_model(model)]
|
||||
if len(kept) == len(list(models)):
|
||||
return models
|
||||
try:
|
||||
return type(models)(kept)
|
||||
except TypeError:
|
||||
return kept
|
||||
|
||||
|
||||
def chat_catalog_ids(items: Any) -> list[str]:
|
||||
"""Model ids from a ``/models`` payload, minus generation rows."""
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
ids: list[str] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or "id" not in item:
|
||||
continue
|
||||
if note_catalog_item(item):
|
||||
continue
|
||||
ids.append(item["id"])
|
||||
return ids
|
||||
@@ -1610,6 +1610,15 @@ def _validate_switch(st: _Switch) -> Optional[ModelSwitchResult]:
|
||||
st.new_model = _resolve_named_custom_model_id(st.new_model, st.target_provider, st.custom_providers)
|
||||
st.new_model = normalize_model_for_provider(st.new_model, st.target_provider)
|
||||
|
||||
from hermes_cli.chat_catalog import is_known_non_chat_model
|
||||
if is_known_non_chat_model(st.new_model):
|
||||
return st.fail(
|
||||
f"`{st.new_model}` is a generation model and cannot be used for chat. "
|
||||
"Pick a chat model, or use image generation for image models.",
|
||||
new_model=st.new_model, target_provider=st.target_provider,
|
||||
provider_label=st.provider_label,
|
||||
)
|
||||
|
||||
if st.target_provider.strip().lower() == "ollama":
|
||||
headers = {} if st.suppress_ollama_headers else (st.validation_headers or _get_ollama_request_headers())
|
||||
else:
|
||||
|
||||
@@ -412,12 +412,14 @@ def _live_or_curated_ids(slug: str, curated: dict, *fallback_keys: str, merge_mo
|
||||
``non_blocking`` (GUI read path) reads the disk cache only — a provider that is slow or down
|
||||
contributes its curated list instead of stalling the whole picker (#114215)."""
|
||||
from hermes_cli.models import _MODELS_DEV_PREFERRED, _merge_with_models_dev, cached_provider_model_ids
|
||||
from hermes_cli.chat_catalog import without_generation_models
|
||||
|
||||
model_ids = cached_provider_model_ids(slug, non_blocking=non_blocking)
|
||||
if not model_ids:
|
||||
model_ids = _first_curated(curated, fallback_keys or (slug,))
|
||||
if merge_models_dev and slug in _MODELS_DEV_PREFERRED:
|
||||
model_ids = _merge_with_models_dev(slug, model_ids)
|
||||
return model_ids
|
||||
return without_generation_models(model_ids)
|
||||
|
||||
|
||||
def _first_curated(curated: dict, keys) -> list:
|
||||
|
||||
@@ -1596,6 +1596,12 @@ def _drop_delisted_opencode_models(normalized: str, rows: Optional[list[str]]) -
|
||||
return rows
|
||||
|
||||
|
||||
def _chat_catalog_rows(models):
|
||||
"""Drop generation ids from a chat-catalog list, keeping its list subclass."""
|
||||
from hermes_cli.chat_catalog import without_generation_models
|
||||
return without_generation_models(models)
|
||||
|
||||
|
||||
def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) -> list[str]:
|
||||
"""Best known model catalog for a provider: per-provider live fetchers, then the generic profile
|
||||
fetch, then the static list (merged with models.dev for ``_MODELS_DEV_PREFERRED`` providers)."""
|
||||
@@ -1608,13 +1614,13 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
|
||||
if fetcher is not None:
|
||||
models = fetcher(normalized, force_refresh)
|
||||
if models is not None:
|
||||
return models
|
||||
return _chat_catalog_rows(models)
|
||||
try:
|
||||
models = _profile_live_catalog(normalized)
|
||||
except Exception:
|
||||
models = None
|
||||
if models is not None:
|
||||
return models
|
||||
return _chat_catalog_rows(models)
|
||||
|
||||
# Merge static curated list with live API results so models that the live endpoint omits (stale cache,
|
||||
# partial rollout) still appear in the picker. Single providers (kimi, zai) use curated-first (commit
|
||||
@@ -1628,10 +1634,10 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
|
||||
# source is serving its authoritative catalog.
|
||||
curated_static = (CuratedFallbackModels if fetcher is not None else list)(_PROVIDER_MODELS.get(normalized, []))
|
||||
if normalized not in _MODELS_DEV_PREFERRED:
|
||||
return _drop_delisted_opencode_models(normalized, curated_static)
|
||||
return _chat_catalog_rows(_drop_delisted_opencode_models(normalized, curated_static))
|
||||
# models.dev keeps listing retired Zen ids too: filter after the merge, not before.
|
||||
merged = _drop_delisted_opencode_models(normalized, _merge_with_models_dev(normalized, curated_static))
|
||||
return _xai_finalize_catalog(merged) if normalized in {"xai", "xai-oauth"} else merged
|
||||
return _chat_catalog_rows(_xai_finalize_catalog(merged) if normalized in {"xai", "xai-oauth"} else merged)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1904,7 +1910,7 @@ def cached_provider_model_ids(
|
||||
if tier is not None:
|
||||
if tier == "stale":
|
||||
_spawn_swr_refresh(normalized)
|
||||
return list(entry["models"])
|
||||
return _chat_catalog_rows(list(entry["models"]))
|
||||
|
||||
if non_blocking and not force_refresh:
|
||||
# Read path: never touch the network in the caller's thread. A same-credentials row past the
|
||||
@@ -1912,8 +1918,9 @@ def cached_provider_model_ids(
|
||||
# warms the next open; a cold row returns [] and the caller falls back to its curated list.
|
||||
_spawn_swr_refresh(normalized)
|
||||
if _cache_entry_valid(entry, fp, allow_empty=is_ollama):
|
||||
return [model for model in entry["models"]
|
||||
if not _model_requires_account_discovery(normalized, model)]
|
||||
return _chat_catalog_rows([
|
||||
model for model in entry["models"]
|
||||
if not _model_requires_account_discovery(normalized, model)])
|
||||
return []
|
||||
|
||||
live = provider_model_ids(normalized, force_refresh=force_refresh)
|
||||
@@ -1921,9 +1928,9 @@ def cached_provider_model_ids(
|
||||
fresh = _live_result_entry(fp, live, entry, now)
|
||||
if fresh is None:
|
||||
# The live fetch degraded to the curated list; the account's real catalog is on disk.
|
||||
return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)]
|
||||
return _chat_catalog_rows([model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)])
|
||||
_store_cache_entry(normalized, fresh, cache)
|
||||
return list(live)
|
||||
return _chat_catalog_rows(list(live))
|
||||
|
||||
if is_ollama:
|
||||
if _ollama_native_probe_reachable():
|
||||
@@ -1934,13 +1941,13 @@ def cached_provider_model_ids(
|
||||
# the picker during a transient outage.
|
||||
same_creds = isinstance(entry, dict) and entry.get("fp") == fp
|
||||
if same_creds and isinstance(entry.get("models"), list) and entry["models"]:
|
||||
return list(entry["models"])
|
||||
return _chat_catalog_rows(list(entry["models"]))
|
||||
return []
|
||||
# Live returned nothing: a stale same-fingerprint entry beats an empty result — minus account-gated
|
||||
# models, which only a successful discovery may advertise (the entry itself is untouched, so the
|
||||
# next successful fetch restores them).
|
||||
if _cache_entry_valid(entry, fp):
|
||||
return [model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)]
|
||||
return _chat_catalog_rows([model for model in entry["models"] if not _model_requires_account_discovery(normalized, model)])
|
||||
return []
|
||||
|
||||
|
||||
@@ -2495,8 +2502,15 @@ def probe_api_models(
|
||||
continue
|
||||
if _neg_key is not None:
|
||||
_probe_neg_cache.pop(_neg_key, None)
|
||||
from hermes_cli.chat_catalog import note_catalog_item
|
||||
|
||||
probed = []
|
||||
for item in data.get("data", []):
|
||||
if isinstance(item, dict) and note_catalog_item(item):
|
||||
continue
|
||||
probed.append(item.get("id", ""))
|
||||
return _probe_result(
|
||||
[m.get("id", "") for m in data.get("data", [])], url, candidate_base.rstrip("/"),
|
||||
probed, url, candidate_base.rstrip("/"),
|
||||
alternate_base if alternate_base != candidate_base else normalized, is_fallback)
|
||||
|
||||
if _neg_key is not None and all_timed_out:
|
||||
@@ -2713,7 +2727,8 @@ def cached_fetch_api_models(
|
||||
from hermes_cli.model_switch_providers import _NativePickerModelList
|
||||
|
||||
def _catalog(entry):
|
||||
return (_NativePickerModelList if entry.get("native_catalog") else list)(entry["models"])
|
||||
rows = (_NativePickerModelList if entry.get("native_catalog") else list)(entry["models"])
|
||||
return _chat_catalog_rows(rows)
|
||||
|
||||
def _entry(live, at=None):
|
||||
return {**_cache_entry(fp, live, at), "native_catalog": isinstance(live, _NativePickerModelList)}
|
||||
@@ -2726,7 +2741,7 @@ def cached_fetch_api_models(
|
||||
|
||||
normalized_url = str(base_url or "").strip().rstrip("/").lower()
|
||||
if not normalized_url: # nothing to key the cache on
|
||||
return None if cache_only else _live()
|
||||
return None if cache_only else _chat_catalog_rows(_live())
|
||||
|
||||
# Key on URL AND credential fingerprint: N ``custom_providers`` rows can share one proxy URL
|
||||
# with distinct keys (#106184). A URL-only key let the last probe overwrite its siblings'
|
||||
@@ -2771,7 +2786,7 @@ def cached_fetch_api_models(
|
||||
# (non-empty only: an empty native row is not worth resurrecting over the generic fallback).
|
||||
if _cache_entry_valid(entry, fp):
|
||||
return _catalog(entry)
|
||||
return live
|
||||
return _chat_catalog_rows(live)
|
||||
|
||||
|
||||
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----
|
||||
|
||||
@@ -409,7 +409,9 @@ class ProviderProfile:
|
||||
with open_credentialed_url(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
items = data if isinstance(data, list) else data.get("data", [])
|
||||
return [m["id"] for m in items if isinstance(m, dict) and "id" in m]
|
||||
from hermes_cli.chat_catalog import chat_catalog_ids
|
||||
|
||||
return chat_catalog_ids(items)
|
||||
except Exception as exc:
|
||||
logger.debug("fetch_models(%s): %s", self.name, exc)
|
||||
return None
|
||||
|
||||
63
tests/agent/test_non_chat_primary_restore.py
Normal file
63
tests/agent/test_non_chat_primary_restore.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""A primary already known to be non-chat must not be restored on the next turn."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from run_agent import AIAgent
|
||||
|
||||
_TEST_KEY = "test-" + "key-12345678"
|
||||
|
||||
|
||||
def _make_agent():
|
||||
with (
|
||||
patch("model_tools.get_tool_definitions", return_value=[]),
|
||||
patch("model_tools.check_toolset_requirements", return_value={}),
|
||||
patch("agent.process_bootstrap.OpenAI"),
|
||||
patch("agent.context_compressor.get_model_context_length", return_value=200_000),
|
||||
patch("agent.anthropic_adapter.build_anthropic_client", return_value=MagicMock()),
|
||||
):
|
||||
agent = AIAgent(
|
||||
api_key=_TEST_KEY,
|
||||
base_url="https://my-llm.example.com/v1",
|
||||
provider="custom",
|
||||
quiet_mode=True,
|
||||
skip_context_files=True,
|
||||
skip_memory=True,
|
||||
fallback_model={"provider": "zai", "model": "glm-5.2"},
|
||||
)
|
||||
agent.client = MagicMock()
|
||||
return agent
|
||||
|
||||
|
||||
def _restore_after_fallback(primary_model):
|
||||
agent = _make_agent()
|
||||
agent._primary_runtime["model"] = primary_model
|
||||
agent._primary_runtime["provider"] = "alibaba-token-plan"
|
||||
fb_client = MagicMock()
|
||||
fb_client.api_key = "fallback-" + "key-1234"
|
||||
fb_client.base_url = "https://fallback.example.com/v1"
|
||||
with patch("agent.auxiliary_client.resolve_provider_client", return_value=(fb_client, None)):
|
||||
assert agent._try_activate_fallback() is True
|
||||
emitted = []
|
||||
agent._emit_status = emitted.append
|
||||
with patch("agent.process_bootstrap.OpenAI", return_value=MagicMock()):
|
||||
restored = agent._restore_primary_runtime()
|
||||
return agent, restored, emitted
|
||||
|
||||
|
||||
def test_restore_skips_a_primary_already_known_to_be_non_chat():
|
||||
agent, restored, emitted = _restore_after_fallback("wan2.7-image-pro")
|
||||
|
||||
assert restored is False
|
||||
assert (agent.provider, agent.model, agent._fallback_activated) == ("zai", "glm-5.2", True)
|
||||
assert not any("Primary model restored" in notice for notice in emitted)
|
||||
|
||||
other, other_restored, _emitted = _restore_after_fallback("acme/text-to-image")
|
||||
assert other_restored is False
|
||||
assert other.model == "glm-5.2"
|
||||
|
||||
|
||||
def test_restore_still_returns_a_chat_primary():
|
||||
_agent, restored, emitted = _restore_after_fallback("qwen3.7-plus")
|
||||
|
||||
assert restored is True
|
||||
assert any("Primary model restored" in notice for notice in emitted)
|
||||
122
tests/hermes_cli/test_chat_catalog_excludes_generation_models.py
Normal file
122
tests/hermes_cli/test_chat_catalog_excludes_generation_models.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""Chat catalogs must not offer generation models, and a direct switch must reject them.
|
||||
|
||||
Name shape and capability type are the catalog's own signals — not one model id.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from providers.base import ProviderProfile
|
||||
|
||||
|
||||
def _models_response(items):
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = json.dumps({"data": items}).encode()
|
||||
return response
|
||||
|
||||
|
||||
def test_live_catalog_drops_capability_typed_and_name_shaped_generation_models():
|
||||
profile = ProviderProfile(
|
||||
name="alibaba-token-plan",
|
||||
base_url="https://token-plan.example.test/v1",
|
||||
)
|
||||
items = [
|
||||
{"id": "wan2.7-image-pro"},
|
||||
{"id": "acme/text-to-image"},
|
||||
{"id": "gen-1", "capabilities": {"type": "image"}},
|
||||
{"id": "clip-gen", "capabilities": {"type": "video-generation"}},
|
||||
{"id": "router/flux-pro", "architecture": {"output_modalities": ["image"]}},
|
||||
{"id": "qwen3.7-plus", "capabilities": {"type": "chat"}},
|
||||
{"id": "qwen-vl-max"},
|
||||
{"id": "missing-capabilities"},
|
||||
{"id": "malformed-capabilities", "capabilities": "image"},
|
||||
{"id": "unknown-kind", "capabilities": {"type": "future-kind"}},
|
||||
]
|
||||
|
||||
with patch("hermes_cli.urllib_security.open_credentialed_url", return_value=_models_response(items)):
|
||||
assert profile.fetch_models(api_key="test-key") == [
|
||||
"qwen3.7-plus",
|
||||
"qwen-vl-max",
|
||||
"missing-capabilities",
|
||||
"malformed-capabilities",
|
||||
"unknown-kind",
|
||||
]
|
||||
|
||||
|
||||
def test_shared_chat_catalog_drops_cached_generation_ids():
|
||||
"""A warm picker cache is the list model.options serves. Generation ids must not survive it."""
|
||||
from hermes_cli.models import cached_provider_model_ids
|
||||
|
||||
entry = {
|
||||
"fp": "fp",
|
||||
"at": 10**10,
|
||||
"models": ["qwen3.7-plus", "wan2.7-image-pro", "acme/text-to-video", "qwen-vl-max"],
|
||||
}
|
||||
with (
|
||||
patch("hermes_cli.models._credential_fingerprint", return_value="fp"),
|
||||
patch("hermes_cli.models._load_provider_models_cache", return_value={"alibaba-token-plan": entry}),
|
||||
patch("hermes_cli.models._spawn_swr_refresh"),
|
||||
):
|
||||
assert cached_provider_model_ids("alibaba-token-plan") == ["qwen3.7-plus", "qwen-vl-max"]
|
||||
|
||||
|
||||
def test_direct_switch_rejects_a_generation_model_and_keeps_a_vision_chat_model():
|
||||
from hermes_cli.model_switch import _Switch, _validate_switch
|
||||
|
||||
accepted = {"accepted": True, "persist": True, "recognized": True, "message": ""}
|
||||
|
||||
def _state(model_id):
|
||||
return _Switch(
|
||||
raw_input=model_id,
|
||||
current_provider="alibaba-token-plan",
|
||||
current_model="qwen3.7-plus",
|
||||
current_base_url="https://token-plan.example.test/v1",
|
||||
current_api_key="",
|
||||
is_global=False,
|
||||
explicit_provider="",
|
||||
user_providers=None,
|
||||
custom_providers=None,
|
||||
new_model=model_id,
|
||||
target_provider="alibaba-token-plan",
|
||||
provider_label="Alibaba Cloud (Token Plan)",
|
||||
)
|
||||
|
||||
with patch("hermes_cli.models_validate.validate_requested_model", return_value=accepted):
|
||||
rejected = _validate_switch(_state("vendor/image-pro"))
|
||||
kept = _validate_switch(_state("qwen-vl-max"))
|
||||
|
||||
assert rejected is not None and rejected.success is False
|
||||
assert "chat" in (rejected.error_message or "").lower()
|
||||
assert kept is None
|
||||
|
||||
|
||||
def test_capability_typed_id_is_rejected_once_the_catalog_has_seen_it():
|
||||
"""An innocuous id is still non-chat when the catalog item says so."""
|
||||
from hermes_cli.model_switch import _Switch, _validate_switch
|
||||
|
||||
profile = ProviderProfile(name="example", base_url="https://catalog.example.test/v1")
|
||||
with patch(
|
||||
"hermes_cli.urllib_security.open_credentialed_url",
|
||||
return_value=_models_response([{"id": "gen-1", "capabilities": {"type": "image"}}]),
|
||||
):
|
||||
assert profile.fetch_models(api_key="test-key") == []
|
||||
|
||||
state = _Switch(
|
||||
raw_input="gen-1",
|
||||
current_provider="example",
|
||||
current_model="qwen3.7-plus",
|
||||
current_base_url="https://catalog.example.test/v1",
|
||||
current_api_key="",
|
||||
is_global=False,
|
||||
explicit_provider="",
|
||||
user_providers=None,
|
||||
custom_providers=None,
|
||||
new_model="gen-1",
|
||||
target_provider="example",
|
||||
provider_label="Example",
|
||||
)
|
||||
accepted = {"accepted": True, "persist": True, "recognized": True, "message": ""}
|
||||
with patch("hermes_cli.models_validate.validate_requested_model", return_value=accepted):
|
||||
rejected = _validate_switch(state)
|
||||
|
||||
assert rejected is not None and rejected.success is False
|
||||
Reference in New Issue
Block a user