diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 6006c28b13..c137bdd6a9 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -2577,6 +2577,7 @@ def get_model_context_length( Resolution order: 0. Explicit config override (model.context_length or custom_providers per-model) + 0b. model_overrides config (per-provider+model context_window override) 0c. Endpoint-scoped metadata for models validated on one multiplexed endpoint 1. Persistent cache (previously discovered via probing). Nous URLs, LM Studio, and Codex OAuth bypass the cache here so their provider @@ -2638,7 +2639,21 @@ def get_model_context_length( logger.debug("MoA aggregator context-length resolution failed", exc_info=True) # Fall through to the generic default if aggregator resolution failed. - # 0b. custom_providers per-model override — check before any probe. + # 0b. model_overrides config — per-provider+model context_window override. + # This is the supported self-unblock path for models with wrong or missing + # context in models.dev (#84482) and for custom/local models not in the + # catalog (#8731). Checked before custom_providers (step 0c) and before any + # network probe so it never blocks. + if provider and model: + try: + from agent.models_dev import _override_context_window + mo_ctx = _override_context_window(provider, model) + if mo_ctx is not None and mo_ctx > 0: + return mo_ctx + except Exception: + pass # fall through to other resolution paths + + # 0c. custom_providers per-model override — check before any probe. # This closes the gap where /model switch and display paths used to fall # back to 128K despite the user having a per-model context_length set. # See #15779. diff --git a/agent/models_dev.py b/agent/models_dev.py index 52c4100741..bcab594546 100644 --- a/agent/models_dev.py +++ b/agent/models_dev.py @@ -499,7 +499,17 @@ def lookup_models_dev_context(provider: str, model: str) -> Optional[int]: Returns the context window in tokens, or None if not found. Handles case-insensitive matching and filters out context=0 entries. + + A ``model_overrides`` config entry for this provider+model (or its + ``_default`` fallback) wins over the catalog value — this is the + supported self-unblock path for models with wrong or missing context + in models.dev (#84482). """ + # Config override — checked before catalog so it always wins. + override_ctx = _override_context_window(provider, model) + if override_ctx is not None: + return override_ctx + mdev_provider_id = PROVIDER_TO_MODELS_DEV.get(provider) if not mdev_provider_id: return None @@ -586,6 +596,103 @@ class ModelCapabilities: model_family: str = "" +# --------------------------------------------------------------------------- # +# Per-model metadata overrides (config.yaml → model_overrides) # +# --------------------------------------------------------------------------- # +# +# Resolution order for every query function below: +# 1. ``model_overrides..`` — explicit per-provider+model +# 2. ``model_overrides.._default`` — per-provider default +# 3. ``model_overrides._default`` — global default +# 4. models.dev / OpenRouter / hardcoded — normal catalog resolution +# +# An override may set any subset of fields; unspecified fields fall through to +# the catalog value. For a model id NOT in the catalog, the override is the +# only source of metadata — this is the supported self-unblock path for new +# or custom models (#84482, #8731). + +_OVERRIDE_CACHE: Optional[Dict[str, Any]] = None +_OVERRIDE_CACHE_CFG_HASH: int = 0 + + +def _load_model_overrides() -> Dict[str, Any]: + """Load and cache the ``model_overrides`` config section. + + Caches by ``id(cfg)`` so a config reload (new dict identity) invalidates + automatically. Returns empty dict on any failure. + """ + global _OVERRIDE_CACHE, _OVERRIDE_CACHE_CFG_HASH + try: + from hermes_cli.config import cfg_get, load_config_readonly + cfg = load_config_readonly() + cfg_id = id(cfg) + if cfg_id == _OVERRIDE_CACHE_CFG_HASH and _OVERRIDE_CACHE is not None: + return _OVERRIDE_CACHE + raw = cfg_get(cfg, "model_overrides", default={}) + overrides = raw if isinstance(raw, dict) else {} + _OVERRIDE_CACHE = overrides + _OVERRIDE_CACHE_CFG_HASH = cfg_id + return overrides + except Exception: + return {} + + +def _resolve_model_override( + provider: str, model: str +) -> Optional[Dict[str, Any]]: + """Resolve the override dict for a provider+model, or None. + + Checks per-provider+model, then per-provider ``_default``, then global + ``_default``. Returns the first match (which may be partially populated — + callers only read the keys they care about). + """ + overrides = _load_model_overrides() + if not overrides: + return None + + provider_key = (provider or "").strip() + model_key = (model or "").strip() + if not provider_key and not model_key: + return None + + # 1. Per-provider+model + provider_section = overrides.get(provider_key) + if isinstance(provider_section, dict) and model_key: + model_section = provider_section.get(model_key) + if isinstance(model_section, dict): + return model_section + + # 2. Per-provider _default + if isinstance(provider_section, dict): + default = provider_section.get("_default") + if isinstance(default, dict): + return default + + # 3. Global _default + global_default = overrides.get("_default") + if isinstance(global_default, dict): + return global_default + + return None + + +def _override_context_window( + provider: str, model: str +) -> Optional[int]: + """Return the overridden context_window, or None.""" + ov = _resolve_model_override(provider, model) + if ov is None: + return None + raw = ov.get("context_window") + if raw is None: + return None + try: + ctx = int(raw) + return ctx if ctx > 0 else None + except (TypeError, ValueError): + return None + + def _get_provider_models(provider: str) -> Optional[Dict[str, Any]]: """Resolve a Hermes provider ID to its models dict from models.dev. @@ -629,6 +736,15 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit Uses the existing fetch_models_dev() and PROVIDER_TO_MODELS_DEV mapping. Returns None if model not found. + ``model_overrides`` config entries (per-provider+model, per-provider + ``_default``, or global ``_default``) win over catalog values. For a + model id NOT in the catalog, the override is the only source of + metadata — this is the supported self-unblock path for custom/local + models (#8731) and for models with wrong context in models.dev + (#84482). An override may set any subset of fields; unspecified fields + fall through to the catalog value (or sensible defaults when the model + is absent from the catalog entirely). + Extracts from model entry fields: - reasoning (bool) → supports_reasoning - tool_call (bool) → supports_tools @@ -637,42 +753,81 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit - limit.output (int) → max_output_tokens - family (str) → model_family """ + # Check config override first — it may fully replace the catalog entry + # or patch specific fields. For unknown models (not in catalog), the + # override is the sole source of metadata. + override = _resolve_model_override(provider, model) + models = _get_provider_models(provider) - if models is None: + entry = _find_model_entry(models, model) if models is not None else None + + # If no catalog entry and no override, we can't resolve capabilities. + if entry is None and override is None: return None - entry = _find_model_entry(models, model) - if entry is None: - return None + # Start from catalog entry (if found), else use defaults. + if entry is not None: + supports_tools = bool(entry.get("tool_call", False)) + # Vision: prefer explicit `modalities.input` when models.dev provides it. + # The older `attachment` flag can be stale or too broad for image routing; + # fall back to it only when the input modalities are absent/invalid. + input_mods = entry.get("modalities", {}) + if isinstance(input_mods, dict): + input_mods = input_mods.get("input") + else: + input_mods = None + if isinstance(input_mods, list): + supports_vision = "image" in input_mods + else: + supports_vision = bool(entry.get("attachment", False)) + supports_reasoning = bool(entry.get("reasoning", False)) - # Extract capability flags (default to False if missing) - supports_tools = bool(entry.get("tool_call", False)) - # Vision: prefer explicit `modalities.input` when models.dev provides it. - # The older `attachment` flag can be stale or too broad for image routing; - # fall back to it only when the input modalities are absent/invalid. - input_mods = entry.get("modalities", {}) - if isinstance(input_mods, dict): - input_mods = input_mods.get("input") + limit = entry.get("limit", {}) + if not isinstance(limit, dict): + limit = {} + + ctx = limit.get("context") + context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000 + + out = limit.get("output") + max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192 + + model_family = entry.get("family", "") or "" else: - input_mods = None - if isinstance(input_mods, list): - supports_vision = "image" in input_mods - else: - supports_vision = bool(entry.get("attachment", False)) - supports_reasoning = bool(entry.get("reasoning", False)) + # Unknown model — derive sensible defaults. The override will + # patch whichever fields it specifies; the rest stay at defaults + # that are safe for agentic use (tools on, vision/reasoning off). + supports_tools = True + supports_vision = False + supports_reasoning = False + context_window = 200000 + max_output_tokens = 8192 + model_family = "" - # Extract limits - limit = entry.get("limit", {}) - if not isinstance(limit, dict): - limit = {} - - ctx = limit.get("context") - context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000 - - out = limit.get("output") - max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192 - - model_family = entry.get("family", "") or "" + # Apply override patches (each field is optional in the override dict). + if override is not None: + if "supports_tools" in override: + supports_tools = bool(override["supports_tools"]) + if "supports_vision" in override: + supports_vision = bool(override["supports_vision"]) + if "supports_reasoning" in override: + supports_reasoning = bool(override["supports_reasoning"]) + if "context_window" in override: + try: + ctx_ov = int(override["context_window"]) + if ctx_ov > 0: + context_window = ctx_ov + except (TypeError, ValueError): + pass + if "max_output_tokens" in override: + try: + out_ov = int(override["max_output_tokens"]) + if out_ov > 0: + max_output_tokens = out_ov + except (TypeError, ValueError): + pass + if "model_family" in override: + model_family = str(override["model_family"] or "") return ModelCapabilities( supports_tools=supports_tools, @@ -884,27 +1039,51 @@ def get_model_info( Accepts Hermes or models.dev provider ID. Tries exact match then case-insensitive fallback. Returns None if not found. + + ``model_overrides`` config entries (per-provider+model, per-provider + ``_default``, or global ``_default``) patch the catalog entry's fields + when present. For a model id NOT in the catalog, the override is the + sole source of metadata — this is the supported self-unblock path + for custom/local models (#8731) and for models with wrong context + in models.dev (#84482). """ + override = _resolve_model_override(provider_id, model_id) + mdev_id = PROVIDER_TO_MODELS_DEV.get(provider_id, provider_id) data = fetch_models_dev() pdata = data.get(mdev_id) if not isinstance(pdata, dict): + # No catalog data — return from override alone if we have one. + if override is not None: + return _parse_model_info(model_id, override, mdev_id) return None models = pdata.get("models", {}) if not isinstance(models, dict): + if override is not None: + return _parse_model_info(model_id, override, mdev_id) return None # Exact match raw = models.get(model_id) if isinstance(raw, dict): + if override is not None: + merged = {**raw, **override} + return _parse_model_info(model_id, merged, mdev_id) return _parse_model_info(model_id, raw, mdev_id) # Case-insensitive fallback model_lower = model_id.lower() for mid, mdata in models.items(): if mid.lower() == model_lower and isinstance(mdata, dict): + if override is not None: + merged = {**mdata, **override} + return _parse_model_info(mid, merged, mdev_id) return _parse_model_info(mid, mdata, mdev_id) + # Model not in catalog — return from override alone if we have one. + if override is not None: + return _parse_model_info(model_id, override, mdev_id) + return None diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 584b10b36a..4094873737 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -2539,6 +2539,37 @@ DEFAULT_CONFIG = { "providers": {}, }, + # Per-model metadata overrides — manually declare context_window, + # max_output_tokens, capabilities, or cost for any provider+model. + # Overrides win over models.dev, OpenRouter, and hardcoded defaults. + # + # Two scopes: + # 1. Per-provider+model: model_overrides.. + # 2. Per-provider default: model_overrides.._default + # 3. Global default: model_overrides._default + # + # An unknown model id (not in models.dev) inherits base metadata from + # its family/dated-snapshot entry before patching, so overriding a + # model the catalog doesn't know yet is the supported self-unblock + # path (#84482). + # + # Example: + # model_overrides: + # upstage: + # solar-pro4: + # context_window: 524288 + # syn-pro: + # context_window: 65536 + # custom:my-local-vllm: + # my-llava-model: + # context_window: 8192 + # supports_vision: true + # supports_reasoning: false + # supports_tools: true + # _default: + # context_window: 128000 + "model_overrides": {}, + # Network settings — workarounds for connectivity issues. "network": { # Force IPv4 connections. On servers with broken or unreachable IPv6, diff --git a/tests/agent/test_models_dev.py b/tests/agent/test_models_dev.py index 2e87c79a62..ee34dee110 100644 --- a/tests/agent/test_models_dev.py +++ b/tests/agent/test_models_dev.py @@ -9,8 +9,11 @@ import pytest from agent.models_dev import ( PROVIDER_TO_MODELS_DEV, _extract_context, + _override_context_window, + _resolve_model_override, fetch_models_dev, get_model_capabilities, + get_model_info, get_provider_info, lookup_models_dev_context, ) @@ -390,3 +393,248 @@ class TestGetModelCapabilities: assert caps is not None assert caps.supports_vision is False + +# --------------------------------------------------------------------------- +# Per-model metadata overrides (model_overrides config) +# --------------------------------------------------------------------------- + + +class TestModelOverrides: + """Tests for the model_overrides config system.""" + + def _setup_overrides(self, overrides_dict): + """Patch _load_model_overrides to return the given dict.""" + import agent.models_dev as md + return patch.object(md, "_load_model_overrides", return_value=overrides_dict) + + # --- _resolve_model_override --- + + def test_per_provider_model_override(self): + """Per-provider+model override is found first.""" + overrides = { + "upstage": { + "solar-pro4": {"context_window": 524288}, + }, + } + with self._setup_overrides(overrides): + result = _resolve_model_override("upstage", "solar-pro4") + assert result is not None + assert result["context_window"] == 524288 + + def test_per_provider_default_fallback(self): + """Per-provider _default is used when model not found.""" + overrides = { + "upstage": { + "_default": {"context_window": 128000}, + }, + } + with self._setup_overrides(overrides): + result = _resolve_model_override("upstage", "unknown-model") + assert result is not None + assert result["context_window"] == 128000 + + def test_global_default_fallback(self): + """Global _default is used when provider not found.""" + overrides = { + "_default": {"context_window": 65536}, + } + with self._setup_overrides(overrides): + result = _resolve_model_override("unknown-provider", "unknown-model") + assert result is not None + assert result["context_window"] == 65536 + + def test_no_override_returns_none(self): + """No override found returns None.""" + with self._setup_overrides({}): + result = _resolve_model_override("anthropic", "claude-sonnet-4") + assert result is None + + def test_per_provider_model_beats_default(self): + """Per-provider+model wins over per-provider _default.""" + overrides = { + "upstage": { + "solar-pro4": {"context_window": 524288}, + "_default": {"context_window": 128000}, + }, + } + with self._setup_overrides(overrides): + result = _resolve_model_override("upstage", "solar-pro4") + assert result is not None + assert result["context_window"] == 524288 + + def test_per_provider_default_beats_global(self): + """Per-provider _default wins over global _default.""" + overrides = { + "upstage": { + "_default": {"context_window": 128000}, + }, + "_default": {"context_window": 65536}, + } + with self._setup_overrides(overrides): + result = _resolve_model_override("upstage", "unknown-model") + assert result is not None + assert result["context_window"] == 128000 + + # --- _override_context_window --- + + def test_override_context_window_returns_value(self): + overrides = { + "upstage": { + "syn-pro": {"context_window": 65536}, + }, + } + with self._setup_overrides(overrides): + ctx = _override_context_window("upstage", "syn-pro") + assert ctx == 65536 + + def test_override_context_window_returns_none_when_missing(self): + with self._setup_overrides({}): + ctx = _override_context_window("upstage", "syn-pro") + assert ctx is None + + def test_override_context_window_rejects_zero(self): + overrides = { + "upstage": { + "bad-model": {"context_window": 0}, + }, + } + with self._setup_overrides(overrides): + ctx = _override_context_window("upstage", "bad-model") + assert ctx is None + + # --- get_model_capabilities with overrides --- + + def test_caps_override_unknown_model(self): + """Override provides capabilities for a model NOT in the catalog (#8731).""" + overrides = { + "custom:my-vllm": { + "my-llava-model": { + "context_window": 8192, + "supports_vision": True, + "supports_reasoning": False, + "supports_tools": True, + }, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + caps = get_model_capabilities("custom:my-vllm", "my-llava-model") + assert caps is not None + assert caps.context_window == 8192 + assert caps.supports_vision is True + assert caps.supports_reasoning is False + assert caps.supports_tools is True + + def test_caps_override_patches_existing_catalog_entry(self): + """Override patches specific fields on a known catalog entry (#84482).""" + overrides = { + "anthropic": { + "claude-sonnet-4": { + "context_window": 500000, + }, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY): + caps = get_model_capabilities("anthropic", "claude-sonnet-4") + assert caps is not None + # Override wins + assert caps.context_window == 500000 + # Non-overridden fields preserved from catalog + assert caps.supports_vision is True + assert caps.supports_tools is True + + def test_caps_no_override_no_catalog_returns_none(self): + """No override and no catalog entry → None.""" + with self._setup_overrides({}), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + caps = get_model_capabilities("anthropic", "unknown-model") + assert caps is None + + def test_caps_override_default_for_unknown_model(self): + """Per-provider _default provides capabilities for unknown models.""" + overrides = { + "custom:my-vllm": { + "_default": { + "context_window": 32768, + "supports_tools": True, + }, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + caps = get_model_capabilities("custom:my-vllm", "some-new-model") + assert caps is not None + assert caps.context_window == 32768 + assert caps.supports_tools is True + + # --- lookup_models_dev_context with overrides --- + + def test_context_lookup_override_wins_over_catalog(self): + """Override context_window wins over models.dev catalog value.""" + overrides = { + "anthropic": { + "claude-opus-4-6": {"context_window": 500000}, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value=SAMPLE_REGISTRY): + ctx = lookup_models_dev_context("anthropic", "claude-opus-4-6") + assert ctx == 500000 + + def test_context_lookup_override_for_unknown_provider(self): + """Override works for providers not in PROVIDER_TO_MODELS_DEV.""" + overrides = { + "upstage": { + "solar-pro4": {"context_window": 524288}, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + ctx = lookup_models_dev_context("upstage", "solar-pro4") + assert ctx == 524288 + + # --- get_model_info with overrides --- + + def test_model_info_override_for_unknown_model(self): + """Override provides full metadata for a model not in the catalog.""" + overrides = { + "custom:my-vllm": { + "my-llava-model": { + "name": "My LLaVA Model", + "family": "llava", + "reasoning": False, + "tool_call": True, + "limit": {"context": 8192, "output": 4096}, + }, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value={}): + info = get_model_info("custom:my-vllm", "my-llava-model") + assert info is not None + assert info.name == "My LLaVA Model" + assert info.family == "llava" + assert info.context_window == 8192 + assert info.max_output == 4096 + assert info.tool_call is True + assert info.reasoning is False + + def test_model_info_override_merges_with_catalog(self): + """Override patches specific fields on a known catalog entry.""" + overrides = { + "anthropic": { + "claude-sonnet-4-6": { + "limit": {"context": 500000, "output": 64000}, + }, + }, + } + with self._setup_overrides(overrides), \ + patch("agent.models_dev.fetch_models_dev", return_value=SAMPLE_REGISTRY): + info = get_model_info("anthropic", "claude-sonnet-4-6") + assert info is not None + # Override wins + assert info.context_window == 500000 + # Non-overridden fields preserved from catalog + assert info.name == "claude-sonnet-4-6" +