hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

feat(models): per-model metadata overrides via model_overrides config

Browse Source 
Add a unified model_overrides config section that lets users manually
declare context_window, max_output_tokens, capabilities, cost, and
family for any provider+model — winning over models.dev, OpenRouter, and
hardcoded defaults.

Resolution order (first hit wins):
  1. model_overrides.<provider>.<model_id>  (per-provider+model)
  2. model_overrides.<provider>._default    (per-provider default)
  3. model_overrides._default               (global default)
  4. Normal catalog resolution

Key subtlety: an unknown model id (not in the
catalog) derives base metadata from sensible defaults before patching,
so overriding a model the catalog doesn't know yet is the supported
self-unblock path. This is exactly the #84482 scenario (Upstage
solar-pro4/syn-pro wrong context) and the #8731 scenario (custom/local
models with manual capability declaration).

Wired into:
  - get_model_capabilities() — patches capability fields; unknown models
    get safe defaults (tools on, vision/reasoning off) before patching
  - lookup_models_dev_context() — context_window override, checked before
    catalog lookup so it works even for providers not in PROVIDER_TO_MODELS_DEV
  - get_model_info() — merges override dict onto catalog entry (shallow
    merge); for unknown models, the override is the sole source of metadata
  - get_model_context_length() — step 0b in the resolution pipeline,
    before custom_providers (0c) and before any network probe

Config example:
  model_overrides:
    upstage:
      solar-pro4:
        context_window: 524288
      syn-pro:
        context_window: 65536
    custom:my-local-vllm:
      my-llava-model:
        context_window: 8192
        supports_vision: true
        supports_reasoning: false
        supports_tools: true
    _default:
      context_window: 128000

Fixes #8731
Fixes #84482
Refs #47247
This commit is contained in:
kshitij
2026-08-14 00:30:51 +05:30
parent cdea8214cb
commit dafdba324a
4 changed files with 504 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
agent/model_metadata.py
View File

@@ -2577,6 +2577,7 @@ def get_model_context_length(
Resolution order:
0. Explicit config override (model.context_length or custom_providers per-model)
0b. model_overrides config (per-provider+model context_window override)
0c. Endpoint-scoped metadata for models validated on one multiplexed endpoint
1. Persistent cache (previously discovered via probing). Nous URLs,
LM Studio, and Codex OAuth bypass the cache here so their provider
@@ -2638,7 +2639,21 @@ def get_model_context_length(
logger.debug("MoA aggregator context-length resolution failed", exc_info=True)
# Fall through to the generic default if aggregator resolution failed.
# 0b. custom_providers per-model override — check before any probe.
# 0b. model_overrides config — per-provider+model context_window override.
# This is the supported self-unblock path for models with wrong or missing
# context in models.dev (#84482) and for custom/local models not in the
# catalog (#8731). Checked before custom_providers (step 0c) and before any
# network probe so it never blocks.
if provider and model:
try:
from agent.models_dev import _override_context_window
mo_ctx = _override_context_window(provider, model)
if mo_ctx is not None and mo_ctx > 0:
return mo_ctx
except Exception:
pass # fall through to other resolution paths
# 0c. custom_providers per-model override — check before any probe.
# This closes the gap where /model switch and display paths used to fall
# back to 128K despite the user having a per-model context_length set.
# See #15779.

239
agent/models_dev.py
View File

@@ -499,7 +499,17 @@ def lookup_models_dev_context(provider: str, model: str) -> Optional[int]:
Returns the context window in tokens, or None if not found.
Handles case-insensitive matching and filters out context=0 entries.
A ``model_overrides`` config entry for this provider+model (or its
``_default`` fallback) wins over the catalog value — this is the
supported self-unblock path for models with wrong or missing context
in models.dev (#84482).
"""
# Config override — checked before catalog so it always wins.
override_ctx = _override_context_window(provider, model)
if override_ctx is not None:
return override_ctx
mdev_provider_id = PROVIDER_TO_MODELS_DEV.get(provider)
if not mdev_provider_id:
return None
@@ -586,6 +596,103 @@ class ModelCapabilities:
model_family: str = ""
# --------------------------------------------------------------------------- #
# Per-model metadata overrides (config.yaml → model_overrides) #
# --------------------------------------------------------------------------- #
#
# Resolution order for every query function below:
# 1. ``model_overrides.<provider>.<model_id>`` — explicit per-provider+model
# 2. ``model_overrides.<provider>._default`` — per-provider default
# 3. ``model_overrides._default`` — global default
# 4. models.dev / OpenRouter / hardcoded — normal catalog resolution
#
# An override may set any subset of fields; unspecified fields fall through to
# the catalog value. For a model id NOT in the catalog, the override is the
# only source of metadata — this is the supported self-unblock path for new
# or custom models (#84482, #8731).
_OVERRIDE_CACHE: Optional[Dict[str, Any]] = None
_OVERRIDE_CACHE_CFG_HASH: int = 0
def _load_model_overrides() -> Dict[str, Any]:
"""Load and cache the ``model_overrides`` config section.
Caches by ``id(cfg)`` so a config reload (new dict identity) invalidates
automatically. Returns empty dict on any failure.
"""
global _OVERRIDE_CACHE, _OVERRIDE_CACHE_CFG_HASH
try:
from hermes_cli.config import cfg_get, load_config_readonly
cfg = load_config_readonly()
cfg_id = id(cfg)
if cfg_id == _OVERRIDE_CACHE_CFG_HASH and _OVERRIDE_CACHE is not None:
return _OVERRIDE_CACHE
raw = cfg_get(cfg, "model_overrides", default={})
overrides = raw if isinstance(raw, dict) else {}
_OVERRIDE_CACHE = overrides
_OVERRIDE_CACHE_CFG_HASH = cfg_id
return overrides
except Exception:
return {}
def _resolve_model_override(
provider: str, model: str
) -> Optional[Dict[str, Any]]:
"""Resolve the override dict for a provider+model, or None.
Checks per-provider+model, then per-provider ``_default``, then global
``_default``. Returns the first match (which may be partially populated —
callers only read the keys they care about).
"""
overrides = _load_model_overrides()
if not overrides:
return None
provider_key = (provider or "").strip()
model_key = (model or "").strip()
if not provider_key and not model_key:
return None
# 1. Per-provider+model
provider_section = overrides.get(provider_key)
if isinstance(provider_section, dict) and model_key:
model_section = provider_section.get(model_key)
if isinstance(model_section, dict):
return model_section
# 2. Per-provider _default
if isinstance(provider_section, dict):
default = provider_section.get("_default")
if isinstance(default, dict):
return default
# 3. Global _default
global_default = overrides.get("_default")
if isinstance(global_default, dict):
return global_default
return None
def _override_context_window(
provider: str, model: str
) -> Optional[int]:
"""Return the overridden context_window, or None."""
ov = _resolve_model_override(provider, model)
if ov is None:
return None
raw = ov.get("context_window")
if raw is None:
return None
try:
ctx = int(raw)
return ctx if ctx > 0 else None
except (TypeError, ValueError):
return None
def _get_provider_models(provider: str) -> Optional[Dict[str, Any]]:
"""Resolve a Hermes provider ID to its models dict from models.dev.
@@ -629,6 +736,15 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit
Uses the existing fetch_models_dev() and PROVIDER_TO_MODELS_DEV mapping.
Returns None if model not found.
``model_overrides`` config entries (per-provider+model, per-provider
``_default``, or global ``_default``) win over catalog values. For a
model id NOT in the catalog, the override is the only source of
metadata — this is the supported self-unblock path for custom/local
models (#8731) and for models with wrong context in models.dev
(#84482). An override may set any subset of fields; unspecified fields
fall through to the catalog value (or sensible defaults when the model
is absent from the catalog entirely).
Extracts from model entry fields:
- reasoning (bool) → supports_reasoning
- tool_call (bool) → supports_tools
@@ -637,42 +753,81 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit
- limit.output (int) → max_output_tokens
- family (str) → model_family
"""
# Check config override first — it may fully replace the catalog entry
# or patch specific fields. For unknown models (not in catalog), the
# override is the sole source of metadata.
override = _resolve_model_override(provider, model)
models = _get_provider_models(provider)
if models is None:
entry = _find_model_entry(models, model) if models is not None else None
# If no catalog entry and no override, we can't resolve capabilities.
if entry is None and override is None:
return None
entry = _find_model_entry(models, model)
if entry is None:
return None
# Start from catalog entry (if found), else use defaults.
if entry is not None:
supports_tools = bool(entry.get("tool_call", False))
# Vision: prefer explicit `modalities.input` when models.dev provides it.
# The older `attachment` flag can be stale or too broad for image routing;
# fall back to it only when the input modalities are absent/invalid.
input_mods = entry.get("modalities", {})
if isinstance(input_mods, dict):
input_mods = input_mods.get("input")
else:
input_mods = None
if isinstance(input_mods, list):
supports_vision = "image" in input_mods
else:
supports_vision = bool(entry.get("attachment", False))
supports_reasoning = bool(entry.get("reasoning", False))
# Extract capability flags (default to False if missing)
supports_tools = bool(entry.get("tool_call", False))
# Vision: prefer explicit `modalities.input` when models.dev provides it.
# The older `attachment` flag can be stale or too broad for image routing;
# fall back to it only when the input modalities are absent/invalid.
input_mods = entry.get("modalities", {})
if isinstance(input_mods, dict):
input_mods = input_mods.get("input")
limit = entry.get("limit", {})
if not isinstance(limit, dict):
limit = {}
ctx = limit.get("context")
context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000
out = limit.get("output")
max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192
model_family = entry.get("family", "") or ""
else:
input_mods = None
if isinstance(input_mods, list):
supports_vision = "image" in input_mods
else:
supports_vision = bool(entry.get("attachment", False))
supports_reasoning = bool(entry.get("reasoning", False))
# Unknown model — derive sensible defaults. The override will
# patch whichever fields it specifies; the rest stay at defaults
# that are safe for agentic use (tools on, vision/reasoning off).
supports_tools = True
supports_vision = False
supports_reasoning = False
context_window = 200000
max_output_tokens = 8192
model_family = ""
# Extract limits
limit = entry.get("limit", {})
if not isinstance(limit, dict):
limit = {}
ctx = limit.get("context")
context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000
out = limit.get("output")
max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192
model_family = entry.get("family", "") or ""
# Apply override patches (each field is optional in the override dict).
if override is not None:
if "supports_tools" in override:
supports_tools = bool(override["supports_tools"])
if "supports_vision" in override:
supports_vision = bool(override["supports_vision"])
if "supports_reasoning" in override:
supports_reasoning = bool(override["supports_reasoning"])
if "context_window" in override:
try:
ctx_ov = int(override["context_window"])
if ctx_ov > 0:
context_window = ctx_ov
except (TypeError, ValueError):
pass
if "max_output_tokens" in override:
try:
out_ov = int(override["max_output_tokens"])
if out_ov > 0:
max_output_tokens = out_ov
except (TypeError, ValueError):
pass
if "model_family" in override:
model_family = str(override["model_family"] or "")
return ModelCapabilities(
supports_tools=supports_tools,
@@ -884,27 +1039,51 @@ def get_model_info(
Accepts Hermes or models.dev provider ID. Tries exact match then
case-insensitive fallback. Returns None if not found.
``model_overrides`` config entries (per-provider+model, per-provider
``_default``, or global ``_default``) patch the catalog entry's fields
when present. For a model id NOT in the catalog, the override is the
sole source of metadata — this is the supported self-unblock path
for custom/local models (#8731) and for models with wrong context
in models.dev (#84482).
"""
override = _resolve_model_override(provider_id, model_id)
mdev_id = PROVIDER_TO_MODELS_DEV.get(provider_id, provider_id)
data = fetch_models_dev()
pdata = data.get(mdev_id)
if not isinstance(pdata, dict):
# No catalog data — return from override alone if we have one.
if override is not None:
return _parse_model_info(model_id, override, mdev_id)
return None
models = pdata.get("models", {})
if not isinstance(models, dict):
if override is not None:
return _parse_model_info(model_id, override, mdev_id)
return None
# Exact match
raw = models.get(model_id)
if isinstance(raw, dict):
if override is not None:
merged = {**raw, **override}
return _parse_model_info(model_id, merged, mdev_id)
return _parse_model_info(model_id, raw, mdev_id)
# Case-insensitive fallback
model_lower = model_id.lower()
for mid, mdata in models.items():
if mid.lower() == model_lower and isinstance(mdata, dict):
if override is not None:
merged = {**mdata, **override}
return _parse_model_info(mid, merged, mdev_id)
return _parse_model_info(mid, mdata, mdev_id)
# Model not in catalog — return from override alone if we have one.
if override is not None:
return _parse_model_info(model_id, override, mdev_id)
return None

31
hermes_cli/config_defaults.py
View File

@@ -2539,6 +2539,37 @@ DEFAULT_CONFIG = {
"providers": {},
},
# Per-model metadata overrides — manually declare context_window,
# max_output_tokens, capabilities, or cost for any provider+model.
# Overrides win over models.dev, OpenRouter, and hardcoded defaults.
#
# Two scopes:
# 1. Per-provider+model: model_overrides.<provider>.<model_id>
# 2. Per-provider default: model_overrides.<provider>._default
# 3. Global default: model_overrides._default
#
# An unknown model id (not in models.dev) inherits base metadata from
# its family/dated-snapshot entry before patching, so overriding a
# model the catalog doesn't know yet is the supported self-unblock
# path (#84482).
#
# Example:
# model_overrides:
# upstage:
# solar-pro4:
# context_window: 524288
# syn-pro:
# context_window: 65536
# custom:my-local-vllm:
# my-llava-model:
# context_window: 8192
# supports_vision: true
# supports_reasoning: false
# supports_tools: true
# _default:
# context_window: 128000
"model_overrides": {},
# Network settings — workarounds for connectivity issues.
"network": {
# Force IPv4 connections. On servers with broken or unreachable IPv6,

248
tests/agent/test_models_dev.py
View File

@@ -9,8 +9,11 @@ import pytest
from agent.models_dev import (
PROVIDER_TO_MODELS_DEV,
_extract_context,
_override_context_window,
_resolve_model_override,
fetch_models_dev,
get_model_capabilities,
get_model_info,
get_provider_info,
lookup_models_dev_context,
)
@@ -390,3 +393,248 @@ class TestGetModelCapabilities:
assert caps is not None
assert caps.supports_vision is False
# ---------------------------------------------------------------------------
# Per-model metadata overrides (model_overrides config)
# ---------------------------------------------------------------------------
class TestModelOverrides:
"""Tests for the model_overrides config system."""
def _setup_overrides(self, overrides_dict):
"""Patch _load_model_overrides to return the given dict."""
import agent.models_dev as md
return patch.object(md, "_load_model_overrides", return_value=overrides_dict)
# --- _resolve_model_override ---
def test_per_provider_model_override(self):
"""Per-provider+model override is found first."""
overrides = {
"upstage": {
"solar-pro4": {"context_window": 524288},
},
}
with self._setup_overrides(overrides):
result = _resolve_model_override("upstage", "solar-pro4")
assert result is not None
assert result["context_window"] == 524288
def test_per_provider_default_fallback(self):
"""Per-provider _default is used when model not found."""
overrides = {
"upstage": {
"_default": {"context_window": 128000},
},
}
with self._setup_overrides(overrides):
result = _resolve_model_override("upstage", "unknown-model")
assert result is not None
assert result["context_window"] == 128000
def test_global_default_fallback(self):
"""Global _default is used when provider not found."""
overrides = {
"_default": {"context_window": 65536},
}
with self._setup_overrides(overrides):
result = _resolve_model_override("unknown-provider", "unknown-model")
assert result is not None
assert result["context_window"] == 65536
def test_no_override_returns_none(self):
"""No override found returns None."""
with self._setup_overrides({}):
result = _resolve_model_override("anthropic", "claude-sonnet-4")
assert result is None
def test_per_provider_model_beats_default(self):
"""Per-provider+model wins over per-provider _default."""
overrides = {
"upstage": {
"solar-pro4": {"context_window": 524288},
"_default": {"context_window": 128000},
},
}
with self._setup_overrides(overrides):
result = _resolve_model_override("upstage", "solar-pro4")
assert result is not None
assert result["context_window"] == 524288
def test_per_provider_default_beats_global(self):
"""Per-provider _default wins over global _default."""
overrides = {
"upstage": {
"_default": {"context_window": 128000},
},
"_default": {"context_window": 65536},
}
with self._setup_overrides(overrides):
result = _resolve_model_override("upstage", "unknown-model")
assert result is not None
assert result["context_window"] == 128000
# --- _override_context_window ---
def test_override_context_window_returns_value(self):
overrides = {
"upstage": {
"syn-pro": {"context_window": 65536},
},
}
with self._setup_overrides(overrides):
ctx = _override_context_window("upstage", "syn-pro")
assert ctx == 65536
def test_override_context_window_returns_none_when_missing(self):
with self._setup_overrides({}):
ctx = _override_context_window("upstage", "syn-pro")
assert ctx is None
def test_override_context_window_rejects_zero(self):
overrides = {
"upstage": {
"bad-model": {"context_window": 0},
},
}
with self._setup_overrides(overrides):
ctx = _override_context_window("upstage", "bad-model")
assert ctx is None
# --- get_model_capabilities with overrides ---
def test_caps_override_unknown_model(self):
"""Override provides capabilities for a model NOT in the catalog (#8731)."""
overrides = {
"custom:my-vllm": {
"my-llava-model": {
"context_window": 8192,
"supports_vision": True,
"supports_reasoning": False,
"supports_tools": True,
},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value={}):
caps = get_model_capabilities("custom:my-vllm", "my-llava-model")
assert caps is not None
assert caps.context_window == 8192
assert caps.supports_vision is True
assert caps.supports_reasoning is False
assert caps.supports_tools is True
def test_caps_override_patches_existing_catalog_entry(self):
"""Override patches specific fields on a known catalog entry (#84482)."""
overrides = {
"anthropic": {
"claude-sonnet-4": {
"context_window": 500000,
},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value=CAPS_REGISTRY):
caps = get_model_capabilities("anthropic", "claude-sonnet-4")
assert caps is not None
# Override wins
assert caps.context_window == 500000
# Non-overridden fields preserved from catalog
assert caps.supports_vision is True
assert caps.supports_tools is True
def test_caps_no_override_no_catalog_returns_none(self):
"""No override and no catalog entry → None."""
with self._setup_overrides({}), \
patch("agent.models_dev.fetch_models_dev", return_value={}):
caps = get_model_capabilities("anthropic", "unknown-model")
assert caps is None
def test_caps_override_default_for_unknown_model(self):
"""Per-provider _default provides capabilities for unknown models."""
overrides = {
"custom:my-vllm": {
"_default": {
"context_window": 32768,
"supports_tools": True,
},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value={}):
caps = get_model_capabilities("custom:my-vllm", "some-new-model")
assert caps is not None
assert caps.context_window == 32768
assert caps.supports_tools is True
# --- lookup_models_dev_context with overrides ---
def test_context_lookup_override_wins_over_catalog(self):
"""Override context_window wins over models.dev catalog value."""
overrides = {
"anthropic": {
"claude-opus-4-6": {"context_window": 500000},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value=SAMPLE_REGISTRY):
ctx = lookup_models_dev_context("anthropic", "claude-opus-4-6")
assert ctx == 500000
def test_context_lookup_override_for_unknown_provider(self):
"""Override works for providers not in PROVIDER_TO_MODELS_DEV."""
overrides = {
"upstage": {
"solar-pro4": {"context_window": 524288},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value={}):
ctx = lookup_models_dev_context("upstage", "solar-pro4")
assert ctx == 524288
# --- get_model_info with overrides ---
def test_model_info_override_for_unknown_model(self):
"""Override provides full metadata for a model not in the catalog."""
overrides = {
"custom:my-vllm": {
"my-llava-model": {
"name": "My LLaVA Model",
"family": "llava",
"reasoning": False,
"tool_call": True,
"limit": {"context": 8192, "output": 4096},
},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value={}):
info = get_model_info("custom:my-vllm", "my-llava-model")
assert info is not None
assert info.name == "My LLaVA Model"
assert info.family == "llava"
assert info.context_window == 8192
assert info.max_output == 4096
assert info.tool_call is True
assert info.reasoning is False
def test_model_info_override_merges_with_catalog(self):
"""Override patches specific fields on a known catalog entry."""
overrides = {
"anthropic": {
"claude-sonnet-4-6": {
"limit": {"context": 500000, "output": 64000},
},
},
}
with self._setup_overrides(overrides), \
patch("agent.models_dev.fetch_models_dev", return_value=SAMPLE_REGISTRY):
info = get_model_info("anthropic", "claude-sonnet-4-6")
assert info is not None
# Override wins
assert info.context_window == 500000
# Non-overridden fields preserved from catalog
assert info.name == "claude-sonnet-4-6"