hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity

fix(models): resolve context length for OpenRouter :nitro/:floor routing variants

Browse Source 
`:nitro`, `:floor`, `:exacto`, and `:online` are request-time routing
modifiers, not catalog models — OpenRouter's /models lists only the base
id, and a variant runs the same model with the same context window.

`get_model_context_length()` keyed every lookup on the full suffixed id,
so each one missed and the resolver fell through to a generic family
default or the 256K fallback:

  openai/gpt-5.5:nitro           -> 256K   (real 1.05M)
  x-ai/grok-4.6:nitro            -> 131K   (generic "grok" catch-all)
  anthropic/claude-opus-4.6:nitro-> 200K   (generic "claude" catch-all)

The window silently shrank, triggering early compression and a wrong
/usage readout. f14059fa fixed the sibling half of this bug class in
/model validation; this fixes the metadata half.

Strip a recognized variant suffix for LOOKUP only, keeping the suffixed
id on the wire so the routing opt-in survives. Applied after the explicit
config overrides (steps 0b/0c) so a user-pinned value still wins, and
before every cache/catalog lookup. Gated on the request actually routing
through OpenRouter, so a local Ollama `model:tag` is untouched.

`:free`/`:batch`/`:thinking` are deliberately excluded — those ARE
distinct catalog SKUs with their own windows, so stripping them would
report the wrong number.

The suffix set and base-id split move to hermes_constants (import-safe,
dependency-free) so the metadata layer shares one definition with
hermes_cli.models instead of duplicating it.
This commit is contained in:
jakobdylanc
2026-09-01 12:11:52 -04:00
committed by Teknium
parent bf51fee548
commit 284ee03df1
5 changed files with 240 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
agent/model_metadata.py
View File

@@ -24,7 +24,7 @@ if TYPE_CHECKING: # pragma: no cover — runtime import is lazy (see below)
from utils import atomic_json_write, atomic_yaml_write, base_url_host_matches, base_url_hostname
from hermes_constants import OPENROUTER_MODELS_URL
from hermes_constants import OPENROUTER_MODELS_URL, openrouter_variant_base
from agent.message_metadata import PERSISTENCE_ONLY_MESSAGE_FIELDS
logger = logging.getLogger(__name__)
@@ -476,6 +476,45 @@ def _infer_provider_from_url(base_url: str) -> Optional[str]:
return None
def _strip_openrouter_routing_variant(
model: str, base_url: str = "", provider: str = ""
) -> str:
"""Strip an OpenRouter routing-variant suffix for catalog lookup.
``:nitro`` / ``:floor`` / ``:exacto`` / ``:online`` are request-time
routing modifiers, NOT catalog entries — OpenRouter's ``/models`` lists
only the base id, and a variant shares the base model's context window.
Without this, every lookup below misses and the resolver falls through to
a generic family default (``x-ai/grok-4.6:nitro`` → the 131K ``grok``
catch-all instead of its real 2M window).
Only the id used for LOOKUP is rewritten. The suffixed id the caller holds
stays on the wire, so the routing opt-in is preserved — the same rule
:func:`hermes_cli.models.validate_requested_model` applies. Sharing the
base's cache key is intentional: the window is identical, so a variant and
its base must never disagree.
Narrow by design: only applied when the request actually routes through
OpenRouter, so a local ``model:tag`` that happens to end in one of these
words is untouched.
"""
if not model:
return model
is_openrouter = (provider or "").strip().lower() == "openrouter" or (
bool(base_url) and _infer_provider_from_url(base_url) == "openrouter"
)
if not is_openrouter:
return model
base = openrouter_variant_base(model)
if base is None:
return model
logger.debug(
"Resolving context length for OpenRouter routing variant %r via base id %r",
model, base,
)
return base
def _is_known_provider_base_url(base_url: str) -> bool:
return _infer_provider_from_url(base_url) is not None
@@ -1897,6 +1936,14 @@ def get_model_context_length(
logger.info("No model id provided for context length resolution — defaulting to %s tokens.", f"{DEFAULT_FALLBACK_CONTEXT:,}")
return DEFAULT_FALLBACK_CONTEXT
model = _strip_provider_prefix(model) # "local:x" -> "x"; Ollama "model:tag" colons preserved
# OpenRouter routing variants (":nitro", ":floor", ...) are request-time
# modifiers, not catalog entries — resolve the window from the BASE id.
# Deliberately placed AFTER the explicit config overrides above (0b/0c) so
# a user who pinned the fully-suffixed id keeps winning, and BEFORE every
# cache/catalog lookup below so the base's real window is found instead of
# a generic family default. Mirrors the validation path's base/suffix split
# in hermes_cli.models.validate_requested_model.
model = _strip_openrouter_routing_variant(model, base_url=base_url, provider=provider)
# Endpoint-scoped metadata goes AHEAD of the persistent cache so a value learned on a
# multiplexed provider's other endpoint cannot override it.
endpoint_context = _endpoint_scoped_context_length(model, base_url)

11
hermes_cli/models.py
View File

@@ -60,6 +60,7 @@ from hermes_cli.models_local import (
_ollama_probe_cache_key,
_root_for_ollama_native_api,
fetch_ollama_cloud_models)
from hermes_constants import OPENROUTER_VARIANT_SUFFIXES, openrouter_variant_base
logger = logging.getLogger(__name__)
@@ -838,11 +839,11 @@ def _model_in_provider_catalog(name_lower: str, providers: set[str]) -> bool:
for model in _provider_catalog_names(provider))
def _openrouter_variant_base(model_id: str) -> Optional[str]:
"""Base model id when ``model_id`` carries a recognized OpenRouter routing-variant suffix
(``x-ai/grok-4:nitro`` → ``x-ai/grok-4``), else ``None``."""
base, sep, suffix = (model_id or "").rpartition(":")
return base if sep and base and suffix.lower() in _OPENROUTER_VARIANT_SUFFIXES else None
# Canonical suffix set lives in ``hermes_constants`` so the metadata layer can
# share it without importing the CLI. Re-exported here under the historical
# private names used by ``validate_requested_model``.
_OPENROUTER_VARIANT_SUFFIXES = OPENROUTER_VARIANT_SUFFIXES
_openrouter_variant_base = openrouter_variant_base
def _resolve_static_model_alias(

12
hermes_cli/models_catalog_static.py
View File

@@ -8,6 +8,8 @@ from __future__ import annotations
from typing import NamedTuple
from hermes_constants import OPENROUTER_VARIANT_SUFFIXES
# Fallback OpenRouter snapshot used when the live catalog is unavailable, as
# ``(model_id, description shown in menus)``. ``:free`` SKUs are described "free".
@@ -506,12 +508,10 @@ _PROVIDER_RETIRED_ALIASES: dict[str, tuple[str, ...]] = {
_AGGREGATOR_PROVIDERS = frozenset({"nous", "openrouter", "ai-gateway", "copilot", "kilocode"})
# OpenRouter request-time routing variants (docs: guides/routing/model-variants): per-request
# modifiers valid on ANY model id (":nitro" throughput sort + priority tier, ":floor" price sort +
# flex tier, ":exacto" quality-first provider sort, ":online" web plugin). Never separate catalog
# entries — /models lists only the base id. NOT here: ":free", ":batch", ":thinking", ":extended"
# — those ARE distinct SKUs that appear in /models when they exist, so absence is authoritative.
_OPENROUTER_VARIANT_SUFFIXES = frozenset({"nitro", "floor", "exacto", "online"})
# Canonical suffix set lives in ``hermes_constants`` so the metadata layer can
# share it without importing the CLI. Re-exported here under the historical
# private name used by ``hermes_cli.models``.
_OPENROUTER_VARIANT_SUFFIXES = OPENROUTER_VARIANT_SUFFIXES
# Subscription/OAuth providers whose catalogs RE-EXPOSE other vendors' models; tried only as a last

41
hermes_constants.py
View File

@@ -1156,6 +1156,47 @@ PARTIAL_STREAM_STUB_ID = "partial-stream-stub"
FINISH_REASON_LENGTH = "length"
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
OPENROUTER_MODELS_URL = f"{OPENROUTER_BASE_URL}/models"
# OpenRouter request-time routing variants (docs: guides/routing/model-variants).
# These suffixes are per-request routing modifiers valid on ANY model id —
# ":nitro" sorts the endpoint pool by throughput and admits priority-tier
# endpoints, ":floor" sorts by price and admits flex-tier endpoints, ":exacto"
# applies quality-first provider sorting, ":online" attaches the web plugin.
# They are never separate catalog entries: /models lists only the base id, so
# every catalog lookup must key on the BASE while the suffixed id stays on the
# wire.
# NOT in this set: ":free", ":batch", ":thinking", ":extended" — those ARE
# distinct catalog SKUs with their own /models entries (and their own context
# windows), so stripping them would resolve the wrong window.
OPENROUTER_VARIANT_SUFFIXES: frozenset[str] = frozenset(
{"nitro", "floor", "exacto", "online"}
)
def openrouter_variant_base(model_id: str) -> str | None:
"""Return the base model id when ``model_id`` carries a recognized
OpenRouter routing-variant suffix (e.g. ``x-ai/grok-4:nitro`` →
``x-ai/grok-4``), else ``None``.
Lives here rather than in ``hermes_cli.models`` so the metadata layer
(``agent.model_metadata``) can share one definition without importing the
CLI — this module is dependency-free by contract.
>>> openrouter_variant_base("x-ai/grok-4:nitro")
'x-ai/grok-4'
>>> openrouter_variant_base("x-ai/grok-4:free") is None
True
>>> openrouter_variant_base("x-ai/grok-4") is None
True
"""
base, sep, suffix = (model_id or "").rpartition(":")
if not sep or not base:
return None
if suffix.lower() in OPENROUTER_VARIANT_SUFFIXES:
return base
return None
AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1"

139
tests/agent/test_model_metadata.py
View File

@@ -1941,3 +1941,142 @@ class TestFallbackWarning:
if r.levelno == logging.WARNING and "falling back" in r.getMessage()
]
assert len(fallback_warnings) == 0
# =========================================================================
# get_model_context_length — OpenRouter routing-variant suffixes
# =========================================================================
class TestOpenRouterRoutingVariantContextLength:
"""OpenRouter's `:nitro`, `:floor`, `:exacto`, `:online` are request-time
routing modifiers, not catalog models — /models lists only the base id and
a variant runs the same model, so it has the SAME context window.
The validation path (tests/hermes_cli/test_model_validation.py::
TestValidateOpenRouterVariantSuffixes) already preserves the suffixed id;
this pins the metadata half. Before the fix, `model:nitro` missed every
catalog lookup and fell through to a generic family default — e.g.
`x-ai/grok-4.6:nitro` reported the 131K `grok` catch-all instead of 2M,
silently shrinking the usable window and triggering early compression.
The invariant asserted here is a relation, not a snapshot: a variant must
resolve to whatever its base resolves to.
"""
_CATALOG = {
"x-ai/grok-4.6": {"context_length": 2_000_000},
"anthropic/claude-opus-4.6": {"context_length": 1_000_000},
}
_VARIANTS = ["nitro", "floor", "exacto", "online"]
@pytest.mark.parametrize("suffix", _VARIANTS)
@pytest.mark.parametrize("model", sorted(_CATALOG))
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_matches_base_context(
self, mock_fetch, mock_models_dev, mock_cache, model, suffix
):
"""A routing variant resolves to exactly its base model's window."""
mock_fetch.return_value = self._CATALOG
base_ctx = get_model_context_length(model, provider="openrouter")
variant_ctx = get_model_context_length(
f"{model}:{suffix}", provider="openrouter"
)
assert variant_ctx == base_ctx
assert variant_ctx == self._CATALOG[model]["context_length"]
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_does_not_fall_back_to_family_default(
self, mock_fetch, mock_models_dev, mock_cache
):
"""The reported symptom: the generic `grok` catch-all (131K) winning
over the catalog's real window for a `:nitro` id."""
mock_fetch.return_value = self._CATALOG
result = get_model_context_length(
"x-ai/grok-4.6:nitro", provider="openrouter"
)
assert result == 2_000_000
assert result != DEFAULT_CONTEXT_LENGTHS.get("grok")
assert result != DEFAULT_FALLBACK_CONTEXT
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_uppercase_suffix_resolves(self, mock_fetch, mock_models_dev, mock_cache):
"""Suffix matching is case-insensitive, matching the validation path."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:NITRO", provider="openrouter"
) == 2_000_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_resolves_via_base_url_without_explicit_provider(
self, mock_fetch, mock_models_dev, mock_cache
):
"""Callers that pass only base_url (no provider=) must strip too —
the OpenRouter host is enough to know the suffix is a routing hint."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:nitro",
base_url="https://openrouter.ai/api/v1",
) == 2_000_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_free_sku_suffix_is_not_stripped(
self, mock_fetch, mock_models_dev, mock_cache
):
"""`:free` / `:batch` / `:thinking` ARE distinct catalog SKUs with
their own windows. Stripping them would report the wrong number, so
the SKU's own entry must win over the base's."""
mock_fetch.return_value = {
"thinkingmachines/inkling": {"context_length": 1_000_000},
"thinkingmachines/inkling:free": {"context_length": 64_000},
}
assert get_model_context_length(
"thinkingmachines/inkling:free", provider="openrouter"
) == 64_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata", return_value={})
def test_non_openrouter_colon_tag_is_untouched(
self, mock_fetch, mock_models_dev, mock_cache
):
"""An Ollama `model:tag` that happens to end in a variant word must
keep its full id — the carve-out is OpenRouter-only."""
with patch(
"agent.model_metadata._query_local_context_length", return_value=None
), patch("agent.model_metadata._query_ollama_api_show", return_value=None):
result = get_model_context_length(
"mymodel:online",
base_url="http://localhost:11434/v1",
)
assert result == DEFAULT_FALLBACK_CONTEXT
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_explicit_config_override_still_wins(
self, mock_fetch, mock_models_dev, mock_cache
):
"""Stripping happens after step 0 — a user who pinned the suffixed id
via model.context_length keeps their value (the documented
self-unblock path must not regress)."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:nitro",
provider="openrouter",
config_context_length=123_456,
) == 123_456