diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 99c5b558b2..a36092ce8d 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: # pragma: no cover — runtime import is lazy (see below) from utils import atomic_json_write, atomic_yaml_write, base_url_host_matches, base_url_hostname -from hermes_constants import OPENROUTER_MODELS_URL +from hermes_constants import OPENROUTER_MODELS_URL, openrouter_variant_base from agent.message_metadata import PERSISTENCE_ONLY_MESSAGE_FIELDS logger = logging.getLogger(__name__) @@ -476,6 +476,45 @@ def _infer_provider_from_url(base_url: str) -> Optional[str]: return None +def _strip_openrouter_routing_variant( + model: str, base_url: str = "", provider: str = "" +) -> str: + """Strip an OpenRouter routing-variant suffix for catalog lookup. + + ``:nitro`` / ``:floor`` / ``:exacto`` / ``:online`` are request-time + routing modifiers, NOT catalog entries — OpenRouter's ``/models`` lists + only the base id, and a variant shares the base model's context window. + Without this, every lookup below misses and the resolver falls through to + a generic family default (``x-ai/grok-4.6:nitro`` → the 131K ``grok`` + catch-all instead of its real 2M window). + + Only the id used for LOOKUP is rewritten. The suffixed id the caller holds + stays on the wire, so the routing opt-in is preserved — the same rule + :func:`hermes_cli.models.validate_requested_model` applies. Sharing the + base's cache key is intentional: the window is identical, so a variant and + its base must never disagree. + + Narrow by design: only applied when the request actually routes through + OpenRouter, so a local ``model:tag`` that happens to end in one of these + words is untouched. + """ + if not model: + return model + is_openrouter = (provider or "").strip().lower() == "openrouter" or ( + bool(base_url) and _infer_provider_from_url(base_url) == "openrouter" + ) + if not is_openrouter: + return model + base = openrouter_variant_base(model) + if base is None: + return model + logger.debug( + "Resolving context length for OpenRouter routing variant %r via base id %r", + model, base, + ) + return base + + def _is_known_provider_base_url(base_url: str) -> bool: return _infer_provider_from_url(base_url) is not None @@ -1897,6 +1936,14 @@ def get_model_context_length( logger.info("No model id provided for context length resolution — defaulting to %s tokens.", f"{DEFAULT_FALLBACK_CONTEXT:,}") return DEFAULT_FALLBACK_CONTEXT model = _strip_provider_prefix(model) # "local:x" -> "x"; Ollama "model:tag" colons preserved + # OpenRouter routing variants (":nitro", ":floor", ...) are request-time + # modifiers, not catalog entries — resolve the window from the BASE id. + # Deliberately placed AFTER the explicit config overrides above (0b/0c) so + # a user who pinned the fully-suffixed id keeps winning, and BEFORE every + # cache/catalog lookup below so the base's real window is found instead of + # a generic family default. Mirrors the validation path's base/suffix split + # in hermes_cli.models.validate_requested_model. + model = _strip_openrouter_routing_variant(model, base_url=base_url, provider=provider) # Endpoint-scoped metadata goes AHEAD of the persistent cache so a value learned on a # multiplexed provider's other endpoint cannot override it. endpoint_context = _endpoint_scoped_context_length(model, base_url) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index fd3ba17d9c..6a475c8a44 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -60,6 +60,7 @@ from hermes_cli.models_local import ( _ollama_probe_cache_key, _root_for_ollama_native_api, fetch_ollama_cloud_models) +from hermes_constants import OPENROUTER_VARIANT_SUFFIXES, openrouter_variant_base logger = logging.getLogger(__name__) @@ -838,11 +839,11 @@ def _model_in_provider_catalog(name_lower: str, providers: set[str]) -> bool: for model in _provider_catalog_names(provider)) -def _openrouter_variant_base(model_id: str) -> Optional[str]: - """Base model id when ``model_id`` carries a recognized OpenRouter routing-variant suffix - (``x-ai/grok-4:nitro`` → ``x-ai/grok-4``), else ``None``.""" - base, sep, suffix = (model_id or "").rpartition(":") - return base if sep and base and suffix.lower() in _OPENROUTER_VARIANT_SUFFIXES else None +# Canonical suffix set lives in ``hermes_constants`` so the metadata layer can +# share it without importing the CLI. Re-exported here under the historical +# private names used by ``validate_requested_model``. +_OPENROUTER_VARIANT_SUFFIXES = OPENROUTER_VARIANT_SUFFIXES +_openrouter_variant_base = openrouter_variant_base def _resolve_static_model_alias( diff --git a/hermes_cli/models_catalog_static.py b/hermes_cli/models_catalog_static.py index 127a160a09..d99faa026a 100644 --- a/hermes_cli/models_catalog_static.py +++ b/hermes_cli/models_catalog_static.py @@ -8,6 +8,8 @@ from __future__ import annotations from typing import NamedTuple +from hermes_constants import OPENROUTER_VARIANT_SUFFIXES + # Fallback OpenRouter snapshot used when the live catalog is unavailable, as # ``(model_id, description shown in menus)``. ``:free`` SKUs are described "free". @@ -506,12 +508,10 @@ _PROVIDER_RETIRED_ALIASES: dict[str, tuple[str, ...]] = { _AGGREGATOR_PROVIDERS = frozenset({"nous", "openrouter", "ai-gateway", "copilot", "kilocode"}) -# OpenRouter request-time routing variants (docs: guides/routing/model-variants): per-request -# modifiers valid on ANY model id (":nitro" throughput sort + priority tier, ":floor" price sort + -# flex tier, ":exacto" quality-first provider sort, ":online" web plugin). Never separate catalog -# entries — /models lists only the base id. NOT here: ":free", ":batch", ":thinking", ":extended" -# — those ARE distinct SKUs that appear in /models when they exist, so absence is authoritative. -_OPENROUTER_VARIANT_SUFFIXES = frozenset({"nitro", "floor", "exacto", "online"}) +# Canonical suffix set lives in ``hermes_constants`` so the metadata layer can +# share it without importing the CLI. Re-exported here under the historical +# private name used by ``hermes_cli.models``. +_OPENROUTER_VARIANT_SUFFIXES = OPENROUTER_VARIANT_SUFFIXES # Subscription/OAuth providers whose catalogs RE-EXPOSE other vendors' models; tried only as a last diff --git a/hermes_constants.py b/hermes_constants.py index 1b3c99532c..3589cf0435 100644 --- a/hermes_constants.py +++ b/hermes_constants.py @@ -1156,6 +1156,47 @@ PARTIAL_STREAM_STUB_ID = "partial-stream-stub" FINISH_REASON_LENGTH = "length" OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" OPENROUTER_MODELS_URL = f"{OPENROUTER_BASE_URL}/models" + +# OpenRouter request-time routing variants (docs: guides/routing/model-variants). +# These suffixes are per-request routing modifiers valid on ANY model id — +# ":nitro" sorts the endpoint pool by throughput and admits priority-tier +# endpoints, ":floor" sorts by price and admits flex-tier endpoints, ":exacto" +# applies quality-first provider sorting, ":online" attaches the web plugin. +# They are never separate catalog entries: /models lists only the base id, so +# every catalog lookup must key on the BASE while the suffixed id stays on the +# wire. +# NOT in this set: ":free", ":batch", ":thinking", ":extended" — those ARE +# distinct catalog SKUs with their own /models entries (and their own context +# windows), so stripping them would resolve the wrong window. +OPENROUTER_VARIANT_SUFFIXES: frozenset[str] = frozenset( + {"nitro", "floor", "exacto", "online"} +) + + +def openrouter_variant_base(model_id: str) -> str | None: + """Return the base model id when ``model_id`` carries a recognized + OpenRouter routing-variant suffix (e.g. ``x-ai/grok-4:nitro`` → + ``x-ai/grok-4``), else ``None``. + + Lives here rather than in ``hermes_cli.models`` so the metadata layer + (``agent.model_metadata``) can share one definition without importing the + CLI — this module is dependency-free by contract. + + >>> openrouter_variant_base("x-ai/grok-4:nitro") + 'x-ai/grok-4' + >>> openrouter_variant_base("x-ai/grok-4:free") is None + True + >>> openrouter_variant_base("x-ai/grok-4") is None + True + """ + base, sep, suffix = (model_id or "").rpartition(":") + if not sep or not base: + return None + if suffix.lower() in OPENROUTER_VARIANT_SUFFIXES: + return base + return None + + AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1" diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 546f1819b2..9ca60830b4 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -1941,3 +1941,142 @@ class TestFallbackWarning: if r.levelno == logging.WARNING and "falling back" in r.getMessage() ] assert len(fallback_warnings) == 0 + + +# ========================================================================= +# get_model_context_length — OpenRouter routing-variant suffixes +# ========================================================================= + +class TestOpenRouterRoutingVariantContextLength: + """OpenRouter's `:nitro`, `:floor`, `:exacto`, `:online` are request-time + routing modifiers, not catalog models — /models lists only the base id and + a variant runs the same model, so it has the SAME context window. + + The validation path (tests/hermes_cli/test_model_validation.py:: + TestValidateOpenRouterVariantSuffixes) already preserves the suffixed id; + this pins the metadata half. Before the fix, `model:nitro` missed every + catalog lookup and fell through to a generic family default — e.g. + `x-ai/grok-4.6:nitro` reported the 131K `grok` catch-all instead of 2M, + silently shrinking the usable window and triggering early compression. + + The invariant asserted here is a relation, not a snapshot: a variant must + resolve to whatever its base resolves to. + """ + + _CATALOG = { + "x-ai/grok-4.6": {"context_length": 2_000_000}, + "anthropic/claude-opus-4.6": {"context_length": 1_000_000}, + } + + _VARIANTS = ["nitro", "floor", "exacto", "online"] + + @pytest.mark.parametrize("suffix", _VARIANTS) + @pytest.mark.parametrize("model", sorted(_CATALOG)) + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_variant_matches_base_context( + self, mock_fetch, mock_models_dev, mock_cache, model, suffix + ): + """A routing variant resolves to exactly its base model's window.""" + mock_fetch.return_value = self._CATALOG + + base_ctx = get_model_context_length(model, provider="openrouter") + variant_ctx = get_model_context_length( + f"{model}:{suffix}", provider="openrouter" + ) + + assert variant_ctx == base_ctx + assert variant_ctx == self._CATALOG[model]["context_length"] + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_variant_does_not_fall_back_to_family_default( + self, mock_fetch, mock_models_dev, mock_cache + ): + """The reported symptom: the generic `grok` catch-all (131K) winning + over the catalog's real window for a `:nitro` id.""" + mock_fetch.return_value = self._CATALOG + + result = get_model_context_length( + "x-ai/grok-4.6:nitro", provider="openrouter" + ) + + assert result == 2_000_000 + assert result != DEFAULT_CONTEXT_LENGTHS.get("grok") + assert result != DEFAULT_FALLBACK_CONTEXT + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_uppercase_suffix_resolves(self, mock_fetch, mock_models_dev, mock_cache): + """Suffix matching is case-insensitive, matching the validation path.""" + mock_fetch.return_value = self._CATALOG + assert get_model_context_length( + "x-ai/grok-4.6:NITRO", provider="openrouter" + ) == 2_000_000 + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_variant_resolves_via_base_url_without_explicit_provider( + self, mock_fetch, mock_models_dev, mock_cache + ): + """Callers that pass only base_url (no provider=) must strip too — + the OpenRouter host is enough to know the suffix is a routing hint.""" + mock_fetch.return_value = self._CATALOG + assert get_model_context_length( + "x-ai/grok-4.6:nitro", + base_url="https://openrouter.ai/api/v1", + ) == 2_000_000 + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_free_sku_suffix_is_not_stripped( + self, mock_fetch, mock_models_dev, mock_cache + ): + """`:free` / `:batch` / `:thinking` ARE distinct catalog SKUs with + their own windows. Stripping them would report the wrong number, so + the SKU's own entry must win over the base's.""" + mock_fetch.return_value = { + "thinkingmachines/inkling": {"context_length": 1_000_000}, + "thinkingmachines/inkling:free": {"context_length": 64_000}, + } + assert get_model_context_length( + "thinkingmachines/inkling:free", provider="openrouter" + ) == 64_000 + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata", return_value={}) + def test_non_openrouter_colon_tag_is_untouched( + self, mock_fetch, mock_models_dev, mock_cache + ): + """An Ollama `model:tag` that happens to end in a variant word must + keep its full id — the carve-out is OpenRouter-only.""" + with patch( + "agent.model_metadata._query_local_context_length", return_value=None + ), patch("agent.model_metadata._query_ollama_api_show", return_value=None): + result = get_model_context_length( + "mymodel:online", + base_url="http://localhost:11434/v1", + ) + assert result == DEFAULT_FALLBACK_CONTEXT + + @patch("agent.model_metadata.get_cached_context_length", return_value=None) + @patch("agent.models_dev.lookup_models_dev_context", return_value=None) + @patch("agent.model_metadata.fetch_model_metadata") + def test_explicit_config_override_still_wins( + self, mock_fetch, mock_models_dev, mock_cache + ): + """Stripping happens after step 0 — a user who pinned the suffixed id + via model.context_length keeps their value (the documented + self-unblock path must not regress).""" + mock_fetch.return_value = self._CATALOG + assert get_model_context_length( + "x-ai/grok-4.6:nitro", + provider="openrouter", + config_context_length=123_456, + ) == 123_456