hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

test(models): trim routing-variant tests to invariants

Browse Source 
One relation test per consumer file: a routed id resolves to its base's
metadata while a real :free SKU keeps its own window.
This commit is contained in:
Teknium
2026-09-11 16:29:59 -07:00
parent 30e4673776
commit 3a2cfb7871
2 changed files with 37 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
tests/agent/test_model_metadata.py
View File

@@ -1948,135 +1948,27 @@ class TestFallbackWarning:
# =========================================================================
class TestOpenRouterRoutingVariantContextLength:
"""OpenRouter's `:nitro`, `:floor`, `:exacto`, `:online` are request-time
routing modifiers, not catalog models — /models lists only the base id and
a variant runs the same model, so it has the SAME context window.
The validation path (tests/hermes_cli/test_model_validation.py::
TestValidateOpenRouterVariantSuffixes) already preserves the suffixed id;
this pins the metadata half. Before the fix, `model:nitro` missed every
catalog lookup and fell through to a generic family default — e.g.
`x-ai/grok-4.6:nitro` reported the 131K `grok` catch-all instead of 2M,
silently shrinking the usable window and triggering early compression.
The invariant asserted here is a relation, not a snapshot: a variant must
resolve to whatever its base resolves to.
"""
"""`:nitro`/`:floor`/`:exacto`/`:online` are request-time routing modifiers, not catalog
models: /models lists only the base id and the variant runs the same model, so a variant
must resolve to whatever its base resolves to instead of a generic family default (#97820).
`:free`/`:batch` are real SKUs with their own windows and must NOT be stripped."""
_CATALOG = {
"x-ai/grok-4.6": {"context_length": 2_000_000},
"anthropic/claude-opus-4.6": {"context_length": 1_000_000},
"thinkingmachines/inkling": {"context_length": 1_000_000},
"thinkingmachines/inkling:free": {"context_length": 64_000},
}
_VARIANTS = ["nitro", "floor", "exacto", "online"]
@pytest.mark.parametrize("suffix", _VARIANTS)
@pytest.mark.parametrize("model", sorted(_CATALOG))
@pytest.mark.parametrize("suffix", ["nitro", "floor", "exacto", "online"])
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_matches_base_context(
self, mock_fetch, mock_models_dev, mock_cache, model, suffix
def test_variant_matches_base_but_real_sku_keeps_own_window(
self, mock_fetch, mock_models_dev, mock_cache, suffix
):
"""A routing variant resolves to exactly its base model's window."""
mock_fetch.return_value = self._CATALOG
base_ctx = get_model_context_length(model, provider="openrouter")
variant_ctx = get_model_context_length(
f"{model}:{suffix}", provider="openrouter"
)
assert variant_ctx == base_ctx
assert variant_ctx == self._CATALOG[model]["context_length"]
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_does_not_fall_back_to_family_default(
self, mock_fetch, mock_models_dev, mock_cache
):
"""The reported symptom: the generic `grok` catch-all (131K) winning
over the catalog's real window for a `:nitro` id."""
mock_fetch.return_value = self._CATALOG
result = get_model_context_length(
"x-ai/grok-4.6:nitro", provider="openrouter"
)
assert result == 2_000_000
assert result != DEFAULT_CONTEXT_LENGTHS.get("grok")
assert result != DEFAULT_FALLBACK_CONTEXT
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_uppercase_suffix_resolves(self, mock_fetch, mock_models_dev, mock_cache):
"""Suffix matching is case-insensitive, matching the validation path."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:NITRO", provider="openrouter"
) == 2_000_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_variant_resolves_via_base_url_without_explicit_provider(
self, mock_fetch, mock_models_dev, mock_cache
):
"""Callers that pass only base_url (no provider=) must strip too —
the OpenRouter host is enough to know the suffix is a routing hint."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:nitro",
base_url="https://openrouter.ai/api/v1",
) == 2_000_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_free_sku_suffix_is_not_stripped(
self, mock_fetch, mock_models_dev, mock_cache
):
"""`:free` / `:batch` / `:thinking` ARE distinct catalog SKUs with
their own windows. Stripping them would report the wrong number, so
the SKU's own entry must win over the base's."""
mock_fetch.return_value = {
"thinkingmachines/inkling": {"context_length": 1_000_000},
"thinkingmachines/inkling:free": {"context_length": 64_000},
}
assert get_model_context_length(
"thinkingmachines/inkling:free", provider="openrouter"
) == 64_000
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata", return_value={})
def test_non_openrouter_colon_tag_is_untouched(
self, mock_fetch, mock_models_dev, mock_cache
):
"""An Ollama `model:tag` that happens to end in a variant word must
keep its full id — the carve-out is OpenRouter-only."""
with patch(
"agent.model_metadata._query_local_context_length", return_value=None
), patch("agent.model_metadata._query_ollama_api_show", return_value=None):
result = get_model_context_length(
"mymodel:online",
base_url="http://localhost:11434/v1",
)
assert result == DEFAULT_FALLBACK_CONTEXT
@patch("agent.model_metadata.get_cached_context_length", return_value=None)
@patch("agent.models_dev.lookup_models_dev_context", return_value=None)
@patch("agent.model_metadata.fetch_model_metadata")
def test_explicit_config_override_still_wins(
self, mock_fetch, mock_models_dev, mock_cache
):
"""Stripping happens after step 0 — a user who pinned the suffixed id
via model.context_length keeps their value (the documented
self-unblock path must not regress)."""
mock_fetch.return_value = self._CATALOG
assert get_model_context_length(
"x-ai/grok-4.6:nitro",
provider="openrouter",
config_context_length=123_456,
) == 123_456
base_ctx = get_model_context_length("x-ai/grok-4.6", provider="openrouter")
variant_ctx = get_model_context_length(f"x-ai/grok-4.6:{suffix}", provider="openrouter")
assert variant_ctx == base_ctx == 2_000_000
assert variant_ctx != DEFAULT_CONTEXT_LENGTHS.get("grok")
assert get_model_context_length("thinkingmachines/inkling:free", provider="openrouter") == 64_000

144
tests/agent/test_models_dev.py
View File

@@ -1342,24 +1342,12 @@ class TestModelOverrides:
# =========================================================================
class TestOpenRouterRoutingVariantCatalogLookup:
"""OpenRouter's `:nitro`/`:floor`/`:exacto`/`:online` are request-time
routing modifiers, not catalog models. models.dev (like OpenRouter's own
/models) lists only the base id, so every catalog consumer must resolve a
routed id to the base model's metadata while the suffixed id stays on the
wire. See issue #97820.
"""models.dev, like OpenRouter's /models, lists only the base id of a routed
`:nitro`/`:floor`/`:exacto`/`:online` model, so every catalog consumer resolves the base's
metadata for it (#97820). `:free` is a real SKU whose window may differ from its base
(z-ai/glm-5.2 1.05M vs :free 256K) — stripping it would over-report the window and fail
at the API, so it keeps exact-match semantics and an absent SKU still misses."""
The counterpart to the context-length half of this fix lives in
tests/agent/test_model_metadata.py::TestOpenRouterRoutingVariantContextLength.
Critically, `:free` and `:batch` are NOT routing variants — they are real
catalog SKUs with their own entries and sometimes their own context
windows. Stripping them would report a window LARGER than the model has,
which fails at the API rather than merely compacting early.
"""
#: Base model plus a `:free` SKU whose window deliberately DIFFERS from
#: it — mirrors real catalog entries such as z-ai/glm-5.2 (1.05M) vs
#: z-ai/glm-5.2:free (256K).
REGISTRY = {
"openrouter": {
"id": "openrouter",
@@ -1370,112 +1358,26 @@ class TestOpenRouterRoutingVariantCatalogLookup:
"tool_call": True,
"reasoning": True,
},
"z-ai/glm-5.2": {
"id": "z-ai/glm-5.2",
"limit": {"context": 1048576, "output": 131072},
},
"z-ai/glm-5.2:free": {
"id": "z-ai/glm-5.2:free",
"limit": {"context": 256000, "output": 131072},
},
"z-ai/glm-5.2": {"id": "z-ai/glm-5.2", "limit": {"context": 1048576, "output": 131072}},
"z-ai/glm-5.2:free": {"id": "z-ai/glm-5.2:free", "limit": {"context": 256000, "output": 131072}},
},
},
}
ROUTING_SUFFIXES = ["nitro", "floor", "exacto", "online"]
@pytest.mark.parametrize("suffix", ["nitro", "floor", "exacto", "online"])
def test_routed_id_matches_base_across_consumers(self, suffix):
with patch("agent.models_dev.fetch_models_dev", return_value=self.REGISTRY):
routed = f"z-ai/glm-5.3-flash:{suffix}"
assert lookup_models_dev_context("openrouter", routed) == 1310720
base_caps = get_model_capabilities("openrouter", "z-ai/glm-5.3-flash")
routed_caps = get_model_capabilities("openrouter", routed)
assert routed_caps.context_window == base_caps.context_window == 1310720
assert routed_caps.supports_tools == base_caps.supports_tools
assert get_model_info("openrouter", routed).context_window == 1310720
# Other providers' colon tags keep exact-match semantics.
assert lookup_models_dev_context("anthropic", f"claude-x:{suffix}") is None
def _patch(self):
return patch(
"agent.models_dev.fetch_models_dev", return_value=self.REGISTRY
)
@pytest.mark.parametrize("suffix", ROUTING_SUFFIXES)
def test_context_lookup_matches_base(self, suffix):
"""A routed id resolves to exactly its base model's context."""
with self._patch():
base = lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash")
routed = lookup_models_dev_context(
"openrouter", f"z-ai/glm-5.3-flash:{suffix}"
)
assert routed == base == 1310720
@pytest.mark.parametrize("suffix", ROUTING_SUFFIXES)
def test_capabilities_match_base(self, suffix):
"""get_model_capabilities resolves routed ids (not just context)."""
with self._patch():
base = get_model_capabilities("openrouter", "z-ai/glm-5.3-flash")
routed = get_model_capabilities(
"openrouter", f"z-ai/glm-5.3-flash:{suffix}"
)
assert routed is not None
assert routed.context_window == base.context_window == 1310720
assert routed.supports_tools == base.supports_tools
assert routed.supports_reasoning == base.supports_reasoning
@pytest.mark.parametrize("suffix", ROUTING_SUFFIXES)
def test_model_info_matches_base(self, suffix):
"""get_model_info resolves routed ids."""
with self._patch():
base = get_model_info("openrouter", "z-ai/glm-5.3-flash")
routed = get_model_info("openrouter", f"z-ai/glm-5.3-flash:{suffix}")
assert routed is not None
assert routed.context_window == base.context_window == 1310720
def test_suffix_match_is_case_insensitive(self):
with self._patch():
assert (
lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:FLOOR")
== 1310720
)
def test_real_free_sku_keeps_its_own_window(self):
"""REGRESSION GUARD: `:free` is a real SKU, not a routing variant.
z-ai/glm-5.2:free is 256K while its base is 1.05M. Treating `:free`
as strippable would report 1.05M — a window the model does not have,
so the request fails at the API instead of compacting early. The SKU's
own entry must win.
"""
with self._patch():
assert (
lookup_models_dev_context("openrouter", "z-ai/glm-5.2:free") == 256000
)
assert lookup_models_dev_context("openrouter", "z-ai/glm-5.2") == 1048576
def test_absent_sku_suffix_still_misses(self):
"""A `:free` id with no catalog entry must MISS rather than fall back
to its base — otherwise model_overrides `_default` fill-gap semantics
break, and an absent free tier would inherit the paid tier's window."""
with self._patch():
assert (
lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:free")
is None
)
def test_non_openrouter_provider_is_never_stripped(self):
"""The carve-out is OpenRouter-only; another provider's colon-suffixed
id (an Ollama tag, a `:cloud` key) keeps exact-match semantics."""
registry = {
"anthropic": {
"id": "anthropic",
"models": {"claude-x": {"limit": {"context": 200000}}},
},
}
with patch("agent.models_dev.fetch_models_dev", return_value=registry):
assert lookup_models_dev_context("anthropic", "claude-x:floor") is None
assert lookup_models_dev_context("anthropic", "claude-x") == 200000
def test_unknown_suffix_is_not_stripped(self):
with self._patch():
assert (
lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:bogus")
is None
)
def test_bare_model_lookup_unaffected(self):
"""No suffix — behavior is byte-identical to before the change."""
with self._patch():
assert (
lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash") == 1310720
)
assert lookup_models_dev_context("openrouter", "nope/missing") is None
def test_real_sku_suffix_is_not_stripped(self):
with patch("agent.models_dev.fetch_models_dev", return_value=self.REGISTRY):
assert lookup_models_dev_context("openrouter", "z-ai/glm-5.2:free") == 256000
assert lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:free") is None