diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 9ca60830b4..485b892f03 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -1948,135 +1948,27 @@ class TestFallbackWarning: # ========================================================================= class TestOpenRouterRoutingVariantContextLength: - """OpenRouter's `:nitro`, `:floor`, `:exacto`, `:online` are request-time - routing modifiers, not catalog models — /models lists only the base id and - a variant runs the same model, so it has the SAME context window. - - The validation path (tests/hermes_cli/test_model_validation.py:: - TestValidateOpenRouterVariantSuffixes) already preserves the suffixed id; - this pins the metadata half. Before the fix, `model:nitro` missed every - catalog lookup and fell through to a generic family default — e.g. - `x-ai/grok-4.6:nitro` reported the 131K `grok` catch-all instead of 2M, - silently shrinking the usable window and triggering early compression. - - The invariant asserted here is a relation, not a snapshot: a variant must - resolve to whatever its base resolves to. - """ + """`:nitro`/`:floor`/`:exacto`/`:online` are request-time routing modifiers, not catalog + models: /models lists only the base id and the variant runs the same model, so a variant + must resolve to whatever its base resolves to instead of a generic family default (#97820). + `:free`/`:batch` are real SKUs with their own windows and must NOT be stripped.""" _CATALOG = { "x-ai/grok-4.6": {"context_length": 2_000_000}, - "anthropic/claude-opus-4.6": {"context_length": 1_000_000}, + "thinkingmachines/inkling": {"context_length": 1_000_000}, + "thinkingmachines/inkling:free": {"context_length": 64_000}, } - _VARIANTS = ["nitro", "floor", "exacto", "online"] - - @pytest.mark.parametrize("suffix", _VARIANTS) - @pytest.mark.parametrize("model", sorted(_CATALOG)) + @pytest.mark.parametrize("suffix", ["nitro", "floor", "exacto", "online"]) @patch("agent.model_metadata.get_cached_context_length", return_value=None) @patch("agent.models_dev.lookup_models_dev_context", return_value=None) @patch("agent.model_metadata.fetch_model_metadata") - def test_variant_matches_base_context( - self, mock_fetch, mock_models_dev, mock_cache, model, suffix + def test_variant_matches_base_but_real_sku_keeps_own_window( + self, mock_fetch, mock_models_dev, mock_cache, suffix ): - """A routing variant resolves to exactly its base model's window.""" mock_fetch.return_value = self._CATALOG - - base_ctx = get_model_context_length(model, provider="openrouter") - variant_ctx = get_model_context_length( - f"{model}:{suffix}", provider="openrouter" - ) - - assert variant_ctx == base_ctx - assert variant_ctx == self._CATALOG[model]["context_length"] - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata") - def test_variant_does_not_fall_back_to_family_default( - self, mock_fetch, mock_models_dev, mock_cache - ): - """The reported symptom: the generic `grok` catch-all (131K) winning - over the catalog's real window for a `:nitro` id.""" - mock_fetch.return_value = self._CATALOG - - result = get_model_context_length( - "x-ai/grok-4.6:nitro", provider="openrouter" - ) - - assert result == 2_000_000 - assert result != DEFAULT_CONTEXT_LENGTHS.get("grok") - assert result != DEFAULT_FALLBACK_CONTEXT - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata") - def test_uppercase_suffix_resolves(self, mock_fetch, mock_models_dev, mock_cache): - """Suffix matching is case-insensitive, matching the validation path.""" - mock_fetch.return_value = self._CATALOG - assert get_model_context_length( - "x-ai/grok-4.6:NITRO", provider="openrouter" - ) == 2_000_000 - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata") - def test_variant_resolves_via_base_url_without_explicit_provider( - self, mock_fetch, mock_models_dev, mock_cache - ): - """Callers that pass only base_url (no provider=) must strip too — - the OpenRouter host is enough to know the suffix is a routing hint.""" - mock_fetch.return_value = self._CATALOG - assert get_model_context_length( - "x-ai/grok-4.6:nitro", - base_url="https://openrouter.ai/api/v1", - ) == 2_000_000 - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata") - def test_free_sku_suffix_is_not_stripped( - self, mock_fetch, mock_models_dev, mock_cache - ): - """`:free` / `:batch` / `:thinking` ARE distinct catalog SKUs with - their own windows. Stripping them would report the wrong number, so - the SKU's own entry must win over the base's.""" - mock_fetch.return_value = { - "thinkingmachines/inkling": {"context_length": 1_000_000}, - "thinkingmachines/inkling:free": {"context_length": 64_000}, - } - assert get_model_context_length( - "thinkingmachines/inkling:free", provider="openrouter" - ) == 64_000 - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata", return_value={}) - def test_non_openrouter_colon_tag_is_untouched( - self, mock_fetch, mock_models_dev, mock_cache - ): - """An Ollama `model:tag` that happens to end in a variant word must - keep its full id — the carve-out is OpenRouter-only.""" - with patch( - "agent.model_metadata._query_local_context_length", return_value=None - ), patch("agent.model_metadata._query_ollama_api_show", return_value=None): - result = get_model_context_length( - "mymodel:online", - base_url="http://localhost:11434/v1", - ) - assert result == DEFAULT_FALLBACK_CONTEXT - - @patch("agent.model_metadata.get_cached_context_length", return_value=None) - @patch("agent.models_dev.lookup_models_dev_context", return_value=None) - @patch("agent.model_metadata.fetch_model_metadata") - def test_explicit_config_override_still_wins( - self, mock_fetch, mock_models_dev, mock_cache - ): - """Stripping happens after step 0 — a user who pinned the suffixed id - via model.context_length keeps their value (the documented - self-unblock path must not regress).""" - mock_fetch.return_value = self._CATALOG - assert get_model_context_length( - "x-ai/grok-4.6:nitro", - provider="openrouter", - config_context_length=123_456, - ) == 123_456 + base_ctx = get_model_context_length("x-ai/grok-4.6", provider="openrouter") + variant_ctx = get_model_context_length(f"x-ai/grok-4.6:{suffix}", provider="openrouter") + assert variant_ctx == base_ctx == 2_000_000 + assert variant_ctx != DEFAULT_CONTEXT_LENGTHS.get("grok") + assert get_model_context_length("thinkingmachines/inkling:free", provider="openrouter") == 64_000 diff --git a/tests/agent/test_models_dev.py b/tests/agent/test_models_dev.py index 3b60133235..180fdac845 100644 --- a/tests/agent/test_models_dev.py +++ b/tests/agent/test_models_dev.py @@ -1342,24 +1342,12 @@ class TestModelOverrides: # ========================================================================= class TestOpenRouterRoutingVariantCatalogLookup: - """OpenRouter's `:nitro`/`:floor`/`:exacto`/`:online` are request-time - routing modifiers, not catalog models. models.dev (like OpenRouter's own - /models) lists only the base id, so every catalog consumer must resolve a - routed id to the base model's metadata while the suffixed id stays on the - wire. See issue #97820. + """models.dev, like OpenRouter's /models, lists only the base id of a routed + `:nitro`/`:floor`/`:exacto`/`:online` model, so every catalog consumer resolves the base's + metadata for it (#97820). `:free` is a real SKU whose window may differ from its base + (z-ai/glm-5.2 1.05M vs :free 256K) — stripping it would over-report the window and fail + at the API, so it keeps exact-match semantics and an absent SKU still misses.""" - The counterpart to the context-length half of this fix lives in - tests/agent/test_model_metadata.py::TestOpenRouterRoutingVariantContextLength. - - Critically, `:free` and `:batch` are NOT routing variants — they are real - catalog SKUs with their own entries and sometimes their own context - windows. Stripping them would report a window LARGER than the model has, - which fails at the API rather than merely compacting early. - """ - - #: Base model plus a `:free` SKU whose window deliberately DIFFERS from - #: it — mirrors real catalog entries such as z-ai/glm-5.2 (1.05M) vs - #: z-ai/glm-5.2:free (256K). REGISTRY = { "openrouter": { "id": "openrouter", @@ -1370,112 +1358,26 @@ class TestOpenRouterRoutingVariantCatalogLookup: "tool_call": True, "reasoning": True, }, - "z-ai/glm-5.2": { - "id": "z-ai/glm-5.2", - "limit": {"context": 1048576, "output": 131072}, - }, - "z-ai/glm-5.2:free": { - "id": "z-ai/glm-5.2:free", - "limit": {"context": 256000, "output": 131072}, - }, + "z-ai/glm-5.2": {"id": "z-ai/glm-5.2", "limit": {"context": 1048576, "output": 131072}}, + "z-ai/glm-5.2:free": {"id": "z-ai/glm-5.2:free", "limit": {"context": 256000, "output": 131072}}, }, }, } - ROUTING_SUFFIXES = ["nitro", "floor", "exacto", "online"] + @pytest.mark.parametrize("suffix", ["nitro", "floor", "exacto", "online"]) + def test_routed_id_matches_base_across_consumers(self, suffix): + with patch("agent.models_dev.fetch_models_dev", return_value=self.REGISTRY): + routed = f"z-ai/glm-5.3-flash:{suffix}" + assert lookup_models_dev_context("openrouter", routed) == 1310720 + base_caps = get_model_capabilities("openrouter", "z-ai/glm-5.3-flash") + routed_caps = get_model_capabilities("openrouter", routed) + assert routed_caps.context_window == base_caps.context_window == 1310720 + assert routed_caps.supports_tools == base_caps.supports_tools + assert get_model_info("openrouter", routed).context_window == 1310720 + # Other providers' colon tags keep exact-match semantics. + assert lookup_models_dev_context("anthropic", f"claude-x:{suffix}") is None - def _patch(self): - return patch( - "agent.models_dev.fetch_models_dev", return_value=self.REGISTRY - ) - - @pytest.mark.parametrize("suffix", ROUTING_SUFFIXES) - def test_context_lookup_matches_base(self, suffix): - """A routed id resolves to exactly its base model's context.""" - with self._patch(): - base = lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash") - routed = lookup_models_dev_context( - "openrouter", f"z-ai/glm-5.3-flash:{suffix}" - ) - assert routed == base == 1310720 - - @pytest.mark.parametrize("suffix", ROUTING_SUFFIXES) - def test_capabilities_match_base(self, suffix): - """get_model_capabilities resolves routed ids (not just context).""" - with self._patch(): - base = get_model_capabilities("openrouter", "z-ai/glm-5.3-flash") - routed = get_model_capabilities( - "openrouter", f"z-ai/glm-5.3-flash:{suffix}" - ) - assert routed is not None - assert routed.context_window == base.context_window == 1310720 - assert routed.supports_tools == base.supports_tools - assert routed.supports_reasoning == base.supports_reasoning - - @pytest.mark.parametrize("suffix", ROUTING_SUFFIXES) - def test_model_info_matches_base(self, suffix): - """get_model_info resolves routed ids.""" - with self._patch(): - base = get_model_info("openrouter", "z-ai/glm-5.3-flash") - routed = get_model_info("openrouter", f"z-ai/glm-5.3-flash:{suffix}") - assert routed is not None - assert routed.context_window == base.context_window == 1310720 - - def test_suffix_match_is_case_insensitive(self): - with self._patch(): - assert ( - lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:FLOOR") - == 1310720 - ) - - def test_real_free_sku_keeps_its_own_window(self): - """REGRESSION GUARD: `:free` is a real SKU, not a routing variant. - - z-ai/glm-5.2:free is 256K while its base is 1.05M. Treating `:free` - as strippable would report 1.05M — a window the model does not have, - so the request fails at the API instead of compacting early. The SKU's - own entry must win. - """ - with self._patch(): - assert ( - lookup_models_dev_context("openrouter", "z-ai/glm-5.2:free") == 256000 - ) - assert lookup_models_dev_context("openrouter", "z-ai/glm-5.2") == 1048576 - - def test_absent_sku_suffix_still_misses(self): - """A `:free` id with no catalog entry must MISS rather than fall back - to its base — otherwise model_overrides `_default` fill-gap semantics - break, and an absent free tier would inherit the paid tier's window.""" - with self._patch(): - assert ( - lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:free") - is None - ) - - def test_non_openrouter_provider_is_never_stripped(self): - """The carve-out is OpenRouter-only; another provider's colon-suffixed - id (an Ollama tag, a `:cloud` key) keeps exact-match semantics.""" - registry = { - "anthropic": { - "id": "anthropic", - "models": {"claude-x": {"limit": {"context": 200000}}}, - }, - } - with patch("agent.models_dev.fetch_models_dev", return_value=registry): - assert lookup_models_dev_context("anthropic", "claude-x:floor") is None - assert lookup_models_dev_context("anthropic", "claude-x") == 200000 - - def test_unknown_suffix_is_not_stripped(self): - with self._patch(): - assert ( - lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:bogus") - is None - ) - - def test_bare_model_lookup_unaffected(self): - """No suffix — behavior is byte-identical to before the change.""" - with self._patch(): - assert ( - lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash") == 1310720 - ) - assert lookup_models_dev_context("openrouter", "nope/missing") is None + def test_real_sku_suffix_is_not_stripped(self): + with patch("agent.models_dev.fetch_models_dev", return_value=self.REGISTRY): + assert lookup_models_dev_context("openrouter", "z-ai/glm-5.2:free") == 256000 + assert lookup_models_dev_context("openrouter", "z-ai/glm-5.3-flash:free") is None