diff --git a/agent/model_metadata.py b/agent/model_metadata.py index c3a0fc4b41..7655bb2a9b 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -2347,6 +2347,51 @@ _CODEX_OAUTH_CONTEXT_FALLBACK: Dict[str, int] = { "gpt-5": 272_000, } +# Codex OAuth advertises 272K via /backend-api/codex/models for these +# families, but the backend actually ACCEPTS more (verified live Aug 16 2026 +# against chatgpt.com/backend-api/codex/responses: ~371K input tokens +# completed OK for gpt-5.6-sol/terra/luna and gpt-5.4; ~382K+ rejected with +# ``context_length_exceeded``; gpt-5.5 rejected 360K, so its 272K +# advertisement is real and it is NOT listed). 350K keeps ~22K margin under +# the observed ~372K enforcement. +# +# Applied ONLY when the resolved value (live probe or fallback table) is +# exactly the known-stale 272,000 advertisement — if OpenAI moves the +# advertised number in either direction (the gpt-5.6 family shifted +# 272K → 372K → 272K during July 2026), the catalog is trusted again and +# this table is inert. ``gpt-5.6`` is a FAMILY PREFIX (sol/terra/luna and +# dated snapshots; ``-pro`` slugs are not routable on Codex OAuth — the +# backend 400s them — so over-matching there is moot). ``gpt-5.4`` is EXACT: +# gpt-5.4-mini was probed and genuinely enforces 272K (rejected 360K), so +# prefix-matching the 5.4 family would over-report for mini. +_CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES: Dict[str, int] = { + "gpt-5.6": 350_000, # sol / terra / luna — all three verified live +} +_CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT: Dict[str, int] = { + "gpt-5.4": 350_000, # verified live; gpt-5.4-mini rejected 360K — excluded +} + +# The advertised value the verified-above table is allowed to override. +_CODEX_OAUTH_STALE_ADVERTISED_CTX = 272_000 + + +def _verified_codex_ctx_for_slug(model_bare: str) -> Optional[int]: + """Return the live-verified Codex cap for a slug, or ``None``. + + Exact slugs first, then family prefixes (````, ``-``, + ``.``) so dated snapshots of a verified family inherit the bump. + """ + slug = (model_bare or "").strip().lower() + if not slug: + return None + exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(slug) + if exact is not None: + return exact + for key, ctx in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES.items(): + if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."): + return ctx + return None + _codex_oauth_context_cache: Dict[str, Tuple[Dict[str, int], float]] = {} _CODEX_OAUTH_CONTEXT_CACHE_TTL = 3600 # 1 hour @@ -2474,16 +2519,33 @@ def _resolve_codex_oauth_context_length_with_source( if not model_bare: return None, "" + def _apply_verified_bump(ctx: int, source: str) -> Tuple[int, str]: + """Lift a known-stale 272K advertisement to the live-verified cap. + + Only fires when the resolved value is EXACTLY the stale 272,000 + advertisement for a slug we have probed above it (see + ``_verified_codex_ctx_for_slug``). Any other advertised value — + higher or lower — is trusted as a real server-side change. + """ + bumped = _verified_codex_ctx_for_slug(model_bare) + if bumped is not None and ctx == _CODEX_OAUTH_STALE_ADVERTISED_CTX: + logger.debug( + "Codex OAuth context for %s: advertised %d raised to " + "live-verified %d", model_bare, ctx, bumped, + ) + return bumped, source + return ctx, source + if access_token: live, fresh_probe = _fetch_codex_oauth_context_lengths_with_source(access_token) live_source = "live" if fresh_probe else "memory" if model_bare in live: - return live[model_bare], live_source + return _apply_verified_bump(live[model_bare], live_source) # Case-insensitive match in case casing drifts model_lower = model_bare.lower() for slug, ctx in live.items(): if slug.lower() == model_lower: - return ctx, live_source + return _apply_verified_bump(ctx, live_source) # Fallback: longest-key-first substring match over hardcoded defaults. model_lower = model_bare.lower() @@ -2491,7 +2553,7 @@ def _resolve_codex_oauth_context_length_with_source( _CODEX_OAUTH_CONTEXT_FALLBACK.items(), key=lambda x: len(x[0]), reverse=True ): if slug in model_lower: - return ctx, "fallback" + return _apply_verified_bump(ctx, "fallback") return None, "" diff --git a/tests/agent/test_model_metadata.py b/tests/agent/test_model_metadata.py index 3ab8dc0509..6c026c3453 100644 --- a/tests/agent/test_model_metadata.py +++ b/tests/agent/test_model_metadata.py @@ -401,12 +401,12 @@ class TestCodexOAuthContextLength: first_response = MagicMock() first_response.status_code = 200 first_response.json.return_value = { - "models": [{"slug": "gpt-5.6-terra", "context_window": 272_000}] + "models": [{"slug": "gpt-5.5", "context_window": 272_000}] } second_response = MagicMock() second_response.status_code = 200 second_response.json.return_value = { - "models": [{"slug": "gpt-5.6-terra", "context_window": 372_000}] + "models": [{"slug": "gpt-5.5", "context_window": 372_000}] } with patch( @@ -414,19 +414,19 @@ class TestCodexOAuthContextLength: side_effect=[first_response, second_response], ) as mock_get, patch("agent.model_metadata.save_context_length") as mock_save: first = get_model_context_length( - "gpt-5.6-terra", + "gpt-5.5", base_url="https://chatgpt.com/backend-api/codex", api_key="token-account-a", provider="openai-codex", ) first_again = get_model_context_length( - "gpt-5.6-terra", + "gpt-5.5", base_url="https://chatgpt.com/backend-api/codex", api_key="token-account-a", provider="openai-codex", ) second = get_model_context_length( - "gpt-5.6-terra", + "gpt-5.5", base_url="https://chatgpt.com/backend-api/codex", api_key="token-account-b", provider="openai-codex", @@ -478,7 +478,7 @@ class TestCodexOAuthContextLength: monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file) base_url = "https://chatgpt.com/backend-api/codex" - stale_key = f"gpt-5.6-terra@{base_url}" + stale_key = f"gpt-5.5@{base_url}" other_key = "other-model@https://api.openai.com/v1/" import yaml as _yaml cache_file.write_text(_yaml.dump({"context_lengths": { @@ -489,14 +489,14 @@ class TestCodexOAuthContextLength: fake_response = MagicMock() fake_response.status_code = 200 fake_response.json.return_value = { - "models": [{"slug": "gpt-5.6-terra", "context_window": live_context}] + "models": [{"slug": "gpt-5.5", "context_window": live_context}] } # Exercise real persistence here: this test verifies that a live value # replaces the stale on-disk entry. Failure-path tests below mock the # writer because they assert that fallback values are not persisted. with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get: ctx = mm.get_model_context_length( - model="gpt-5.6-terra", + model="gpt-5.5", base_url=base_url, api_key="fake-token", provider="openai-codex", @@ -510,6 +510,103 @@ class TestCodexOAuthContextLength: assert remaining.get(stale_key) == live_context assert remaining.get(other_key) == 128_000 + @pytest.mark.parametrize( + "slug", + [ + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.6-luna", + "gpt-5.6-sol-2026-07-09", # dated snapshot via gpt-5.6 family prefix + "gpt-5.4", + ], + ) + def test_stale_272k_advertisement_bumped_to_live_verified_350k(self, slug): + """Codex advertises 272K for these slugs but the backend accepts ~372K + (verified live Aug 2026); the resolver lifts exactly-272K to 350K.""" + from agent.model_metadata import get_model_context_length + + fake_response = MagicMock() + fake_response.status_code = 200 + fake_response.json.return_value = { + "models": [{"slug": slug, "context_window": 272_000}] + } + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ + patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.save_context_length"): + ctx = get_model_context_length( + model=slug, + base_url="https://chatgpt.com/backend-api/codex", + api_key="fake-token", + provider="openai-codex", + ) + assert ctx == 350_000 + + def test_non_272k_advertisement_is_trusted_verbatim(self): + """Any advertised value other than the known-stale 272,000 — higher or + lower — is a real server-side change and must NOT be overridden.""" + from agent.model_metadata import get_model_context_length + + for advertised in (372_000, 200_000, 1_050_000): + fake_response = MagicMock() + fake_response.status_code = 200 + fake_response.json.return_value = { + "models": [{"slug": "gpt-5.6-sol", "context_window": advertised}] + } + import agent.model_metadata as mm + mm._codex_oauth_context_cache = {} + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ + patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.save_context_length"): + ctx = get_model_context_length( + model="gpt-5.6-sol", + base_url="https://chatgpt.com/backend-api/codex", + api_key="fake-token", + provider="openai-codex", + ) + assert ctx == advertised, f"advertised {advertised} must be trusted" + + @pytest.mark.parametrize("slug", ["gpt-5.5", "gpt-5.4-mini"]) + def test_slugs_that_enforce_272k_keep_advertised_value(self, slug): + """gpt-5.5 and gpt-5.4-mini both rejected 360K in the live probe — + their 272K advertisement is real enforcement, so no bump applies + (gpt-5.4 is an exact-match entry precisely to exclude -mini).""" + from agent.model_metadata import get_model_context_length + + fake_response = MagicMock() + fake_response.status_code = 200 + fake_response.json.return_value = { + "models": [{"slug": slug, "context_window": 272_000}] + } + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ + patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.save_context_length"): + ctx = get_model_context_length( + model=slug, + base_url="https://chatgpt.com/backend-api/codex", + api_key="fake-token", + provider="openai-codex", + ) + assert ctx == 272_000 + + def test_fallback_table_resolution_also_bumped(self): + """When the live probe fails, the 272K fallback-table value for a + verified slug is bumped the same way (same enforcement applies).""" + from agent.model_metadata import get_model_context_length + + fake_response = MagicMock() + fake_response.status_code = 401 + fake_response.json.return_value = {} + with patch("agent.model_metadata.requests.get", return_value=fake_response), \ + patch("agent.model_metadata.get_cached_context_length", return_value=None), \ + patch("agent.model_metadata.save_context_length"): + ctx = get_model_context_length( + model="gpt-5.6-sol", + base_url="https://chatgpt.com/backend-api/codex", + api_key="expired-token", + provider="openai-codex", + ) + assert ctx == 350_000 +