hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

feat: raise Codex OAuth context to live-verified 350K for gpt-5.6 family and gpt-5.4

Browse Source 
The Codex /models catalog advertises 272K for the gpt-5.6 (sol/terra/luna)
and gpt-5.4 slugs, but the backend actually accepts ~371K input tokens
(verified live against chatgpt.com/backend-api/codex/responses, Aug 16 2026:
~371K completed OK on all four slugs; ~382K+ rejected with
context_length_exceeded). 350K keeps ~22K margin under the observed ~372K
enforcement.

The bump applies ONLY when the resolved value is exactly the known-stale
272,000 advertisement — any other advertised value (higher or lower) is
trusted as a real server-side change, so a future catalog correction
deactivates the override automatically. gpt-5.5 and gpt-5.4-mini both
genuinely enforce 272K (rejected 360K live) and are excluded.
This commit is contained in:
Teknium
2026-08-16 15:35:03 -07:00
parent 86b2057a1b
commit 5229975438
2 changed files with 170 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
agent/model_metadata.py
View File

@@ -2347,6 +2347,51 @@ _CODEX_OAUTH_CONTEXT_FALLBACK: Dict[str, int] = {
"gpt-5": 272_000,
}
# Codex OAuth advertises 272K via /backend-api/codex/models for these
# families, but the backend actually ACCEPTS more (verified live Aug 16 2026
# against chatgpt.com/backend-api/codex/responses: ~371K input tokens
# completed OK for gpt-5.6-sol/terra/luna and gpt-5.4; ~382K+ rejected with
# ``context_length_exceeded``; gpt-5.5 rejected 360K, so its 272K
# advertisement is real and it is NOT listed). 350K keeps ~22K margin under
# the observed ~372K enforcement.
#
# Applied ONLY when the resolved value (live probe or fallback table) is
# exactly the known-stale 272,000 advertisement — if OpenAI moves the
# advertised number in either direction (the gpt-5.6 family shifted
# 272K → 372K → 272K during July 2026), the catalog is trusted again and
# this table is inert. ``gpt-5.6`` is a FAMILY PREFIX (sol/terra/luna and
# dated snapshots; ``-pro`` slugs are not routable on Codex OAuth — the
# backend 400s them — so over-matching there is moot). ``gpt-5.4`` is EXACT:
# gpt-5.4-mini was probed and genuinely enforces 272K (rejected 360K), so
# prefix-matching the 5.4 family would over-report for mini.
_CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES: Dict[str, int] = {
"gpt-5.6": 350_000, # sol / terra / luna — all three verified live
}
_CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT: Dict[str, int] = {
"gpt-5.4": 350_000, # verified live; gpt-5.4-mini rejected 360K — excluded
}
# The advertised value the verified-above table is allowed to override.
_CODEX_OAUTH_STALE_ADVERTISED_CTX = 272_000
def _verified_codex_ctx_for_slug(model_bare: str) -> Optional[int]:
"""Return the live-verified Codex cap for a slug, or ``None``.
Exact slugs first, then family prefixes (``<key>``, ``<key>-``,
``<key>.``) so dated snapshots of a verified family inherit the bump.
"""
slug = (model_bare or "").strip().lower()
if not slug:
return None
exact = _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_EXACT.get(slug)
if exact is not None:
return exact
for key, ctx in _CODEX_OAUTH_VERIFIED_ABOVE_ADVERTISED_PREFIXES.items():
if slug == key or slug.startswith(key + "-") or slug.startswith(key + "."):
return ctx
return None
_codex_oauth_context_cache: Dict[str, Tuple[Dict[str, int], float]] = {}
_CODEX_OAUTH_CONTEXT_CACHE_TTL = 3600 # 1 hour
@@ -2474,16 +2519,33 @@ def _resolve_codex_oauth_context_length_with_source(
if not model_bare:
return None, ""
def _apply_verified_bump(ctx: int, source: str) -> Tuple[int, str]:
"""Lift a known-stale 272K advertisement to the live-verified cap.
Only fires when the resolved value is EXACTLY the stale 272,000
advertisement for a slug we have probed above it (see
``_verified_codex_ctx_for_slug``). Any other advertised value —
higher or lower — is trusted as a real server-side change.
"""
bumped = _verified_codex_ctx_for_slug(model_bare)
if bumped is not None and ctx == _CODEX_OAUTH_STALE_ADVERTISED_CTX:
logger.debug(
"Codex OAuth context for %s: advertised %d raised to "
"live-verified %d", model_bare, ctx, bumped,
)
return bumped, source
return ctx, source
if access_token:
live, fresh_probe = _fetch_codex_oauth_context_lengths_with_source(access_token)
live_source = "live" if fresh_probe else "memory"
if model_bare in live:
return live[model_bare], live_source
return _apply_verified_bump(live[model_bare], live_source)
# Case-insensitive match in case casing drifts
model_lower = model_bare.lower()
for slug, ctx in live.items():
if slug.lower() == model_lower:
return ctx, live_source
return _apply_verified_bump(ctx, live_source)
# Fallback: longest-key-first substring match over hardcoded defaults.
model_lower = model_bare.lower()
@@ -2491,7 +2553,7 @@ def _resolve_codex_oauth_context_length_with_source(
_CODEX_OAUTH_CONTEXT_FALLBACK.items(), key=lambda x: len(x[0]), reverse=True
):
if slug in model_lower:
return ctx, "fallback"
return _apply_verified_bump(ctx, "fallback")
return None, ""

113
tests/agent/test_model_metadata.py
View File

@@ -401,12 +401,12 @@ class TestCodexOAuthContextLength:
first_response = MagicMock()
first_response.status_code = 200
first_response.json.return_value = {
"models": [{"slug": "gpt-5.6-terra", "context_window": 272_000}]
"models": [{"slug": "gpt-5.5", "context_window": 272_000}]
}
second_response = MagicMock()
second_response.status_code = 200
second_response.json.return_value = {
"models": [{"slug": "gpt-5.6-terra", "context_window": 372_000}]
"models": [{"slug": "gpt-5.5", "context_window": 372_000}]
}
with patch(
@@ -414,19 +414,19 @@ class TestCodexOAuthContextLength:
side_effect=[first_response, second_response],
) as mock_get, patch("agent.model_metadata.save_context_length") as mock_save:
first = get_model_context_length(
"gpt-5.6-terra",
"gpt-5.5",
base_url="https://chatgpt.com/backend-api/codex",
api_key="token-account-a",
provider="openai-codex",
)
first_again = get_model_context_length(
"gpt-5.6-terra",
"gpt-5.5",
base_url="https://chatgpt.com/backend-api/codex",
api_key="token-account-a",
provider="openai-codex",
)
second = get_model_context_length(
"gpt-5.6-terra",
"gpt-5.5",
base_url="https://chatgpt.com/backend-api/codex",
api_key="token-account-b",
provider="openai-codex",
@@ -478,7 +478,7 @@ class TestCodexOAuthContextLength:
monkeypatch.setattr(mm, "_get_context_cache_path", lambda: cache_file)
base_url = "https://chatgpt.com/backend-api/codex"
stale_key = f"gpt-5.6-terra@{base_url}"
stale_key = f"gpt-5.5@{base_url}"
other_key = "other-model@https://api.openai.com/v1/"
import yaml as _yaml
cache_file.write_text(_yaml.dump({"context_lengths": {
@@ -489,14 +489,14 @@ class TestCodexOAuthContextLength:
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.json.return_value = {
"models": [{"slug": "gpt-5.6-terra", "context_window": live_context}]
"models": [{"slug": "gpt-5.5", "context_window": live_context}]
}
# Exercise real persistence here: this test verifies that a live value
# replaces the stale on-disk entry. Failure-path tests below mock the
# writer because they assert that fallback values are not persisted.
with patch("agent.model_metadata.requests.get", return_value=fake_response) as mock_get:
ctx = mm.get_model_context_length(
model="gpt-5.6-terra",
model="gpt-5.5",
base_url=base_url,
api_key="fake-token",
provider="openai-codex",
@@ -510,6 +510,103 @@ class TestCodexOAuthContextLength:
assert remaining.get(stale_key) == live_context
assert remaining.get(other_key) == 128_000
@pytest.mark.parametrize(
"slug",
[
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-5.6-luna",
"gpt-5.6-sol-2026-07-09", # dated snapshot via gpt-5.6 family prefix
"gpt-5.4",
],
)
def test_stale_272k_advertisement_bumped_to_live_verified_350k(self, slug):
"""Codex advertises 272K for these slugs but the backend accepts ~372K
(verified live Aug 2026); the resolver lifts exactly-272K to 350K."""
from agent.model_metadata import get_model_context_length
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.json.return_value = {
"models": [{"slug": slug, "context_window": 272_000}]
}
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.save_context_length"):
ctx = get_model_context_length(
model=slug,
base_url="https://chatgpt.com/backend-api/codex",
api_key="fake-token",
provider="openai-codex",
)
assert ctx == 350_000
def test_non_272k_advertisement_is_trusted_verbatim(self):
"""Any advertised value other than the known-stale 272,000 — higher or
lower — is a real server-side change and must NOT be overridden."""
from agent.model_metadata import get_model_context_length
for advertised in (372_000, 200_000, 1_050_000):
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.json.return_value = {
"models": [{"slug": "gpt-5.6-sol", "context_window": advertised}]
}
import agent.model_metadata as mm
mm._codex_oauth_context_cache = {}
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.save_context_length"):
ctx = get_model_context_length(
model="gpt-5.6-sol",
base_url="https://chatgpt.com/backend-api/codex",
api_key="fake-token",
provider="openai-codex",
)
assert ctx == advertised, f"advertised {advertised} must be trusted"
@pytest.mark.parametrize("slug", ["gpt-5.5", "gpt-5.4-mini"])
def test_slugs_that_enforce_272k_keep_advertised_value(self, slug):
"""gpt-5.5 and gpt-5.4-mini both rejected 360K in the live probe —
their 272K advertisement is real enforcement, so no bump applies
(gpt-5.4 is an exact-match entry precisely to exclude -mini)."""
from agent.model_metadata import get_model_context_length
fake_response = MagicMock()
fake_response.status_code = 200
fake_response.json.return_value = {
"models": [{"slug": slug, "context_window": 272_000}]
}
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.save_context_length"):
ctx = get_model_context_length(
model=slug,
base_url="https://chatgpt.com/backend-api/codex",
api_key="fake-token",
provider="openai-codex",
)
assert ctx == 272_000
def test_fallback_table_resolution_also_bumped(self):
"""When the live probe fails, the 272K fallback-table value for a
verified slug is bumped the same way (same enforcement applies)."""
from agent.model_metadata import get_model_context_length
fake_response = MagicMock()
fake_response.status_code = 401
fake_response.json.return_value = {}
with patch("agent.model_metadata.requests.get", return_value=fake_response), \
patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.save_context_length"):
ctx = get_model_context_length(
model="gpt-5.6-sol",
base_url="https://chatgpt.com/backend-api/codex",
api_key="expired-token",
provider="openai-codex",
)
assert ctx == 350_000