hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

fix(auth): bound device-poll edge backoff and gate 403 on x-vercel-mitigated

Browse Source 
The edge/WAF backoff in the shared Nous+xAI device-code poll loop had three
gaps found in review:

- Retry-After was parsed with a bare int() and never capped, so a
  `Retry-After: 3600` slept an hour past a 5-15 minute device code. Parse it
  with the shared agent.retry_utils.parse_retry_after_seconds and bound every
  sleep by min(60, time left before the device-code deadline).
- The backoff was written into current_interval, so after a block normal
  authorization_pending polls kept the inflated interval and slow_down grew
  from it. Keep it in its own edge_backoff, reset on any OAuth JSON response.
- Any non-JSON 403 was treated as transient, disagreeing with the refresh
  classifier from the previous commit. Only a 403 carrying
  x-vercel-mitigated is the edge speaking; a header-less non-JSON 403 raises
  as before. 408/429/5xx stay transient.

Tests reduced to the two invariants: recovery after edge blocks (each sleep
<= 60, back to the server interval afterwards) and a persistent block ends at
the deadline without oversleeping.
This commit is contained in:
kshitijk4poor
2026-09-24 17:00:10 +05:30
committed by kshitij
parent ad8880e398
commit 1086bd6ccc
2 changed files with 46 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
hermes_cli/auth_device_flow.py
View File

@@ -321,13 +321,15 @@ def _poll_device_token_generic(
"""RFC 8628 device-code polling loop shared by the Nous and xAI flows.
``authorization_pending`` sleeps and retries; ``slow_down`` grows the interval by 1s (cap 30s).
A non-JSON 403/408/429/5xx (edge/WAF mitigation, never a real OAuth error) backs off — honoring
``Retry-After`` — instead of aborting a login the user may still be approving. Every other error,
a non-JSON error body, and the deadline become provider-specific exceptions via the supplied
A non-JSON 408/429/5xx, or a 403 carrying ``x-vercel-mitigated`` (edge/WAF mitigation, never a
real OAuth error), backs off — honoring ``Retry-After``, capped at 60s and at the device-code
deadline — instead of aborting a login the user may still be approving. Every other error, a
non-JSON error body, and the deadline become provider-specific exceptions via the supplied
factories so each caller keeps its exact error contract.
"""
deadline = time.monotonic() + max(1, expires_in)
current_interval = poll_interval
edge_backoff = 0.0 # kept apart from current_interval so slow_down/pending pacing is untouched
while time.monotonic() < deadline:
response = post()
if response.status_code == 200:
@@ -337,22 +339,21 @@ def _poll_device_token_generic(
try:
error_payload = response.json()
except Exception:
# Edge/WAF mitigation is not an OAuth error. Vercel fronts the
# Portal and answers rate-limited clients with a text/plain 403
# (x-vercel-mitigated: deny) or 429 — no JSON body, so it can
# never carry authorization_pending/slow_down. Aborting here
# kills a login the user is still approving in the browser.
# Back off and keep polling until the device code expires.
if response.status_code in {403, 408, 429} or response.status_code >= 500:
try:
backoff = max(current_interval, int(response.headers["retry-after"]))
except (KeyError, TypeError, ValueError):
backoff = min(max(current_interval * 2, 5), 60)
current_interval = backoff
time.sleep(backoff)
status = response.status_code
# Edge/WAF mitigation: back off and keep polling until the device code expires.
if status in {408, 429} or status >= 500 or (
status == 403 and response.headers.get("x-vercel-mitigated")):
from agent.retry_utils import parse_retry_after_seconds
retry_after = parse_retry_after_seconds(response.headers)
if retry_after is not None:
edge_backoff = min(max(current_interval, retry_after), 60)
else:
edge_backoff = min(max(edge_backoff * 2, current_interval * 2, 5), 60)
time.sleep(max(0.0, min(edge_backoff, deadline - time.monotonic())))
continue
response.raise_for_status()
raise on_non_json_error(response)
edge_backoff = 0.0
error_code = str(error_payload.get("error") or "")
if error_code == "authorization_pending":
time.sleep(current_interval)

82
tests/hermes_cli/test_device_poll_waf_backoff.py
View File

@@ -7,10 +7,10 @@ carry ``authorization_pending``/``slow_down``. Before the fix, the generic
device-token poll loop hit ``response.raise_for_status()`` on the first such
response and killed a login the user might still be approving in the browser.
The loop now treats non-JSON 403/408/429/5xx as transient: it backs off
(honoring ``Retry-After``, doubling capped at 60s) and keeps polling until the
device code expires. Non-JSON statuses outside that set still abort, and JSON
OAuth errors keep their caller-specific error contract.
The loop now treats non-JSON 408/429/5xx and an ``x-vercel-mitigated`` 403 as
transient: it backs off (honoring ``Retry-After``, capped at 60s and at the
device-code deadline) and keeps polling until the code expires, then returns to
the server's polling interval once the Portal answers with OAuth JSON again.
"""
import httpx
@@ -47,64 +47,40 @@ def _poll(post, *, expires_in=600, poll_interval=5):
on_timeout=lambda: TimeoutError("device code expired"))
def test_waf_statuses_back_off_and_keep_polling(monkeypatch):
def _fake_clock(monkeypatch):
clock = [1000.0]
sleeps = []
monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s))
def sleep(seconds):
sleeps.append(seconds)
clock[0] += seconds
monkeypatch.setattr(adf.time, "monotonic", lambda: clock[0])
monkeypatch.setattr(adf.time, "sleep", sleep)
return sleeps
def test_recovers_after_edge_blocked_responses(monkeypatch):
sleeps = _fake_clock(monkeypatch)
pending = httpx.Response(400, request=_REQ, json={"error": "authorization_pending"})
post = _post_returning(
_non_json(403, headers={"x-vercel-mitigated": "deny"}),
_non_json(429),
_non_json(429, headers={"retry-after": "3600"}),
_non_json(503),
pending, pending,
_ok({"access_token": "late-token"}))
result = _poll(post)
result = _poll(post, poll_interval=5)
assert result == {"access_token": "late-token"}
assert sleeps == [10, 20, 40] # doubling from interval 5, floor 5, cap 60
assert all(s <= 60 for s in sleeps)
assert sleeps[-2:] == [5, 5] # back to the server interval once OAuth JSON returns
def test_retry_after_header_is_honored(monkeypatch):
sleeps = []
monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s))
post = _post_returning(
_non_json(429, headers={"retry-after": "7"}),
_ok())
_poll(post)
assert sleeps == [7]
def test_backoff_doubles_but_caps_at_60(monkeypatch):
sleeps = []
monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s))
post = _post_returning(*([_non_json(429)] * 6), _ok())
_poll(post)
assert sleeps == [10, 20, 40, 60, 60, 60]
def test_non_json_status_outside_waf_set_still_aborts(monkeypatch):
monkeypatch.setattr(adf.time, "sleep", lambda s: None)
with pytest.raises(httpx.HTTPStatusError):
_poll(_post_returning(_non_json(400)))
def test_json_oauth_error_contract_unchanged(monkeypatch):
monkeypatch.setattr(adf.time, "sleep", lambda s: None)
denied = httpx.Response(400, request=_REQ, json={"error": "access_denied"})
with pytest.raises(RuntimeError, match="oauth:access_denied"):
_poll(_post_returning(denied))
def test_deadline_still_ends_the_login(monkeypatch):
# Fake clock: first reading starts the loop, the second is past the deadline,
# so a persistently mitigated endpoint ends with the caller's timeout error.
readings = iter([1000.0, 1000.0 + 10 * 365 * 24 * 3600])
monkeypatch.setattr(adf.time, "monotonic", lambda: next(readings))
monkeypatch.setattr(adf.time, "sleep", lambda s: None)
def test_persistent_block_ends_at_deadline_without_oversleeping(monkeypatch):
sleeps = _fake_clock(monkeypatch)
with pytest.raises(TimeoutError, match="device code expired"):
_poll(lambda: _non_json(403), expires_in=5)
_poll(lambda: _non_json(429, headers={"retry-after": "3600"}), expires_in=300)
assert sum(sleeps) <= 300