diff --git a/hermes_cli/auth_device_flow.py b/hermes_cli/auth_device_flow.py index 3b41966150..1627970912 100644 --- a/hermes_cli/auth_device_flow.py +++ b/hermes_cli/auth_device_flow.py @@ -321,13 +321,15 @@ def _poll_device_token_generic( """RFC 8628 device-code polling loop shared by the Nous and xAI flows. ``authorization_pending`` sleeps and retries; ``slow_down`` grows the interval by 1s (cap 30s). - A non-JSON 403/408/429/5xx (edge/WAF mitigation, never a real OAuth error) backs off — honoring - ``Retry-After`` — instead of aborting a login the user may still be approving. Every other error, - a non-JSON error body, and the deadline become provider-specific exceptions via the supplied + A non-JSON 408/429/5xx, or a 403 carrying ``x-vercel-mitigated`` (edge/WAF mitigation, never a + real OAuth error), backs off — honoring ``Retry-After``, capped at 60s and at the device-code + deadline — instead of aborting a login the user may still be approving. Every other error, a + non-JSON error body, and the deadline become provider-specific exceptions via the supplied factories so each caller keeps its exact error contract. """ deadline = time.monotonic() + max(1, expires_in) current_interval = poll_interval + edge_backoff = 0.0 # kept apart from current_interval so slow_down/pending pacing is untouched while time.monotonic() < deadline: response = post() if response.status_code == 200: @@ -337,22 +339,21 @@ def _poll_device_token_generic( try: error_payload = response.json() except Exception: - # Edge/WAF mitigation is not an OAuth error. Vercel fronts the - # Portal and answers rate-limited clients with a text/plain 403 - # (x-vercel-mitigated: deny) or 429 — no JSON body, so it can - # never carry authorization_pending/slow_down. Aborting here - # kills a login the user is still approving in the browser. - # Back off and keep polling until the device code expires. - if response.status_code in {403, 408, 429} or response.status_code >= 500: - try: - backoff = max(current_interval, int(response.headers["retry-after"])) - except (KeyError, TypeError, ValueError): - backoff = min(max(current_interval * 2, 5), 60) - current_interval = backoff - time.sleep(backoff) + status = response.status_code + # Edge/WAF mitigation: back off and keep polling until the device code expires. + if status in {408, 429} or status >= 500 or ( + status == 403 and response.headers.get("x-vercel-mitigated")): + from agent.retry_utils import parse_retry_after_seconds + retry_after = parse_retry_after_seconds(response.headers) + if retry_after is not None: + edge_backoff = min(max(current_interval, retry_after), 60) + else: + edge_backoff = min(max(edge_backoff * 2, current_interval * 2, 5), 60) + time.sleep(max(0.0, min(edge_backoff, deadline - time.monotonic()))) continue response.raise_for_status() raise on_non_json_error(response) + edge_backoff = 0.0 error_code = str(error_payload.get("error") or "") if error_code == "authorization_pending": time.sleep(current_interval) diff --git a/tests/hermes_cli/test_device_poll_waf_backoff.py b/tests/hermes_cli/test_device_poll_waf_backoff.py index c2c1cf01d5..83b49f62ad 100644 --- a/tests/hermes_cli/test_device_poll_waf_backoff.py +++ b/tests/hermes_cli/test_device_poll_waf_backoff.py @@ -7,10 +7,10 @@ carry ``authorization_pending``/``slow_down``. Before the fix, the generic device-token poll loop hit ``response.raise_for_status()`` on the first such response and killed a login the user might still be approving in the browser. -The loop now treats non-JSON 403/408/429/5xx as transient: it backs off -(honoring ``Retry-After``, doubling capped at 60s) and keeps polling until the -device code expires. Non-JSON statuses outside that set still abort, and JSON -OAuth errors keep their caller-specific error contract. +The loop now treats non-JSON 408/429/5xx and an ``x-vercel-mitigated`` 403 as +transient: it backs off (honoring ``Retry-After``, capped at 60s and at the +device-code deadline) and keeps polling until the code expires, then returns to +the server's polling interval once the Portal answers with OAuth JSON again. """ import httpx @@ -47,64 +47,40 @@ def _poll(post, *, expires_in=600, poll_interval=5): on_timeout=lambda: TimeoutError("device code expired")) -def test_waf_statuses_back_off_and_keep_polling(monkeypatch): +def _fake_clock(monkeypatch): + clock = [1000.0] sleeps = [] - monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s)) + + def sleep(seconds): + sleeps.append(seconds) + clock[0] += seconds + + monkeypatch.setattr(adf.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(adf.time, "sleep", sleep) + return sleeps + + +def test_recovers_after_edge_blocked_responses(monkeypatch): + sleeps = _fake_clock(monkeypatch) + pending = httpx.Response(400, request=_REQ, json={"error": "authorization_pending"}) post = _post_returning( _non_json(403, headers={"x-vercel-mitigated": "deny"}), - _non_json(429), + _non_json(429, headers={"retry-after": "3600"}), _non_json(503), + pending, pending, _ok({"access_token": "late-token"})) - result = _poll(post) + result = _poll(post, poll_interval=5) assert result == {"access_token": "late-token"} - assert sleeps == [10, 20, 40] # doubling from interval 5, floor 5, cap 60 + assert all(s <= 60 for s in sleeps) + assert sleeps[-2:] == [5, 5] # back to the server interval once OAuth JSON returns -def test_retry_after_header_is_honored(monkeypatch): - sleeps = [] - monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s)) - post = _post_returning( - _non_json(429, headers={"retry-after": "7"}), - _ok()) - - _poll(post) - - assert sleeps == [7] - - -def test_backoff_doubles_but_caps_at_60(monkeypatch): - sleeps = [] - monkeypatch.setattr(adf.time, "sleep", lambda s: sleeps.append(s)) - post = _post_returning(*([_non_json(429)] * 6), _ok()) - - _poll(post) - - assert sleeps == [10, 20, 40, 60, 60, 60] - - -def test_non_json_status_outside_waf_set_still_aborts(monkeypatch): - monkeypatch.setattr(adf.time, "sleep", lambda s: None) - - with pytest.raises(httpx.HTTPStatusError): - _poll(_post_returning(_non_json(400))) - - -def test_json_oauth_error_contract_unchanged(monkeypatch): - monkeypatch.setattr(adf.time, "sleep", lambda s: None) - denied = httpx.Response(400, request=_REQ, json={"error": "access_denied"}) - - with pytest.raises(RuntimeError, match="oauth:access_denied"): - _poll(_post_returning(denied)) - - -def test_deadline_still_ends_the_login(monkeypatch): - # Fake clock: first reading starts the loop, the second is past the deadline, - # so a persistently mitigated endpoint ends with the caller's timeout error. - readings = iter([1000.0, 1000.0 + 10 * 365 * 24 * 3600]) - monkeypatch.setattr(adf.time, "monotonic", lambda: next(readings)) - monkeypatch.setattr(adf.time, "sleep", lambda s: None) +def test_persistent_block_ends_at_deadline_without_oversleeping(monkeypatch): + sleeps = _fake_clock(monkeypatch) with pytest.raises(TimeoutError, match="device code expired"): - _poll(lambda: _non_json(403), expires_in=5) + _poll(lambda: _non_json(429, headers={"retry-after": "3600"}), expires_in=300) + + assert sum(sleeps) <= 300