From 59fad62a4049413f7fa420c30ca7928673d14899 Mon Sep 17 00:00:00 2001 From: Robin Fernandes Date: Mon, 14 Sep 2026 15:50:28 +1000 Subject: [PATCH] =?UTF-8?q?fix(free-tier):=20review=20follow-ups=20?= =?UTF-8?q?=E2=80=94=20read=20the=20classifier's=20context,=20never=20repl?= =?UTF-8?q?ace=20a=20locked=20identity,=20re-inventory=20on=20retry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctness - The welcome-tier recovery hooks (model_not_free move, wrong-host heal) and the long-wait rate-limit check read the turn's extract_api_error_context() dict, which never carries welcome_refusal / welcome_route. They now read classified.error_context, where _nous_welcome_tier parks them; the guard records the classifier's reset_at. Tests drive the real classifier and the real extractor so the two-context boundary is exercised. - The connector path caught every AnonCredentialDead and re-minted; a locked account (anon_account_locked) is now retired without replacement, matching the inference resolver. - A background bootstrap retry reused the boot-time provider inventory; it re-inventories, so a provider connected during the cooldown keeps inference. - The desktop's setup.ready listener only refreshes an untouched picker (oauth mode, no local endpoint, idle flow) and re-checks after the readiness round, so an API-key form opened meanwhile is never dismissed. - /__log on the rehearsal server sent its response while holding the state lock that _send re-acquires; the log is copied out first. Reductions - One shared FakePortal / install_portal (tests/hermes_cli/anon_portal.py) behind both free-tier fixtures, with a single httpx.Client transport seam. - The rehearsal server's static inference answers are a table; dead scaffolding (REAL_PAID_URL, claim_codes, the no-op dead_once branch, extra_headers) removed. - Setup-notice copy is a code-to-key map; its test uses real codes (the old loop built nonexistent ones and only exercised the fallback). - The ineffective FreeTierErrorCode union is gone. Co-Authored-By: Claude Fable 5.1 --- agent/turn_recovery.py | 32 +++-- .../onboarding/free-tier-intro.test.ts | 9 ++ .../free-tier-setup-notice.test.tsx | 32 ++--- .../onboarding/free-tier-setup-notice.tsx | 45 +++---- .../src/components/onboarding/index.tsx | 22 +++- apps/desktop/src/store/free-tier.ts | 5 +- apps/desktop/src/store/onboarding.ts | 12 +- apps/desktop/src/types/hermes.ts | 11 -- hermes_cli/free_tier_bootstrap.py | 4 +- scripts/free_tier_fault_server.py | 114 ++++++++---------- tests/agent/test_welcome_tier_recovery.py | 77 +++++++++--- tests/hermes_cli/anon_portal.py | 105 ++++++++++++++++ tests/hermes_cli/test_anon_auth_core.py | 77 +----------- tests/hermes_cli/test_anon_failure_modes.py | 103 +++++----------- tests/scripts/test_free_tier_fault_server.py | 6 + tools/managed_tool_gateway.py | 11 +- 16 files changed, 351 insertions(+), 314 deletions(-) create mode 100644 tests/hermes_cli/anon_portal.py diff --git a/agent/turn_recovery.py b/agent/turn_recovery.py index e0ac112910..d14a24e8dd 100644 --- a/agent/turn_recovery.py +++ b/agent/turn_recovery.py @@ -480,14 +480,18 @@ def _recover_format_errors( return False -def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState, error_context: Any) -> bool: +def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState) -> bool: """Two one-shot repairs for the Nous free tier, both silent on the wire and named once in chat. ``model_not_free``: the session asked the welcome host for a model it does not serve; move to the first alternate the gateway named (its own model) and retry, instead of failing the turn. ``anon_on_paid_host``: this process is pointed at the paid host with a free-tier - identity (a stale route); re-read the credentials, which heals the URL, and retry.""" - ctx = error_context if isinstance(error_context, dict) else (getattr(classified, "error_context", None) or {}) + identity (a stale route); re-read the credentials, which heals the URL, and retry. + + Reads the CLASSIFIER's context (``classified.error_context``): that is where + ``_nous_welcome_tier`` parks ``welcome_refusal`` / ``welcome_route``. The turn's other context + (``extract_api_error_context``) never carries them.""" + ctx = getattr(classified, "error_context", None) or {} refusal = ctx.get("welcome_refusal") if isinstance(ctx, dict) else None if isinstance(refusal, dict) and refusal.get("reason") == "model_not_free" and not _retry.welcome_model_switch_attempted: _retry.welcome_model_switch_attempted = True @@ -529,7 +533,7 @@ def recover_after_classification( Returns ``(retry_now, recovered_with_pool)``; the latter feeds the Nous rate-limit guard.""" from agent.conversation_loop import _is_nous_inference_route - if _recover_welcome_tier(agent, classified, _retry, error_context): + if _recover_welcome_tier(agent, classified, _retry): return True, False if ( @@ -1361,23 +1365,27 @@ def _eager_fallback_status(classified: Any, is_upstream: bool, is_transport_fail return "⚠️ Rate limited — switching to fallback provider..." -def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: Any) -> bool: +def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: Any, classified: Any = None) -> bool: """Record a genuine account-level Nous 429 to the cross-session breaker; upstream - capacity 429s (no exhausted bucket in headers or last-known state) are left alone.""" + capacity 429s (no exhausted bucket in headers or last-known state) are left alone. + + *error_context* is the turn's (``extract_api_error_context``); *classified* brings the + classifier's own context, where a welcome-tier ``rate_limited`` refusal and its ``reset_at`` + live. A long welcome reset is an exhausted allowance whatever the headers say, and the one + place the user is told that signing in lifts it.""" _genuine = False try: from agent.nous_rate_guard import ( is_genuine_nous_rate_limit, is_long_welcome_rate_limit, record_nous_rate_limit) _err_resp = getattr(api_error, "response", None) _err_hdrs = getattr(_err_resp, "headers", None) if _err_resp else None - # The welcome tier's structured ``rate_limited`` refusal names its own reset; a long one - # is an exhausted allowance whatever the headers say, and the one place the user is told - # that signing in lifts it. + _classified_ctx = getattr(classified, "error_context", None) or {} _genuine = ( - is_long_welcome_rate_limit(error_context) + is_long_welcome_rate_limit(_classified_ctx) or is_genuine_nous_rate_limit(headers=_err_hdrs, last_known_state=agent._rate_limit_state)) if _genuine: - record_nous_rate_limit(headers=_err_hdrs, error_context=error_context) + _merged = {**(error_context if isinstance(error_context, dict) else {}), **_classified_ctx} + record_nous_rate_limit(headers=_err_hdrs, error_context=_merged) else: logger.info( "Nous 429 looks like upstream capacity " @@ -1553,7 +1561,7 @@ def route_classified_error( and agent.provider == "nous" and classified.reason == FailoverReason.rate_limit and not recovered_with_pool - and _is_genuine_nous_rate_limit(agent, api_error, error_context) + and _is_genuine_nous_rate_limit(agent, api_error, error_context, classified) ): # Re-enter the loop exactly once so the top-of-loop Nous guard runs # (retry_count = max_retries would skip it entirely). diff --git a/apps/desktop/src/components/onboarding/free-tier-intro.test.ts b/apps/desktop/src/components/onboarding/free-tier-intro.test.ts index 2db4cb25a3..73f8eff31f 100644 --- a/apps/desktop/src/components/onboarding/free-tier-intro.test.ts +++ b/apps/desktop/src/components/onboarding/free-tier-intro.test.ts @@ -75,3 +75,12 @@ describe('acknowledging the introduction', () => { expect(await ackFreeTierIntro({ requestGateway: gatewayReturning(READY) })).toBe(false) // status stub returns no {acked: true} }) }) + +describe('a background readiness round', () => { + it('never completes onboarding when the picker was touched during the round', async () => { + // The `setup.ready` listener passes `stillWanted`; a user opening the API-key form while the + // readiness request was out must not have it dismissed by a late "ready". + expect(await refreshOnboarding({ requestGateway: gatewayReturning(READY) }, () => false)).toBe(false) + expect($desktopOnboarding.get().freeTierReady).toBe(false) + }) +}) diff --git a/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx b/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx index 57a5ec2167..d0172b3c17 100644 --- a/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx +++ b/apps/desktop/src/components/onboarding/free-tier-setup-notice.test.tsx @@ -54,33 +54,25 @@ describe('setupFailureCopy', () => { ['anon_unreachable', copy.unreachable], ['anon_server_error', copy.serverError], ['anon_pow_required', copy.powRequired], - ['anon_account_locked', copy.locked] - ])('%s has its own sentence', (code, expected) => { - const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: code }) + ['anon_account_locked', copy.locked], + ['anon_rate_limited', copy.rateLimited('about 5 minutes')] + ])('%s has its own sentence, in the agreed voice', (code, expected) => { + const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: code, retry_after: 300 }) + const text = failure ? setupFailureCopy(failure, copy) : '' - expect(failure && setupFailureCopy(failure, copy)).toBe(expected) - }) - - it('speaks the wait for a rate limit', () => { - const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: 'anon_rate_limited', retry_after: 300 }) - - expect(failure && setupFailureCopy(failure, copy)).toContain('about 5 minutes') + expect(text).toBe(expected) + // Never "the free service is off" — what is unavailable is using Hermes without signing in — + // and no jargon a first-time user would not know. + expect(text.toLowerCase()).not.toMatch(/free (service|model|tier) is (off|switched off|unavailable|down)/) + expect(text.toLowerCase()).not.toMatch(/anonymous|guest|credential|token|rate limit/) }) it('falls back to the backend sentence for a code this build does not know', () => { const failure = freeTierSetupFailure({ ...NO_IDENTITY, error: 'Something new.', error_code: 'anon_newer' }) expect(failure && setupFailureCopy(failure, copy)).toBe('Something new.') - }) - - it('never says the free service or the free model is off', () => { - for (const code of Object.keys(copy)) { - const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: `anon_${code}` }) - const text = failure ? setupFailureCopy(failure, copy).toLowerCase() : '' - - expect(text).not.toMatch(/free (service|model|tier) is (off|switched off|unavailable|down)/) - expect(text).not.toMatch(/anonymous|guest|credential|token|rate limit/) - } + // Prototype names are not codes. + expect(setupFailureCopy({ ...failure!, code: 'constructor', message: '' }, copy)).toBe(copy.generic) }) }) diff --git a/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx b/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx index 446156bfd8..b0fa2bc0fa 100644 --- a/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx +++ b/apps/desktop/src/components/onboarding/free-tier-setup-notice.tsx @@ -15,34 +15,27 @@ import { type OnboardingContext, refreshOnboarding } from '@/store/onboarding' type SetupFailedCopy = Translations['freeTier']['setupFailed'] -/** One sentence per backend code. The backend's own sentence is the fallback for - * a code this build does not know, so a newer backend still reads as words. */ +// One sentence per backend code (`hermes_cli/anon_auth.py::ANON_*`). +const COPY_KEY_BY_CODE: Record = { + anon_account_locked: 'locked', + anon_gate_closed: 'gateClosed', + anon_gate_paused: 'paused', + anon_pow_required: 'powRequired', + anon_server_error: 'serverError', + anon_unreachable: 'unreachable' +} + +/** The sentence for a failure. The backend's own sentence is the fallback for a + * code this build does not know, so a newer backend still reads as words. */ export function setupFailureCopy(failure: FreeTierSetupFailure, copy: SetupFailedCopy): string { - switch (failure.code) { - case 'anon_gate_closed': - return copy.gateClosed - - case 'anon_gate_paused': - return copy.paused - - case 'anon_rate_limited': - return copy.rateLimited(friendlyWait(failure.retryAfter || 60)) - - case 'anon_unreachable': - return copy.unreachable - - case 'anon_server_error': - return copy.serverError - - case 'anon_pow_required': - return copy.powRequired - - case 'anon_account_locked': - return copy.locked - - default: - return failure.message || copy.generic + if (failure.code === 'anon_rate_limited') { + return copy.rateLimited(friendlyWait(failure.retryAfter || 60)) } + + const key = Object.hasOwn(COPY_KEY_BY_CODE, failure.code) ? COPY_KEY_BY_CODE[failure.code] : null + const sentence = key ? copy[key] : null + + return typeof sentence === 'string' ? sentence : failure.message || copy.generic } /** diff --git a/apps/desktop/src/components/onboarding/index.tsx b/apps/desktop/src/components/onboarding/index.tsx index c64f385a37..42482347a0 100644 --- a/apps/desktop/src/components/onboarding/index.tsx +++ b/apps/desktop/src/components/onboarding/index.tsx @@ -287,15 +287,27 @@ export function DesktopOnboardingOverlay({ // The boot bootstrap re-announces `setup.ready` when a background retry of // the free-tier set-up succeeds after a failed first attempt. A picker that // is up only because that set-up failed re-checks readiness and gives way - // on its own; a manual open, or a picker the user is mid-flow in, is left - // alone. + // on its own. An untouched picker only: a manual open, a provider flow in + // progress, or the API-key form (which leaves the flow idle while the user + // types) is left alone, and the check is repeated after the readiness + // round so a key form opened in the meantime survives too. useEffect( () => $setupReadyTick.listen(() => { - const current = $desktopOnboarding.get() + const untouched = () => { + const current = $desktopOnboarding.get() - if (!current.manual && current.configured === false && current.flow.status === 'idle') { - void refreshOnboarding(ctx) + return ( + !current.manual && + current.configured === false && + current.flow.status === 'idle' && + current.mode === 'oauth' && + !current.localEndpoint + ) + } + + if (untouched()) { + void refreshOnboarding(ctx, untouched) } }), [ctx] diff --git a/apps/desktop/src/store/free-tier.ts b/apps/desktop/src/store/free-tier.ts index b545e13689..4591b73a21 100644 --- a/apps/desktop/src/store/free-tier.ts +++ b/apps/desktop/src/store/free-tier.ts @@ -1,7 +1,7 @@ import { atom } from 'nanostores' import { onboardingSurfaceActive } from '@/store/onboarding-presence' -import type { FreeTierErrorCode, FreeTierStatus } from '@/types/hermes' +import type { FreeTierStatus } from '@/types/hermes' /** The model the free-tier route runs on. Used to recognise a session that is * still homed on the free tier after a sign-in. */ @@ -62,7 +62,8 @@ export async function refreshFreeTierStatus(requestGateway: FreeTierRequester): * codes get "try again / another provider" only. */ export interface FreeTierSetupFailure { - code: FreeTierErrorCode | string + /** One of the backend's `anon_*` codes (`hermes_cli/anon_auth.py`), or a newer one this build does not know. */ + code: string door: 'retry' | 'sign_in' message: string retryAfter: number diff --git a/apps/desktop/src/store/onboarding.ts b/apps/desktop/src/store/onboarding.ts index fada42bd92..8225d3c23d 100644 --- a/apps/desktop/src/store/onboarding.ts +++ b/apps/desktop/src/store/onboarding.ts @@ -671,7 +671,13 @@ export function setOnboardingMode(mode: OnboardingMode) { patch({ mode }) } -export async function refreshOnboarding(ctx: OnboardingContext) { +/** + * `stillWanted`, when given, is re-asked after the readiness round: a background + * caller (the `setup.ready` listener) passes it so a user action that started + * during the round — opening the API-key form, picking a provider — is never + * dismissed by a late "ready". + */ +export async function refreshOnboarding(ctx: OnboardingContext, stillWanted?: () => boolean) { // Manual mode (user opened the selector from a working app): never // auto-dismiss on runtime-ready — the whole point is to let them add / // switch a provider while already configured. Just ensure the provider @@ -684,6 +690,10 @@ export async function refreshOnboarding(ctx: OnboardingContext) { const runtime = await checkRuntime(ctx) + if (stillWanted && !stillWanted()) { + return false + } + if (runtime.ready) { completeDesktopOnboarding() await applyFreeTierIntro(ctx, runtime) diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 8440e9d327..30780c322c 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -164,17 +164,6 @@ export interface FreeTierStatus { retry_after?: number } -/** The backend's free-tier failure codes (`hermes_cli/anon_auth.py::ANON_*`). */ -export type FreeTierErrorCode = - | 'anon_account_locked' - | 'anon_credential_dead' - | 'anon_gate_closed' - | 'anon_gate_paused' - | 'anon_pow_required' - | 'anon_rate_limited' - | 'anon_server_error' - | 'anon_unreachable' - export interface MemoryProviderOAuthStatus { auth: 'apikey' | 'oauth' | null connected: boolean diff --git a/hermes_cli/free_tier_bootstrap.py b/hermes_cli/free_tier_bootstrap.py index 030b104970..ebe7b5467b 100644 --- a/hermes_cli/free_tier_bootstrap.py +++ b/hermes_cli/free_tier_bootstrap.py @@ -190,7 +190,9 @@ def retry_bootstrap_mint(*, force: bool = False, announce: bool = True) -> Setup return run_bootstrap(announce=announce) if current.has_identity: return current - record = _build_record(other=current.other_providers, force=force) + # Re-inventory: a provider the user connected during the cooldown must keep inference; the + # boot-time answer is stale by now. + record = _build_record(other=_inventory_other_providers(), force=force) with _lock: _record = record if announce: diff --git a/scripts/free_tier_fault_server.py b/scripts/free_tier_fault_server.py index 468cbca512..98616ef4dc 100644 --- a/scripts/free_tier_fault_server.py +++ b/scripts/free_tier_fault_server.py @@ -43,8 +43,6 @@ from urllib.parse import parse_qs, urlparse WELCOME_MODEL = "nous/welcome" REAL_WELCOME_HOST = "welcome-api.nousresearch.com" -REAL_PAID_URL = "https://inference-api.nousresearch.com" -GENERIC_403 = "You tried to access something that you don't have permissions for." UPGRADE_URL = "https://portal.nousresearch.com/signup" # --- Scenario catalogue -------------------------------------------------------------------------- @@ -83,6 +81,35 @@ INFERENCE_SCENARIOS: Dict[str, str] = { } +_FAIRSHARE_MESSAGE = ("You've reached this model's current fair-share rate limit. It adapts to demand — " + "retry after the indicated delay, or try an alternate model.") + + +def _fairshare(reason: str, message: str, **extra: Any) -> Dict[str, Any]: + return {"status": 429, "message": message, "reason": reason, "alternates": [], "upgrade_url": UPGRADE_URL, **extra} + + +# Static inference answers: scenario -> (status, body, default Retry-After seconds). ``retry_after`` +# in a fairshare body and every wait header are filled in per request from the live override. +INFERENCE_RESPONSES: Dict[str, tuple] = { + "rate_limited": (429, _fairshare("rate_limited", _FAIRSHARE_MESSAGE), 600), + "rate_limited_short": (429, _fairshare("rate_limited", _FAIRSHARE_MESSAGE), 5), + "at_capacity": (429, _fairshare("at_capacity", "The free tier is at capacity and briefly paused. It reopens " + "automatically — retry after the indicated delay."), 30), + "model_not_free": (429, _fairshare("model_not_free", "This model isn't available on the free tier.", + alternates=[WELCOME_MODEL]), 0), + "tier_disabled": (403, {"status": 403, "message": "You tried to access something that you don't have permissions for."}, 0), + "wrong_host": (400, {"status": 400, "message": "This request is not valid. Check the model name and other parameters. " + f"Additional info: Anonymous accounts must use https://{REAL_WELCOME_HOST} for inference."}, 0), + "bare_429": (429, {"status": 429, "message": "Hold up for a bit, you've exceeded the rate limit on your API key."}, 600), + "upstream_503": (503, {"status": 503, "message": "The requested model is currently unavailable."}, 0), + "upstream_500": (500, {"status": 500, "message": "Something unexpected happened while processing your request. " + "Please try again in a moment, or contact us if the issue persists."}, 0), + "invalid_token": (401, {"status": 401, "error": "invalid_token", "subcause": "anonymous_credential_revoked", + "message": "The anonymous account behind this token is gone"}, 0), +} + + def _jwt(**claims: Any) -> str: def seg(obj: Any) -> str: return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() @@ -105,7 +132,6 @@ class State: self.minted = 0 self.dead_tokens: set[str] = set() self.signin: Dict[str, Any] = {"status": "pending"} - self.claim_codes: Dict[str, str] = {} self.log: deque[Dict[str, Any]] = deque(maxlen=100) def snapshot(self) -> Dict[str, Any]: @@ -146,7 +172,6 @@ class State: self.retry_after = None self.dead_tokens.clear() self.signin = {"status": "pending"} - self.claim_codes.clear() self.log.clear() @@ -210,7 +235,8 @@ class Handler(BaseHTTPRequestHandler): self._send(200, STATE.snapshot()) elif path == "/__log": with STATE.lock: - self._send(200, {"requests": list(STATE.log)}) + entries = list(STATE.log) + self._send(200, {"requests": entries}) # outside the lock: _send appends to the log elif path == "/v1/models": self._send(200, {"object": "list", "data": [{"id": WELCOME_MODEL, "object": "model", "owned_by": "nous"}]}) elif path.startswith("/__claim"): @@ -292,11 +318,8 @@ class Handler(BaseHTTPRequestHandler): with STATE.lock: STATE.minted += 1 n = STATE.minted - token = f"anon_rehearsal_{n:04d}_{secrets.token_hex(4)}" - if scenario == "dead_once": - pass # the credential itself is fine; its first exchange is what dies self._send(201, {"user_id": f"nas_user:rehearsal-{n}", "org_id": f"nas_org:rehearsal-{n}", - "token": token, "idle_ttl_days": 14}) + "token": f"anon_rehearsal_{n:04d}_{secrets.token_hex(4)}", "idle_ttl_days": 14}) return if path == "/api/anonymous/token": token = str(body.get("token") or "") @@ -356,7 +379,6 @@ class Handler(BaseHTTPRequestHandler): return code = f"{secrets.token_hex(2).upper()}-{secrets.token_hex(2).upper()}" with STATE.lock: - STATE.claim_codes[code] = str(body.get("token") or "") STATE.signin = {"status": "pending"} host = self.headers.get("Host") or "127.0.0.1" self._send(200, {"claim_code": code, "claim_url": f"http://{host}/__claim?code={code}", @@ -388,65 +410,31 @@ class Handler(BaseHTTPRequestHandler): # --- the welcome inference host -------------------------------------------------------------- - def _refusal(self, status: int, message: str, *, reason: str, retry_after: int, - alternates: Optional[list] = None, extra_headers: Optional[Dict[str, str]] = None) -> None: - headers = {"Retry-After": str(retry_after)} - if reason in ("rate_limited", "at_capacity"): - headers["RateLimit-Policy"] = '"fairshare";q=0;qu="tokens";w=60' - headers["RateLimit"] = f'"fairshare";r=0;t={retry_after}' - headers.update(extra_headers or {}) - STATE.consume_once("inference") - self._send(status, {"status": status, "message": message, "reason": reason, "retry_after": retry_after, - "alternates": alternates or [], "upgrade_url": UPGRADE_URL}, headers=headers) - def _inference(self, body: Dict[str, Any]) -> None: with STATE.lock: scenario = STATE.inference - model = str(body.get("model") or WELCOME_MODEL) - if scenario == "rate_limited": - self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — " - "retry after the indicated delay, or try an alternate model.", - reason="rate_limited", retry_after=self._retry_after(600)) - elif scenario == "rate_limited_short": - self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — " - "retry after the indicated delay, or try an alternate model.", - reason="rate_limited", retry_after=self._retry_after(5)) - elif scenario == "at_capacity": - self._refusal(429, "The free tier is at capacity and briefly paused. It reopens automatically — " - "retry after the indicated delay.", reason="at_capacity", retry_after=self._retry_after(30)) - elif scenario == "model_not_free": - self._refusal(429, "This model isn't available on the free tier.", reason="model_not_free", - retry_after=0, alternates=[WELCOME_MODEL]) - elif scenario == "tier_disabled": - STATE.consume_once("inference") - self._send(403, {"status": 403, "message": GENERIC_403}) - elif scenario == "wrong_host": - STATE.consume_once("inference") - self._send(400, {"status": 400, "message": "This request is not valid. Check the model name and other " - f"parameters. Additional info: Anonymous accounts must use https://{REAL_WELCOME_HOST} for inference."}) - elif scenario == "bare_429": - STATE.consume_once("inference") - self._send(429, {"status": 429, "message": "Hold up for a bit, you've exceeded the rate limit on your API key."}, - headers={"x-ratelimit-limit-requests": "30", "x-ratelimit-remaining-requests": "0", - "x-ratelimit-reset-requests": str(self._retry_after(600)), - "Retry-After": str(self._retry_after(600))}) - elif scenario == "upstream_503": - STATE.consume_once("inference") - self._send(503, {"status": 503, "message": "The requested model is currently unavailable."}) - elif scenario == "upstream_500": - STATE.consume_once("inference") - self._send(500, {"status": 500, "message": "Something unexpected happened while processing your request. " - "Please try again in a moment, or contact us if the issue persists."}) - elif scenario == "invalid_token": - STATE.consume_once("inference") - self._send(401, {"status": 401, "error": "invalid_token", "subcause": "anonymous_credential_revoked", - "message": "The anonymous account behind this token is gone"}) - elif scenario == "timeout": + if scenario == "timeout": STATE.consume_once("inference") time.sleep(120) self._send(504, {"status": 504, "message": "timed out"}) - else: - self._reply(model, stream=bool(body.get("stream"))) + return + canned = INFERENCE_RESPONSES.get(scenario) + if canned is None: + self._reply(str(body.get("model") or WELCOME_MODEL), stream=bool(body.get("stream"))) + return + status, payload, default_wait = canned + wait = self._retry_after(default_wait) + headers = {"Retry-After": str(wait)} if default_wait or "reason" in payload else {} + if payload.get("reason") in ("rate_limited", "at_capacity"): + headers["RateLimit-Policy"] = '"fairshare";q=0;qu="tokens";w=60' + headers["RateLimit"] = f'"fairshare";r=0;t={wait}' + if scenario == "bare_429": + headers.update({"x-ratelimit-limit-requests": "30", "x-ratelimit-remaining-requests": "0", + "x-ratelimit-reset-requests": str(wait)}) + if "reason" in payload: + payload = {**payload, "retry_after": wait} + STATE.consume_once("inference") + self._send(status, payload, headers=headers) def _reply(self, model: str, *, stream: bool) -> None: text = ("Hello from the free tier fault server. Everything is working; switch a scenario on " diff --git a/tests/agent/test_welcome_tier_recovery.py b/tests/agent/test_welcome_tier_recovery.py index dc00a99a6a..ed8facc336 100644 --- a/tests/agent/test_welcome_tier_recovery.py +++ b/tests/agent/test_welcome_tier_recovery.py @@ -1,6 +1,10 @@ """Nous free tier, inference side: the dark-tier 403 keyed on the route, the one-shot model move after ``model_not_free``, the wrong-host heal, the long-wait rule for structured ``rate_limited`` -refusals, and the plain outage sentence once retries are spent.""" +refusals, and the plain outage sentence once retries are spent. + +The recovery hooks are driven from the REAL producer boundary: a gateway body goes through +``classify_api_error`` (and the turn's own ``extract_api_error_context``) exactly as the turn loop +feeds them, so a wiring slip between the two contexts fails here.""" from __future__ import annotations @@ -8,6 +12,7 @@ from types import SimpleNamespace import pytest +from agent.agent_runtime_helpers import extract_api_error_context from agent.error_classifier import FailoverReason, classify_api_error from agent.turn_retry_state import TurnRetryState @@ -23,21 +28,32 @@ class MockAPIError(Exception): self.body = body +def _gateway_error(status: int, body: dict) -> MockAPIError: + return MockAPIError(f"Error code: {status} - {body}", status_code=status, body=body) + + def _generic_403(): - body = {"status": 403, "message": "You tried to access something that you don't have permissions for."} - return MockAPIError(f"Error code: 403 - {body}", status_code=403, body=body) + return _gateway_error(403, {"status": 403, "message": "You tried to access something that you don't have permissions for."}) + + +def _refusal(reason: str, *, retry_after: int = 0, alternates=None) -> MockAPIError: + return _gateway_error(429, {"status": 429, "message": "refused", "reason": reason, "retry_after": retry_after, + "alternates": alternates or [], "upgrade_url": "https://portal.example/signup"}) + + +def _classify(err: MockAPIError, *, model: str = "nous/welcome", base_url: str = WELCOME): + return classify_api_error(err, provider="nous", model=model, base_url=base_url) class TestDarkTier403: def test_a_generic_403_from_the_welcome_host_is_the_tier_refusing(self): - result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=WELCOME) + result = _classify(_generic_403()) assert result.reason == FailoverReason.auth_permanent assert result.retryable is False and result.should_fallback is True assert result.error_context["welcome_route"] == "tier_disabled" def test_the_same_403_from_the_paid_host_stays_an_ordinary_403(self): - result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=PAID) - assert "welcome_route" not in result.error_context + assert "welcome_route" not in _classify(_generic_403(), base_url=PAID).error_context def test_a_403_from_another_provider_on_any_host_is_untouched(self): result = classify_api_error(_generic_403(), provider="openrouter", base_url=WELCOME) @@ -47,7 +63,7 @@ class TestDarkTier403: def _agent(**overrides): lines = [] agent = SimpleNamespace( - provider="nous", model="gpt-5", base_url=WELCOME, log_prefix="", + provider="nous", model="gpt-5", base_url=WELCOME, log_prefix="", _rate_limit_state=None, _vprint=lambda text, force=False: lines.append(text), _try_refresh_nous_client_credentials=lambda **kw: True, ) @@ -61,38 +77,39 @@ class TestOneShotRecoveries: def test_model_not_free_moves_the_session_onto_the_alternate_and_retries_once(self): from agent.turn_recovery import _recover_welcome_tier agent = _agent() - ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0, - "alternates": ["nous/welcome"], "upgrade_url": ""}} + classified = _classify(_refusal("model_not_free", alternates=["nous/welcome"]), model="gpt-5") retry = TurnRetryState() - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True + assert _recover_welcome_tier(agent, classified, retry) is True assert agent.model == "nous/welcome" assert agent._nous_model_switch == ("gpt-5", "nous/welcome") assert "without signing in" in agent.lines[0] # Once: a second refusal in the same attempt falls through to the terminal path. - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False + assert _recover_welcome_tier(agent, classified, retry) is False def test_model_not_free_without_an_alternate_does_nothing(self): from agent.turn_recovery import _recover_welcome_tier agent = _agent() - ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0, "alternates": [], "upgrade_url": ""}} - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False + classified = _classify(_refusal("model_not_free"), model="gpt-5") + assert _recover_welcome_tier(agent, classified, TurnRetryState()) is False assert agent.model == "gpt-5" def test_a_wrong_host_refusal_re_reads_the_route_once(self): from agent.turn_recovery import _recover_welcome_tier calls = [] agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: calls.append(kw) or True) - ctx = {"welcome_route": "anon_on_paid_host"} + body = {"status": 400, "message": "Anonymous accounts must use https://welcome-api.nousresearch.com for inference."} + classified = _classify(_gateway_error(400, body), base_url=PAID) + assert classified.error_context["welcome_route"] == "anon_on_paid_host" retry = TurnRetryState() - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True + assert _recover_welcome_tier(agent, classified, retry) is True assert calls == [{"force": True}] - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False + assert _recover_welcome_tier(agent, classified, retry) is False def test_a_wrong_host_refusal_whose_heal_fails_falls_through(self): from agent.turn_recovery import _recover_welcome_tier agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: False) - ctx = {"welcome_route": "anon_on_paid_host"} - assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False + body = {"status": 400, "message": "Anonymous accounts must use https://welcome-api.nousresearch.com for inference."} + assert _recover_welcome_tier(agent, _classify(_gateway_error(400, body), base_url=PAID), TurnRetryState()) is False class TestLongWaitRule: @@ -102,13 +119,33 @@ class TestLongWaitRule: ]) def test_only_a_long_rate_limited_refusal_is_an_exhausted_allowance(self, reason, retry_after, expected): from agent.nous_rate_guard import is_long_welcome_rate_limit - ctx = {"welcome_refusal": {"reason": reason, "retry_after": retry_after, "alternates": [], "upgrade_url": ""}} - assert is_long_welcome_rate_limit(ctx) is expected + classified = _classify(_refusal(reason, retry_after=retry_after)) + assert is_long_welcome_rate_limit(classified.error_context) is expected def test_no_refusal_is_not_long(self): from agent.nous_rate_guard import is_long_welcome_rate_limit assert is_long_welcome_rate_limit({}) is False and is_long_welcome_rate_limit(None) is False + def test_the_turn_records_a_long_refusal_from_the_classifiers_context(self, monkeypatch): + """The turn hands the guard TWO contexts: its own (``extract_api_error_context``), which + never carries ``welcome_refusal``, and the classifier's, which does. The breaker must key on + the latter and record the reset it computed.""" + import agent.nous_rate_guard as guard + from agent.turn_recovery import _is_genuine_nous_rate_limit + recorded = [] + monkeypatch.setattr(guard, "record_nous_rate_limit", lambda **kw: recorded.append(kw)) + err = _refusal("rate_limited", retry_after=600) + turn_ctx = extract_api_error_context(err) + assert "welcome_refusal" not in turn_ctx + classified = _classify(err) + assert _is_genuine_nous_rate_limit(_agent(), err, turn_ctx, classified) is True + assert recorded and recorded[0]["error_context"]["reset_at"] == classified.error_context["reset_at"] + # A short one is not an exhausted allowance: nothing recorded, the turn waits it out. + recorded.clear() + short = _refusal("rate_limited", retry_after=5) + assert _is_genuine_nous_rate_limit(_agent(), short, extract_api_error_context(short), _classify(short)) is False + assert recorded == [] + class TestOutageCopy: @pytest.mark.parametrize("reason", [FailoverReason.timeout, FailoverReason.overloaded, diff --git a/tests/hermes_cli/anon_portal.py b/tests/hermes_cli/anon_portal.py new file mode 100644 index 0000000000..573363b147 --- /dev/null +++ b/tests/hermes_cli/anon_portal.py @@ -0,0 +1,105 @@ +"""The fake NAS anonymous surface shared by the free-tier tests. + +One ``FakePortal`` and one ``install_portal`` behind every ``portal`` fixture: the wire contract is +exercised through Hermes' real client code, never mocked away. Scenarios flip its behaviour +(``gate_closed``, ``dead_tokens``, a canned ``create_response`` / ``token_response``, or a +``raise_transport`` that makes the wire itself fail). +""" + +from __future__ import annotations + +import base64 +import json +import time + +import httpx + +WELCOME = "https://welcome-api.nousresearch.com/v1" +PORTAL = "https://portal.example.test" + + +def make_jwt(**claims) -> str: + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous", + "scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims} + return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig" + + +class FakePortal: + """Minimal NAS anonymous surface. Records every call; scenarios flip its behaviour.""" + + def __init__(self): + self.calls: list[tuple[str, str]] = [] + self.dead_tokens: set[str] = set() + self.gate_closed = False + self.minted = 0 + # What the token exchange names as the inference host; None = an older NAS that omits it. + self.inference_base_url: str | None = WELCOME + # One canned refusal in place of the happy path, and a wire failure in place of any answer. + self.create_response: httpx.Response | None = None + self.token_response: httpx.Response | None = None + self.raise_transport: Exception | None = None + + def creates(self) -> int: + return [p for _, p in self.calls].count("/api/anonymous/create") + + def handler(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + self.calls.append((request.method, path)) + if self.raise_transport is not None: + raise self.raise_transport + if path.startswith("/api/anonymous/") and not request.headers.get("x-anonymous-api-secret"): + return httpx.Response(401, json={"error": "invalid_shared_secret"}) + if self.gate_closed: + return httpx.Response(401, json={"error": "invalid_shared_secret"}) + if path == "/api/anonymous/create": + if self.create_response is not None: + return self.create_response + self.minted += 1 + return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1", + "token": f"anon_{self.minted:04d}", "idle_ttl_days": 14}) + if path == "/api/anonymous/token": + if self.token_response is not None: + return self.token_response + token = json.loads(request.content)["token"] + if token in self.dead_tokens: + return httpx.Response(404, json={"error": "unknown_token"}) + body = {"access_token": make_jwt(), "token_type": "Bearer", "expires_in": 900, + "user_id": "nas_user:1", "org_id": "nas_org:1"} + if self.inference_base_url: + body["inference_base_url"] = self.inference_base_url + return httpx.Response(200, json=body) + return httpx.Response(500, json={"error": f"unexpected {path}"}) + + +def install_portal(monkeypatch, tmp_path, fake: FakePortal | None = None) -> FakePortal: + """Route every Nous HTTP client at *fake*, isolate the stores, and reset the per-process memos. + + One transport seam: ``httpx.Client`` itself, which ``auth_nous._nous_http_client`` and + ``resolve_nous_access_token`` both construct.""" + from hermes_cli import anon_auth, free_tier_bootstrap + from hermes_cli import auth as auth_mod + + fake = fake or FakePortal() + monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL) + monkeypatch.setenv("HERMES_ANON_API_SECRET", "test-secret") + monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store")) + monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1") + for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"): + monkeypatch.delenv(var, raising=False) + real_client = httpx.Client + + class _RoutedClient(real_client): + def __init__(self, *a, **kw): + kw.pop("verify", None) + kw["transport"] = httpx.MockTransport(fake.handler) + super().__init__(*a, **kw) + monkeypatch.setattr(httpx, "Client", _RoutedClient) + monkeypatch.setattr("agent.bedrock_adapter.has_aws_credentials", lambda: False) + anon_auth.reset_mint_memo_for_tests() + free_tier_bootstrap.reset_for_tests() + # resolve_nous_access_token memoises the last token for 5 s per profile home (dict); a token minted + # by an earlier test must not be served to this one. + monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {}) + return fake diff --git a/tests/hermes_cli/test_anon_auth_core.py b/tests/hermes_cli/test_anon_auth_core.py index a04649898b..07f8b244a2 100644 --- a/tests/hermes_cli/test_anon_auth_core.py +++ b/tests/hermes_cli/test_anon_auth_core.py @@ -7,7 +7,6 @@ the wire contract is exercised, never mocked away. from __future__ import annotations -import base64 import json import os import time @@ -18,84 +17,12 @@ import pytest from hermes_cli import anon_auth from hermes_cli.auth import _load_auth_store, resolve_provider - -WELCOME = "https://welcome-api.nousresearch.com/v1" -PORTAL = "https://portal.example.test" - - -def _jwt(**claims) -> str: - def seg(obj): - return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() - payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous", - "scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims} - return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig" - - -class FakePortal: - """Minimal NAS anonymous surface. Records every call; scenarios flip its behaviour.""" - - def __init__(self): - self.calls: list[tuple[str, str]] = [] - self.dead_tokens: set[str] = set() - self.gate_closed = False - self.minted = 0 - # What the token exchange names as the inference host; None = an older NAS that omits it. - self.inference_base_url: str | None = WELCOME - - def handler(self, request: httpx.Request) -> httpx.Response: - path = request.url.path - self.calls.append((request.method, path)) - if path.startswith("/api/anonymous/") and not request.headers.get("x-anonymous-api-secret"): - return httpx.Response(401, json={"error": "invalid_shared_secret"}) - if self.gate_closed: - return httpx.Response(401, json={"error": "invalid_shared_secret"}) - if path == "/api/anonymous/create": - self.minted += 1 - return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1", - "token": f"anon_{self.minted:04d}", "idle_ttl_days": 14}) - if path == "/api/anonymous/token": - token = json.loads(request.content)["token"] - if token in self.dead_tokens: - return httpx.Response(404, json={"error": "unknown_token"}) - body = {"access_token": _jwt(), "token_type": "Bearer", "expires_in": 900, - "user_id": "nas_user:1", "org_id": "nas_org:1"} - if self.inference_base_url: - body["inference_base_url"] = self.inference_base_url - return httpx.Response(200, json=body) - return httpx.Response(500, json={"error": f"unexpected {path}"}) +from tests.hermes_cli.anon_portal import PORTAL, WELCOME, install_portal, make_jwt as _jwt # noqa: F401 @pytest.fixture def portal(monkeypatch, tmp_path): - fake = FakePortal() - monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL) - monkeypatch.setenv("HERMES_ANON_API_SECRET", "test-secret") - monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store")) - monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1") - for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"): - monkeypatch.delenv(var, raising=False) - from hermes_cli import auth_nous - - def _client(timeout_seconds, verify): - return httpx.Client(transport=httpx.MockTransport(fake.handler), base_url=PORTAL) - monkeypatch.setattr(auth_nous, "_nous_http_client", _client) - # resolve_nous_access_token builds its own client; route it through the fake too. - real_client = httpx.Client - - class _RoutedClient(real_client): - def __init__(self, *a, **kw): - kw.pop("verify", None) - kw["transport"] = httpx.MockTransport(fake.handler) - super().__init__(*a, **kw) - monkeypatch.setattr(httpx, "Client", _RoutedClient) - anon_auth.reset_mint_memo_for_tests() - from hermes_cli import free_tier_bootstrap as _fb - _fb.reset_for_tests() - # resolve_nous_access_token memoises the last token for 5 s per profile home (dict); a token minted - # by an earlier test must not be served to this one. - from hermes_cli import auth as auth_mod - monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {}) - return fake + return install_portal(monkeypatch, tmp_path) def _write_config(monkeypatch, **nous): diff --git a/tests/hermes_cli/test_anon_failure_modes.py b/tests/hermes_cli/test_anon_failure_modes.py index 00fcd79c1a..6df978f69e 100644 --- a/tests/hermes_cli/test_anon_failure_modes.py +++ b/tests/hermes_cli/test_anon_failure_modes.py @@ -7,9 +7,7 @@ in ``hermes_cli.anon_auth``), never through mocked-away client code. from __future__ import annotations -import base64 import json -import time import httpx import pytest @@ -17,89 +15,21 @@ import pytest from hermes_cli import anon_auth, anon_sign_in, free_tier_bootstrap from hermes_cli.auth import _load_auth_store -PORTAL = "https://portal.example.test" -WELCOME = "https://welcome-api.nousresearch.com/v1" - - -def _jwt(**claims) -> str: - def seg(obj): - return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() - payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous", - "scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims} - return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig" - - -class FakeNas: - """The anonymous surface as NAS ships it. ``create_response`` / ``token_response`` override the - happy path with one canned refusal; ``raise_transport`` simulates the wire failing.""" - - def __init__(self): - self.calls: list[tuple[str, str]] = [] - self.minted = 0 - self.create_response: httpx.Response | None = None - self.token_response: httpx.Response | None = None - self.raise_transport: Exception | None = None - - def handler(self, request: httpx.Request) -> httpx.Response: - path = request.url.path - self.calls.append((request.method, path)) - if self.raise_transport is not None: - raise self.raise_transport - if path == "/api/anonymous/create": - if self.create_response is not None: - return self.create_response - self.minted += 1 - return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1", - "token": f"anon_{self.minted:04d}", "idle_ttl_days": 14}) - if path == "/api/anonymous/token": - if self.token_response is not None: - return self.token_response - return httpx.Response(200, json={"access_token": _jwt(), "token_type": "Bearer", "expires_in": 900, - "user_id": "nas_user:1", "org_id": "nas_org:1", - "inference_base_url": WELCOME}) - return httpx.Response(500, json={"error": f"unexpected {path}"}) - - def creates(self) -> int: - return [p for _, p in self.calls].count("/api/anonymous/create") +from tests.hermes_cli.anon_portal import PORTAL, WELCOME, install_portal # noqa: F401 @pytest.fixture def nas(monkeypatch, tmp_path): - fake = FakeNas() - monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL) - monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store")) - monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1") - for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"): - monkeypatch.delenv(var, raising=False) - from hermes_cli import auth_nous - - def _client(timeout_seconds, verify): - return httpx.Client(transport=httpx.MockTransport(fake.handler), base_url=PORTAL) - monkeypatch.setattr(auth_nous, "_nous_http_client", _client) - # The runtime resolver builds its own client; route it through the fake too. - real_client = httpx.Client - - class _RoutedClient(real_client): - def __init__(self, *a, **kw): - kw.pop("verify", None) - kw["transport"] = httpx.MockTransport(fake.handler) - super().__init__(*a, **kw) - monkeypatch.setattr(httpx, "Client", _RoutedClient) - monkeypatch.setattr("agent.bedrock_adapter.has_aws_credentials", lambda: False) - anon_auth.reset_mint_memo_for_tests() - free_tier_bootstrap.reset_for_tests() - from hermes_cli import auth as auth_mod - monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {}) - return fake + return install_portal(monkeypatch, tmp_path) -def _mint_error(nas: FakeNas) -> anon_auth.AuthError: +def _mint_error(nas) -> anon_auth.AuthError: with pytest.raises(anon_auth.AuthError) as exc: anon_auth.ensure_portal_identity(explicit=True) return exc.value -def _exchange_error(nas: FakeNas) -> anon_auth.AuthError: +def _exchange_error(nas) -> anon_auth.AuthError: """Mint (the credential is persisted before any exchange), then exchange it at first use.""" from hermes_cli.auth_nous import resolve_nous_runtime_credentials assert anon_auth.is_guest_state(anon_auth.ensure_portal_identity(explicit=True)) @@ -159,6 +89,21 @@ class TestNasRefusalCodes: assert "nous" not in _load_auth_store().get("providers", {}) assert nas.creates() == 1 + def test_a_locked_account_is_never_replaced_through_connectors_either(self, nas): + from hermes_cli.auth import _auth_store_lock, _save_auth_store + from tests.hermes_cli.anon_portal import make_jwt + from tools import managed_tool_gateway as mtg + anon_auth.ensure_portal_identity(explicit=True) + with _auth_store_lock(): + store = _load_auth_store() + store["providers"]["nous"]["expires_at"] = "2000-01-01T00:00:00+00:00" + store["providers"]["nous"]["access_token"] = make_jwt(exp=1) + _save_auth_store(store) + nas.token_response = httpx.Response(403, json={"error": "account_locked"}) + assert mtg.read_nous_access_token() is None + assert "nous" not in _load_auth_store().get("providers", {}) + assert nas.creates() == 1 + def test_unknown_token_is_replaced_once_at_first_use(self, nas): from hermes_cli.auth_nous import resolve_nous_runtime_credentials first = anon_auth.ensure_portal_identity(explicit=True) @@ -293,6 +238,16 @@ class TestBootstrapRecord: assert nas.creates() == 1 + free_tier_bootstrap.BOOTSTRAP_RETRY_ATTEMPTS assert free_tier_bootstrap.current_record().error_code == anon_auth.ANON_UNREACHABLE + def test_a_retry_re_inventories_so_a_provider_connected_meanwhile_keeps_inference(self, nas, monkeypatch): + nas.raise_transport = httpx.ConnectTimeout("no route") + free_tier_bootstrap.run_bootstrap(announce=False) + # The user connected their own provider during the cooldown. + monkeypatch.setattr(free_tier_bootstrap, "_inventory_other_providers", lambda: True) + nas.raise_transport = None + record = free_tier_bootstrap.retry_bootstrap_mint(force=True, announce=False) + assert record.has_identity is True and record.other_providers is True + assert _load_auth_store().get("active_provider") != "nous" + def test_the_desktop_retry_refreshes_the_boot_record(self, nas): nas.raise_transport = httpx.ConnectTimeout("no route") free_tier_bootstrap.run_bootstrap(announce=False) diff --git a/tests/scripts/test_free_tier_fault_server.py b/tests/scripts/test_free_tier_fault_server.py index 4130185b71..84f4213345 100644 --- a/tests/scripts/test_free_tier_fault_server.py +++ b/tests/scripts/test_free_tier_fault_server.py @@ -58,6 +58,12 @@ class TestControlSurface: assert _post(base, "/__reset").json() == {**_post(base, "/__reset").json(), "nas": "ok", "inference": "ok"} + def test_reading_the_log_never_wedges_the_next_request(self, server): + _module, base = server + assert httpx.get(f"{base}/__log", timeout=5.0).status_code == 200 + assert httpx.get(f"{base}/__scenario", timeout=5.0).status_code == 200 + assert httpx.get(f"{base}/__log", timeout=5.0).json()["requests"][-1]["path"] == "/__scenario" + def test_the_control_surface_is_cors_open_for_a_renderer(self, server): _module, base = server preflight = httpx.options(f"{base}/__scenario", timeout=5.0) diff --git a/tools/managed_tool_gateway.py b/tools/managed_tool_gateway.py index 79b022decf..ff45545e04 100644 --- a/tools/managed_tool_gateway.py +++ b/tools/managed_tool_gateway.py @@ -125,16 +125,19 @@ def read_nous_access_token() -> Optional[str]: from hermes_cli.anon_auth import AnonCredentialDead if isinstance(exc, AnonCredentialDead): - return _replace_dead_guest_token(nous_provider) + return _replace_dead_guest_token(nous_provider, str(exc.code or "anon_credential_dead")) logger.debug("Nous access token refresh failed: %s", exc) return cached_token -def _replace_dead_guest_token(dead_state: dict) -> Optional[str]: - from hermes_cli.anon_auth import clear_dead_guest, ensure_portal_identity +def _replace_dead_guest_token(dead_state: dict, code: str = "anon_credential_dead") -> Optional[str]: + from hermes_cli.anon_auth import ANON_ACCOUNT_LOCKED, clear_dead_guest, ensure_portal_identity from hermes_cli.auth import resolve_nous_access_token - clear_dead_guest("anon_credential_dead", dead_token=dead_state.get("anon_token")) + clear_dead_guest(code, dead_token=dead_state.get("anon_token")) + # Same rule as inference: a locked account is retired but never silently replaced. + if code == ANON_ACCOUNT_LOCKED: + return None try: if ensure_portal_identity(explicit=True) is None: return None