fix(free-tier): review follow-ups — read the classifier's context, never replace a locked identity, re-inventory on retry

Correctness
- The welcome-tier recovery hooks (model_not_free move, wrong-host heal) and
  the long-wait rate-limit check read the turn's extract_api_error_context()
  dict, which never carries welcome_refusal / welcome_route. They now read
  classified.error_context, where _nous_welcome_tier parks them; the guard
  records the classifier's reset_at. Tests drive the real classifier and the
  real extractor so the two-context boundary is exercised.
- The connector path caught every AnonCredentialDead and re-minted; a locked
  account (anon_account_locked) is now retired without replacement, matching
  the inference resolver.
- A background bootstrap retry reused the boot-time provider inventory; it
  re-inventories, so a provider connected during the cooldown keeps
  inference.
- The desktop's setup.ready listener only refreshes an untouched picker
  (oauth mode, no local endpoint, idle flow) and re-checks after the
  readiness round, so an API-key form opened meanwhile is never dismissed.
- /__log on the rehearsal server sent its response while holding the state
  lock that _send re-acquires; the log is copied out first.

Reductions
- One shared FakePortal / install_portal (tests/hermes_cli/anon_portal.py)
  behind both free-tier fixtures, with a single httpx.Client transport seam.
- The rehearsal server's static inference answers are a table; dead
  scaffolding (REAL_PAID_URL, claim_codes, the no-op dead_once branch,
  extra_headers) removed.
- Setup-notice copy is a code-to-key map; its test uses real codes (the old
  loop built nonexistent ones and only exercised the fallback).
- The ineffective FreeTierErrorCode union is gone.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Robin Fernandes
2026-09-14 15:50:28 +10:00
committed by kshitij
parent 51e39af967
commit 59fad62a40
16 changed files with 351 additions and 314 deletions

View File

@@ -480,14 +480,18 @@ def _recover_format_errors(
return False
def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState, error_context: Any) -> bool:
def _recover_welcome_tier(agent: Any, classified: Any, _retry: TurnRetryState) -> bool:
"""Two one-shot repairs for the Nous free tier, both silent on the wire and named once in chat.
``model_not_free``: the session asked the welcome host for a model it does not serve; move
to the first alternate the gateway named (its own model) and retry, instead of failing the
turn. ``anon_on_paid_host``: this process is pointed at the paid host with a free-tier
identity (a stale route); re-read the credentials, which heals the URL, and retry."""
ctx = error_context if isinstance(error_context, dict) else (getattr(classified, "error_context", None) or {})
identity (a stale route); re-read the credentials, which heals the URL, and retry.
Reads the CLASSIFIER's context (``classified.error_context``): that is where
``_nous_welcome_tier`` parks ``welcome_refusal`` / ``welcome_route``. The turn's other context
(``extract_api_error_context``) never carries them."""
ctx = getattr(classified, "error_context", None) or {}
refusal = ctx.get("welcome_refusal") if isinstance(ctx, dict) else None
if isinstance(refusal, dict) and refusal.get("reason") == "model_not_free" and not _retry.welcome_model_switch_attempted:
_retry.welcome_model_switch_attempted = True
@@ -529,7 +533,7 @@ def recover_after_classification(
Returns ``(retry_now, recovered_with_pool)``; the latter feeds the Nous rate-limit guard."""
from agent.conversation_loop import _is_nous_inference_route
if _recover_welcome_tier(agent, classified, _retry, error_context):
if _recover_welcome_tier(agent, classified, _retry):
return True, False
if (
@@ -1361,23 +1365,27 @@ def _eager_fallback_status(classified: Any, is_upstream: bool, is_transport_fail
return "⚠️ Rate limited — switching to fallback provider..."
def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: Any) -> bool:
def _is_genuine_nous_rate_limit(agent: Any, api_error: Exception, error_context: Any, classified: Any = None) -> bool:
"""Record a genuine account-level Nous 429 to the cross-session breaker; upstream
capacity 429s (no exhausted bucket in headers or last-known state) are left alone."""
capacity 429s (no exhausted bucket in headers or last-known state) are left alone.
*error_context* is the turn's (``extract_api_error_context``); *classified* brings the
classifier's own context, where a welcome-tier ``rate_limited`` refusal and its ``reset_at``
live. A long welcome reset is an exhausted allowance whatever the headers say, and the one
place the user is told that signing in lifts it."""
_genuine = False
try:
from agent.nous_rate_guard import (
is_genuine_nous_rate_limit, is_long_welcome_rate_limit, record_nous_rate_limit)
_err_resp = getattr(api_error, "response", None)
_err_hdrs = getattr(_err_resp, "headers", None) if _err_resp else None
# The welcome tier's structured ``rate_limited`` refusal names its own reset; a long one
# is an exhausted allowance whatever the headers say, and the one place the user is told
# that signing in lifts it.
_classified_ctx = getattr(classified, "error_context", None) or {}
_genuine = (
is_long_welcome_rate_limit(error_context)
is_long_welcome_rate_limit(_classified_ctx)
or is_genuine_nous_rate_limit(headers=_err_hdrs, last_known_state=agent._rate_limit_state))
if _genuine:
record_nous_rate_limit(headers=_err_hdrs, error_context=error_context)
_merged = {**(error_context if isinstance(error_context, dict) else {}), **_classified_ctx}
record_nous_rate_limit(headers=_err_hdrs, error_context=_merged)
else:
logger.info(
"Nous 429 looks like upstream capacity "
@@ -1553,7 +1561,7 @@ def route_classified_error(
and agent.provider == "nous"
and classified.reason == FailoverReason.rate_limit
and not recovered_with_pool
and _is_genuine_nous_rate_limit(agent, api_error, error_context)
and _is_genuine_nous_rate_limit(agent, api_error, error_context, classified)
):
# Re-enter the loop exactly once so the top-of-loop Nous guard runs
# (retry_count = max_retries would skip it entirely).

View File

@@ -75,3 +75,12 @@ describe('acknowledging the introduction', () => {
expect(await ackFreeTierIntro({ requestGateway: gatewayReturning(READY) })).toBe(false) // status stub returns no {acked: true}
})
})
describe('a background readiness round', () => {
it('never completes onboarding when the picker was touched during the round', async () => {
// The `setup.ready` listener passes `stillWanted`; a user opening the API-key form while the
// readiness request was out must not have it dismissed by a late "ready".
expect(await refreshOnboarding({ requestGateway: gatewayReturning(READY) }, () => false)).toBe(false)
expect($desktopOnboarding.get().freeTierReady).toBe(false)
})
})

View File

@@ -54,33 +54,25 @@ describe('setupFailureCopy', () => {
['anon_unreachable', copy.unreachable],
['anon_server_error', copy.serverError],
['anon_pow_required', copy.powRequired],
['anon_account_locked', copy.locked]
])('%s has its own sentence', (code, expected) => {
const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: code })
['anon_account_locked', copy.locked],
['anon_rate_limited', copy.rateLimited('about 5 minutes')]
])('%s has its own sentence, in the agreed voice', (code, expected) => {
const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: code, retry_after: 300 })
const text = failure ? setupFailureCopy(failure, copy) : ''
expect(failure && setupFailureCopy(failure, copy)).toBe(expected)
})
it('speaks the wait for a rate limit', () => {
const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: 'anon_rate_limited', retry_after: 300 })
expect(failure && setupFailureCopy(failure, copy)).toContain('about 5 minutes')
expect(text).toBe(expected)
// Never "the free service is off" — what is unavailable is using Hermes without signing in —
// and no jargon a first-time user would not know.
expect(text.toLowerCase()).not.toMatch(/free (service|model|tier) is (off|switched off|unavailable|down)/)
expect(text.toLowerCase()).not.toMatch(/anonymous|guest|credential|token|rate limit/)
})
it('falls back to the backend sentence for a code this build does not know', () => {
const failure = freeTierSetupFailure({ ...NO_IDENTITY, error: 'Something new.', error_code: 'anon_newer' })
expect(failure && setupFailureCopy(failure, copy)).toBe('Something new.')
})
it('never says the free service or the free model is off', () => {
for (const code of Object.keys(copy)) {
const failure = freeTierSetupFailure({ ...NO_IDENTITY, error_code: `anon_${code}` })
const text = failure ? setupFailureCopy(failure, copy).toLowerCase() : ''
expect(text).not.toMatch(/free (service|model|tier) is (off|switched off|unavailable|down)/)
expect(text).not.toMatch(/anonymous|guest|credential|token|rate limit/)
}
// Prototype names are not codes.
expect(setupFailureCopy({ ...failure!, code: 'constructor', message: '' }, copy)).toBe(copy.generic)
})
})

View File

@@ -15,34 +15,27 @@ import { type OnboardingContext, refreshOnboarding } from '@/store/onboarding'
type SetupFailedCopy = Translations['freeTier']['setupFailed']
/** One sentence per backend code. The backend's own sentence is the fallback for
* a code this build does not know, so a newer backend still reads as words. */
// One sentence per backend code (`hermes_cli/anon_auth.py::ANON_*`).
const COPY_KEY_BY_CODE: Record<string, keyof SetupFailedCopy> = {
anon_account_locked: 'locked',
anon_gate_closed: 'gateClosed',
anon_gate_paused: 'paused',
anon_pow_required: 'powRequired',
anon_server_error: 'serverError',
anon_unreachable: 'unreachable'
}
/** The sentence for a failure. The backend's own sentence is the fallback for a
* code this build does not know, so a newer backend still reads as words. */
export function setupFailureCopy(failure: FreeTierSetupFailure, copy: SetupFailedCopy): string {
switch (failure.code) {
case 'anon_gate_closed':
return copy.gateClosed
case 'anon_gate_paused':
return copy.paused
case 'anon_rate_limited':
return copy.rateLimited(friendlyWait(failure.retryAfter || 60))
case 'anon_unreachable':
return copy.unreachable
case 'anon_server_error':
return copy.serverError
case 'anon_pow_required':
return copy.powRequired
case 'anon_account_locked':
return copy.locked
default:
return failure.message || copy.generic
if (failure.code === 'anon_rate_limited') {
return copy.rateLimited(friendlyWait(failure.retryAfter || 60))
}
const key = Object.hasOwn(COPY_KEY_BY_CODE, failure.code) ? COPY_KEY_BY_CODE[failure.code] : null
const sentence = key ? copy[key] : null
return typeof sentence === 'string' ? sentence : failure.message || copy.generic
}
/**

View File

@@ -287,15 +287,27 @@ export function DesktopOnboardingOverlay({
// The boot bootstrap re-announces `setup.ready` when a background retry of
// the free-tier set-up succeeds after a failed first attempt. A picker that
// is up only because that set-up failed re-checks readiness and gives way
// on its own; a manual open, or a picker the user is mid-flow in, is left
// alone.
// on its own. An untouched picker only: a manual open, a provider flow in
// progress, or the API-key form (which leaves the flow idle while the user
// types) is left alone, and the check is repeated after the readiness
// round so a key form opened in the meantime survives too.
useEffect(
() =>
$setupReadyTick.listen(() => {
const current = $desktopOnboarding.get()
const untouched = () => {
const current = $desktopOnboarding.get()
if (!current.manual && current.configured === false && current.flow.status === 'idle') {
void refreshOnboarding(ctx)
return (
!current.manual &&
current.configured === false &&
current.flow.status === 'idle' &&
current.mode === 'oauth' &&
!current.localEndpoint
)
}
if (untouched()) {
void refreshOnboarding(ctx, untouched)
}
}),
[ctx]

View File

@@ -1,7 +1,7 @@
import { atom } from 'nanostores'
import { onboardingSurfaceActive } from '@/store/onboarding-presence'
import type { FreeTierErrorCode, FreeTierStatus } from '@/types/hermes'
import type { FreeTierStatus } from '@/types/hermes'
/** The model the free-tier route runs on. Used to recognise a session that is
* still homed on the free tier after a sign-in. */
@@ -62,7 +62,8 @@ export async function refreshFreeTierStatus(requestGateway: FreeTierRequester):
* codes get "try again / another provider" only.
*/
export interface FreeTierSetupFailure {
code: FreeTierErrorCode | string
/** One of the backend's `anon_*` codes (`hermes_cli/anon_auth.py`), or a newer one this build does not know. */
code: string
door: 'retry' | 'sign_in'
message: string
retryAfter: number

View File

@@ -671,7 +671,13 @@ export function setOnboardingMode(mode: OnboardingMode) {
patch({ mode })
}
export async function refreshOnboarding(ctx: OnboardingContext) {
/**
* `stillWanted`, when given, is re-asked after the readiness round: a background
* caller (the `setup.ready` listener) passes it so a user action that started
* during the round — opening the API-key form, picking a provider — is never
* dismissed by a late "ready".
*/
export async function refreshOnboarding(ctx: OnboardingContext, stillWanted?: () => boolean) {
// Manual mode (user opened the selector from a working app): never
// auto-dismiss on runtime-ready — the whole point is to let them add /
// switch a provider while already configured. Just ensure the provider
@@ -684,6 +690,10 @@ export async function refreshOnboarding(ctx: OnboardingContext) {
const runtime = await checkRuntime(ctx)
if (stillWanted && !stillWanted()) {
return false
}
if (runtime.ready) {
completeDesktopOnboarding()
await applyFreeTierIntro(ctx, runtime)

View File

@@ -164,17 +164,6 @@ export interface FreeTierStatus {
retry_after?: number
}
/** The backend's free-tier failure codes (`hermes_cli/anon_auth.py::ANON_*`). */
export type FreeTierErrorCode =
| 'anon_account_locked'
| 'anon_credential_dead'
| 'anon_gate_closed'
| 'anon_gate_paused'
| 'anon_pow_required'
| 'anon_rate_limited'
| 'anon_server_error'
| 'anon_unreachable'
export interface MemoryProviderOAuthStatus {
auth: 'apikey' | 'oauth' | null
connected: boolean

View File

@@ -190,7 +190,9 @@ def retry_bootstrap_mint(*, force: bool = False, announce: bool = True) -> Setup
return run_bootstrap(announce=announce)
if current.has_identity:
return current
record = _build_record(other=current.other_providers, force=force)
# Re-inventory: a provider the user connected during the cooldown must keep inference; the
# boot-time answer is stale by now.
record = _build_record(other=_inventory_other_providers(), force=force)
with _lock:
_record = record
if announce:

View File

@@ -43,8 +43,6 @@ from urllib.parse import parse_qs, urlparse
WELCOME_MODEL = "nous/welcome"
REAL_WELCOME_HOST = "welcome-api.nousresearch.com"
REAL_PAID_URL = "https://inference-api.nousresearch.com"
GENERIC_403 = "You tried to access something that you don't have permissions for."
UPGRADE_URL = "https://portal.nousresearch.com/signup"
# --- Scenario catalogue --------------------------------------------------------------------------
@@ -83,6 +81,35 @@ INFERENCE_SCENARIOS: Dict[str, str] = {
}
_FAIRSHARE_MESSAGE = ("You've reached this model's current fair-share rate limit. It adapts to demand — "
"retry after the indicated delay, or try an alternate model.")
def _fairshare(reason: str, message: str, **extra: Any) -> Dict[str, Any]:
return {"status": 429, "message": message, "reason": reason, "alternates": [], "upgrade_url": UPGRADE_URL, **extra}
# Static inference answers: scenario -> (status, body, default Retry-After seconds). ``retry_after``
# in a fairshare body and every wait header are filled in per request from the live override.
INFERENCE_RESPONSES: Dict[str, tuple] = {
"rate_limited": (429, _fairshare("rate_limited", _FAIRSHARE_MESSAGE), 600),
"rate_limited_short": (429, _fairshare("rate_limited", _FAIRSHARE_MESSAGE), 5),
"at_capacity": (429, _fairshare("at_capacity", "The free tier is at capacity and briefly paused. It reopens "
"automatically — retry after the indicated delay."), 30),
"model_not_free": (429, _fairshare("model_not_free", "This model isn't available on the free tier.",
alternates=[WELCOME_MODEL]), 0),
"tier_disabled": (403, {"status": 403, "message": "You tried to access something that you don't have permissions for."}, 0),
"wrong_host": (400, {"status": 400, "message": "This request is not valid. Check the model name and other parameters. "
f"Additional info: Anonymous accounts must use https://{REAL_WELCOME_HOST} for inference."}, 0),
"bare_429": (429, {"status": 429, "message": "Hold up for a bit, you've exceeded the rate limit on your API key."}, 600),
"upstream_503": (503, {"status": 503, "message": "The requested model is currently unavailable."}, 0),
"upstream_500": (500, {"status": 500, "message": "Something unexpected happened while processing your request. "
"Please try again in a moment, or contact us if the issue persists."}, 0),
"invalid_token": (401, {"status": 401, "error": "invalid_token", "subcause": "anonymous_credential_revoked",
"message": "The anonymous account behind this token is gone"}, 0),
}
def _jwt(**claims: Any) -> str:
def seg(obj: Any) -> str:
return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode()
@@ -105,7 +132,6 @@ class State:
self.minted = 0
self.dead_tokens: set[str] = set()
self.signin: Dict[str, Any] = {"status": "pending"}
self.claim_codes: Dict[str, str] = {}
self.log: deque[Dict[str, Any]] = deque(maxlen=100)
def snapshot(self) -> Dict[str, Any]:
@@ -146,7 +172,6 @@ class State:
self.retry_after = None
self.dead_tokens.clear()
self.signin = {"status": "pending"}
self.claim_codes.clear()
self.log.clear()
@@ -210,7 +235,8 @@ class Handler(BaseHTTPRequestHandler):
self._send(200, STATE.snapshot())
elif path == "/__log":
with STATE.lock:
self._send(200, {"requests": list(STATE.log)})
entries = list(STATE.log)
self._send(200, {"requests": entries}) # outside the lock: _send appends to the log
elif path == "/v1/models":
self._send(200, {"object": "list", "data": [{"id": WELCOME_MODEL, "object": "model", "owned_by": "nous"}]})
elif path.startswith("/__claim"):
@@ -292,11 +318,8 @@ class Handler(BaseHTTPRequestHandler):
with STATE.lock:
STATE.minted += 1
n = STATE.minted
token = f"anon_rehearsal_{n:04d}_{secrets.token_hex(4)}"
if scenario == "dead_once":
pass # the credential itself is fine; its first exchange is what dies
self._send(201, {"user_id": f"nas_user:rehearsal-{n}", "org_id": f"nas_org:rehearsal-{n}",
"token": token, "idle_ttl_days": 14})
"token": f"anon_rehearsal_{n:04d}_{secrets.token_hex(4)}", "idle_ttl_days": 14})
return
if path == "/api/anonymous/token":
token = str(body.get("token") or "")
@@ -356,7 +379,6 @@ class Handler(BaseHTTPRequestHandler):
return
code = f"{secrets.token_hex(2).upper()}-{secrets.token_hex(2).upper()}"
with STATE.lock:
STATE.claim_codes[code] = str(body.get("token") or "")
STATE.signin = {"status": "pending"}
host = self.headers.get("Host") or "127.0.0.1"
self._send(200, {"claim_code": code, "claim_url": f"http://{host}/__claim?code={code}",
@@ -388,65 +410,31 @@ class Handler(BaseHTTPRequestHandler):
# --- the welcome inference host --------------------------------------------------------------
def _refusal(self, status: int, message: str, *, reason: str, retry_after: int,
alternates: Optional[list] = None, extra_headers: Optional[Dict[str, str]] = None) -> None:
headers = {"Retry-After": str(retry_after)}
if reason in ("rate_limited", "at_capacity"):
headers["RateLimit-Policy"] = '"fairshare";q=0;qu="tokens";w=60'
headers["RateLimit"] = f'"fairshare";r=0;t={retry_after}'
headers.update(extra_headers or {})
STATE.consume_once("inference")
self._send(status, {"status": status, "message": message, "reason": reason, "retry_after": retry_after,
"alternates": alternates or [], "upgrade_url": UPGRADE_URL}, headers=headers)
def _inference(self, body: Dict[str, Any]) -> None:
with STATE.lock:
scenario = STATE.inference
model = str(body.get("model") or WELCOME_MODEL)
if scenario == "rate_limited":
self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — "
"retry after the indicated delay, or try an alternate model.",
reason="rate_limited", retry_after=self._retry_after(600))
elif scenario == "rate_limited_short":
self._refusal(429, "You've reached this model's current fair-share rate limit. It adapts to demand — "
"retry after the indicated delay, or try an alternate model.",
reason="rate_limited", retry_after=self._retry_after(5))
elif scenario == "at_capacity":
self._refusal(429, "The free tier is at capacity and briefly paused. It reopens automatically — "
"retry after the indicated delay.", reason="at_capacity", retry_after=self._retry_after(30))
elif scenario == "model_not_free":
self._refusal(429, "This model isn't available on the free tier.", reason="model_not_free",
retry_after=0, alternates=[WELCOME_MODEL])
elif scenario == "tier_disabled":
STATE.consume_once("inference")
self._send(403, {"status": 403, "message": GENERIC_403})
elif scenario == "wrong_host":
STATE.consume_once("inference")
self._send(400, {"status": 400, "message": "This request is not valid. Check the model name and other "
f"parameters. Additional info: Anonymous accounts must use https://{REAL_WELCOME_HOST} for inference."})
elif scenario == "bare_429":
STATE.consume_once("inference")
self._send(429, {"status": 429, "message": "Hold up for a bit, you've exceeded the rate limit on your API key."},
headers={"x-ratelimit-limit-requests": "30", "x-ratelimit-remaining-requests": "0",
"x-ratelimit-reset-requests": str(self._retry_after(600)),
"Retry-After": str(self._retry_after(600))})
elif scenario == "upstream_503":
STATE.consume_once("inference")
self._send(503, {"status": 503, "message": "The requested model is currently unavailable."})
elif scenario == "upstream_500":
STATE.consume_once("inference")
self._send(500, {"status": 500, "message": "Something unexpected happened while processing your request. "
"Please try again in a moment, or contact us if the issue persists."})
elif scenario == "invalid_token":
STATE.consume_once("inference")
self._send(401, {"status": 401, "error": "invalid_token", "subcause": "anonymous_credential_revoked",
"message": "The anonymous account behind this token is gone"})
elif scenario == "timeout":
if scenario == "timeout":
STATE.consume_once("inference")
time.sleep(120)
self._send(504, {"status": 504, "message": "timed out"})
else:
self._reply(model, stream=bool(body.get("stream")))
return
canned = INFERENCE_RESPONSES.get(scenario)
if canned is None:
self._reply(str(body.get("model") or WELCOME_MODEL), stream=bool(body.get("stream")))
return
status, payload, default_wait = canned
wait = self._retry_after(default_wait)
headers = {"Retry-After": str(wait)} if default_wait or "reason" in payload else {}
if payload.get("reason") in ("rate_limited", "at_capacity"):
headers["RateLimit-Policy"] = '"fairshare";q=0;qu="tokens";w=60'
headers["RateLimit"] = f'"fairshare";r=0;t={wait}'
if scenario == "bare_429":
headers.update({"x-ratelimit-limit-requests": "30", "x-ratelimit-remaining-requests": "0",
"x-ratelimit-reset-requests": str(wait)})
if "reason" in payload:
payload = {**payload, "retry_after": wait}
STATE.consume_once("inference")
self._send(status, payload, headers=headers)
def _reply(self, model: str, *, stream: bool) -> None:
text = ("Hello from the free tier fault server. Everything is working; switch a scenario on "

View File

@@ -1,6 +1,10 @@
"""Nous free tier, inference side: the dark-tier 403 keyed on the route, the one-shot model move
after ``model_not_free``, the wrong-host heal, the long-wait rule for structured ``rate_limited``
refusals, and the plain outage sentence once retries are spent."""
refusals, and the plain outage sentence once retries are spent.
The recovery hooks are driven from the REAL producer boundary: a gateway body goes through
``classify_api_error`` (and the turn's own ``extract_api_error_context``) exactly as the turn loop
feeds them, so a wiring slip between the two contexts fails here."""
from __future__ import annotations
@@ -8,6 +12,7 @@ from types import SimpleNamespace
import pytest
from agent.agent_runtime_helpers import extract_api_error_context
from agent.error_classifier import FailoverReason, classify_api_error
from agent.turn_retry_state import TurnRetryState
@@ -23,21 +28,32 @@ class MockAPIError(Exception):
self.body = body
def _gateway_error(status: int, body: dict) -> MockAPIError:
return MockAPIError(f"Error code: {status} - {body}", status_code=status, body=body)
def _generic_403():
body = {"status": 403, "message": "You tried to access something that you don't have permissions for."}
return MockAPIError(f"Error code: 403 - {body}", status_code=403, body=body)
return _gateway_error(403, {"status": 403, "message": "You tried to access something that you don't have permissions for."})
def _refusal(reason: str, *, retry_after: int = 0, alternates=None) -> MockAPIError:
return _gateway_error(429, {"status": 429, "message": "refused", "reason": reason, "retry_after": retry_after,
"alternates": alternates or [], "upgrade_url": "https://portal.example/signup"})
def _classify(err: MockAPIError, *, model: str = "nous/welcome", base_url: str = WELCOME):
return classify_api_error(err, provider="nous", model=model, base_url=base_url)
class TestDarkTier403:
def test_a_generic_403_from_the_welcome_host_is_the_tier_refusing(self):
result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=WELCOME)
result = _classify(_generic_403())
assert result.reason == FailoverReason.auth_permanent
assert result.retryable is False and result.should_fallback is True
assert result.error_context["welcome_route"] == "tier_disabled"
def test_the_same_403_from_the_paid_host_stays_an_ordinary_403(self):
result = classify_api_error(_generic_403(), provider="nous", model="nous/welcome", base_url=PAID)
assert "welcome_route" not in result.error_context
assert "welcome_route" not in _classify(_generic_403(), base_url=PAID).error_context
def test_a_403_from_another_provider_on_any_host_is_untouched(self):
result = classify_api_error(_generic_403(), provider="openrouter", base_url=WELCOME)
@@ -47,7 +63,7 @@ class TestDarkTier403:
def _agent(**overrides):
lines = []
agent = SimpleNamespace(
provider="nous", model="gpt-5", base_url=WELCOME, log_prefix="",
provider="nous", model="gpt-5", base_url=WELCOME, log_prefix="", _rate_limit_state=None,
_vprint=lambda text, force=False: lines.append(text),
_try_refresh_nous_client_credentials=lambda **kw: True,
)
@@ -61,38 +77,39 @@ class TestOneShotRecoveries:
def test_model_not_free_moves_the_session_onto_the_alternate_and_retries_once(self):
from agent.turn_recovery import _recover_welcome_tier
agent = _agent()
ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0,
"alternates": ["nous/welcome"], "upgrade_url": ""}}
classified = _classify(_refusal("model_not_free", alternates=["nous/welcome"]), model="gpt-5")
retry = TurnRetryState()
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True
assert _recover_welcome_tier(agent, classified, retry) is True
assert agent.model == "nous/welcome"
assert agent._nous_model_switch == ("gpt-5", "nous/welcome")
assert "without signing in" in agent.lines[0]
# Once: a second refusal in the same attempt falls through to the terminal path.
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False
assert _recover_welcome_tier(agent, classified, retry) is False
def test_model_not_free_without_an_alternate_does_nothing(self):
from agent.turn_recovery import _recover_welcome_tier
agent = _agent()
ctx = {"welcome_refusal": {"reason": "model_not_free", "retry_after": 0, "alternates": [], "upgrade_url": ""}}
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False
classified = _classify(_refusal("model_not_free"), model="gpt-5")
assert _recover_welcome_tier(agent, classified, TurnRetryState()) is False
assert agent.model == "gpt-5"
def test_a_wrong_host_refusal_re_reads_the_route_once(self):
from agent.turn_recovery import _recover_welcome_tier
calls = []
agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: calls.append(kw) or True)
ctx = {"welcome_route": "anon_on_paid_host"}
body = {"status": 400, "message": "Anonymous accounts must use https://welcome-api.nousresearch.com for inference."}
classified = _classify(_gateway_error(400, body), base_url=PAID)
assert classified.error_context["welcome_route"] == "anon_on_paid_host"
retry = TurnRetryState()
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is True
assert _recover_welcome_tier(agent, classified, retry) is True
assert calls == [{"force": True}]
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), retry, ctx) is False
assert _recover_welcome_tier(agent, classified, retry) is False
def test_a_wrong_host_refusal_whose_heal_fails_falls_through(self):
from agent.turn_recovery import _recover_welcome_tier
agent = _agent(_try_refresh_nous_client_credentials=lambda **kw: False)
ctx = {"welcome_route": "anon_on_paid_host"}
assert _recover_welcome_tier(agent, SimpleNamespace(error_context=ctx), TurnRetryState(), ctx) is False
body = {"status": 400, "message": "Anonymous accounts must use https://welcome-api.nousresearch.com for inference."}
assert _recover_welcome_tier(agent, _classify(_gateway_error(400, body), base_url=PAID), TurnRetryState()) is False
class TestLongWaitRule:
@@ -102,13 +119,33 @@ class TestLongWaitRule:
])
def test_only_a_long_rate_limited_refusal_is_an_exhausted_allowance(self, reason, retry_after, expected):
from agent.nous_rate_guard import is_long_welcome_rate_limit
ctx = {"welcome_refusal": {"reason": reason, "retry_after": retry_after, "alternates": [], "upgrade_url": ""}}
assert is_long_welcome_rate_limit(ctx) is expected
classified = _classify(_refusal(reason, retry_after=retry_after))
assert is_long_welcome_rate_limit(classified.error_context) is expected
def test_no_refusal_is_not_long(self):
from agent.nous_rate_guard import is_long_welcome_rate_limit
assert is_long_welcome_rate_limit({}) is False and is_long_welcome_rate_limit(None) is False
def test_the_turn_records_a_long_refusal_from_the_classifiers_context(self, monkeypatch):
"""The turn hands the guard TWO contexts: its own (``extract_api_error_context``), which
never carries ``welcome_refusal``, and the classifier's, which does. The breaker must key on
the latter and record the reset it computed."""
import agent.nous_rate_guard as guard
from agent.turn_recovery import _is_genuine_nous_rate_limit
recorded = []
monkeypatch.setattr(guard, "record_nous_rate_limit", lambda **kw: recorded.append(kw))
err = _refusal("rate_limited", retry_after=600)
turn_ctx = extract_api_error_context(err)
assert "welcome_refusal" not in turn_ctx
classified = _classify(err)
assert _is_genuine_nous_rate_limit(_agent(), err, turn_ctx, classified) is True
assert recorded and recorded[0]["error_context"]["reset_at"] == classified.error_context["reset_at"]
# A short one is not an exhausted allowance: nothing recorded, the turn waits it out.
recorded.clear()
short = _refusal("rate_limited", retry_after=5)
assert _is_genuine_nous_rate_limit(_agent(), short, extract_api_error_context(short), _classify(short)) is False
assert recorded == []
class TestOutageCopy:
@pytest.mark.parametrize("reason", [FailoverReason.timeout, FailoverReason.overloaded,

View File

@@ -0,0 +1,105 @@
"""The fake NAS anonymous surface shared by the free-tier tests.
One ``FakePortal`` and one ``install_portal`` behind every ``portal`` fixture: the wire contract is
exercised through Hermes' real client code, never mocked away. Scenarios flip its behaviour
(``gate_closed``, ``dead_tokens``, a canned ``create_response`` / ``token_response``, or a
``raise_transport`` that makes the wire itself fail).
"""
from __future__ import annotations
import base64
import json
import time
import httpx
WELCOME = "https://welcome-api.nousresearch.com/v1"
PORTAL = "https://portal.example.test"
def make_jwt(**claims) -> str:
def seg(obj):
return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode()
payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous",
"scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims}
return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig"
class FakePortal:
"""Minimal NAS anonymous surface. Records every call; scenarios flip its behaviour."""
def __init__(self):
self.calls: list[tuple[str, str]] = []
self.dead_tokens: set[str] = set()
self.gate_closed = False
self.minted = 0
# What the token exchange names as the inference host; None = an older NAS that omits it.
self.inference_base_url: str | None = WELCOME
# One canned refusal in place of the happy path, and a wire failure in place of any answer.
self.create_response: httpx.Response | None = None
self.token_response: httpx.Response | None = None
self.raise_transport: Exception | None = None
def creates(self) -> int:
return [p for _, p in self.calls].count("/api/anonymous/create")
def handler(self, request: httpx.Request) -> httpx.Response:
path = request.url.path
self.calls.append((request.method, path))
if self.raise_transport is not None:
raise self.raise_transport
if path.startswith("/api/anonymous/") and not request.headers.get("x-anonymous-api-secret"):
return httpx.Response(401, json={"error": "invalid_shared_secret"})
if self.gate_closed:
return httpx.Response(401, json={"error": "invalid_shared_secret"})
if path == "/api/anonymous/create":
if self.create_response is not None:
return self.create_response
self.minted += 1
return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1",
"token": f"anon_{self.minted:04d}", "idle_ttl_days": 14})
if path == "/api/anonymous/token":
if self.token_response is not None:
return self.token_response
token = json.loads(request.content)["token"]
if token in self.dead_tokens:
return httpx.Response(404, json={"error": "unknown_token"})
body = {"access_token": make_jwt(), "token_type": "Bearer", "expires_in": 900,
"user_id": "nas_user:1", "org_id": "nas_org:1"}
if self.inference_base_url:
body["inference_base_url"] = self.inference_base_url
return httpx.Response(200, json=body)
return httpx.Response(500, json={"error": f"unexpected {path}"})
def install_portal(monkeypatch, tmp_path, fake: FakePortal | None = None) -> FakePortal:
"""Route every Nous HTTP client at *fake*, isolate the stores, and reset the per-process memos.
One transport seam: ``httpx.Client`` itself, which ``auth_nous._nous_http_client`` and
``resolve_nous_access_token`` both construct."""
from hermes_cli import anon_auth, free_tier_bootstrap
from hermes_cli import auth as auth_mod
fake = fake or FakePortal()
monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL)
monkeypatch.setenv("HERMES_ANON_API_SECRET", "test-secret")
monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store"))
monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1")
for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"):
monkeypatch.delenv(var, raising=False)
real_client = httpx.Client
class _RoutedClient(real_client):
def __init__(self, *a, **kw):
kw.pop("verify", None)
kw["transport"] = httpx.MockTransport(fake.handler)
super().__init__(*a, **kw)
monkeypatch.setattr(httpx, "Client", _RoutedClient)
monkeypatch.setattr("agent.bedrock_adapter.has_aws_credentials", lambda: False)
anon_auth.reset_mint_memo_for_tests()
free_tier_bootstrap.reset_for_tests()
# resolve_nous_access_token memoises the last token for 5 s per profile home (dict); a token minted
# by an earlier test must not be served to this one.
monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {})
return fake

View File

@@ -7,7 +7,6 @@ the wire contract is exercised, never mocked away.
from __future__ import annotations
import base64
import json
import os
import time
@@ -18,84 +17,12 @@ import pytest
from hermes_cli import anon_auth
from hermes_cli.auth import _load_auth_store, resolve_provider
WELCOME = "https://welcome-api.nousresearch.com/v1"
PORTAL = "https://portal.example.test"
def _jwt(**claims) -> str:
def seg(obj):
return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode()
payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous",
"scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims}
return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig"
class FakePortal:
"""Minimal NAS anonymous surface. Records every call; scenarios flip its behaviour."""
def __init__(self):
self.calls: list[tuple[str, str]] = []
self.dead_tokens: set[str] = set()
self.gate_closed = False
self.minted = 0
# What the token exchange names as the inference host; None = an older NAS that omits it.
self.inference_base_url: str | None = WELCOME
def handler(self, request: httpx.Request) -> httpx.Response:
path = request.url.path
self.calls.append((request.method, path))
if path.startswith("/api/anonymous/") and not request.headers.get("x-anonymous-api-secret"):
return httpx.Response(401, json={"error": "invalid_shared_secret"})
if self.gate_closed:
return httpx.Response(401, json={"error": "invalid_shared_secret"})
if path == "/api/anonymous/create":
self.minted += 1
return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1",
"token": f"anon_{self.minted:04d}", "idle_ttl_days": 14})
if path == "/api/anonymous/token":
token = json.loads(request.content)["token"]
if token in self.dead_tokens:
return httpx.Response(404, json={"error": "unknown_token"})
body = {"access_token": _jwt(), "token_type": "Bearer", "expires_in": 900,
"user_id": "nas_user:1", "org_id": "nas_org:1"}
if self.inference_base_url:
body["inference_base_url"] = self.inference_base_url
return httpx.Response(200, json=body)
return httpx.Response(500, json={"error": f"unexpected {path}"})
from tests.hermes_cli.anon_portal import PORTAL, WELCOME, install_portal, make_jwt as _jwt # noqa: F401
@pytest.fixture
def portal(monkeypatch, tmp_path):
fake = FakePortal()
monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL)
monkeypatch.setenv("HERMES_ANON_API_SECRET", "test-secret")
monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store"))
monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1")
for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"):
monkeypatch.delenv(var, raising=False)
from hermes_cli import auth_nous
def _client(timeout_seconds, verify):
return httpx.Client(transport=httpx.MockTransport(fake.handler), base_url=PORTAL)
monkeypatch.setattr(auth_nous, "_nous_http_client", _client)
# resolve_nous_access_token builds its own client; route it through the fake too.
real_client = httpx.Client
class _RoutedClient(real_client):
def __init__(self, *a, **kw):
kw.pop("verify", None)
kw["transport"] = httpx.MockTransport(fake.handler)
super().__init__(*a, **kw)
monkeypatch.setattr(httpx, "Client", _RoutedClient)
anon_auth.reset_mint_memo_for_tests()
from hermes_cli import free_tier_bootstrap as _fb
_fb.reset_for_tests()
# resolve_nous_access_token memoises the last token for 5 s per profile home (dict); a token minted
# by an earlier test must not be served to this one.
from hermes_cli import auth as auth_mod
monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {})
return fake
return install_portal(monkeypatch, tmp_path)
def _write_config(monkeypatch, **nous):

View File

@@ -7,9 +7,7 @@ in ``hermes_cli.anon_auth``), never through mocked-away client code.
from __future__ import annotations
import base64
import json
import time
import httpx
import pytest
@@ -17,89 +15,21 @@ import pytest
from hermes_cli import anon_auth, anon_sign_in, free_tier_bootstrap
from hermes_cli.auth import _load_auth_store
PORTAL = "https://portal.example.test"
WELCOME = "https://welcome-api.nousresearch.com/v1"
def _jwt(**claims) -> str:
def seg(obj):
return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode()
payload = {"sub": "nas_user:1", "client_id": "nas-anonymous", "account_tier": "anonymous",
"scope": "inference:invoke tool:invoke", "exp": int(time.time()) + 900, **claims}
return f"{seg({'alg': 'RS256'})}.{seg(payload)}.sig"
class FakeNas:
"""The anonymous surface as NAS ships it. ``create_response`` / ``token_response`` override the
happy path with one canned refusal; ``raise_transport`` simulates the wire failing."""
def __init__(self):
self.calls: list[tuple[str, str]] = []
self.minted = 0
self.create_response: httpx.Response | None = None
self.token_response: httpx.Response | None = None
self.raise_transport: Exception | None = None
def handler(self, request: httpx.Request) -> httpx.Response:
path = request.url.path
self.calls.append((request.method, path))
if self.raise_transport is not None:
raise self.raise_transport
if path == "/api/anonymous/create":
if self.create_response is not None:
return self.create_response
self.minted += 1
return httpx.Response(201, json={"user_id": f"nas_user:{self.minted}", "org_id": "nas_org:1",
"token": f"anon_{self.minted:04d}", "idle_ttl_days": 14})
if path == "/api/anonymous/token":
if self.token_response is not None:
return self.token_response
return httpx.Response(200, json={"access_token": _jwt(), "token_type": "Bearer", "expires_in": 900,
"user_id": "nas_user:1", "org_id": "nas_org:1",
"inference_base_url": WELCOME})
return httpx.Response(500, json={"error": f"unexpected {path}"})
def creates(self) -> int:
return [p for _, p in self.calls].count("/api/anonymous/create")
from tests.hermes_cli.anon_portal import PORTAL, WELCOME, install_portal # noqa: F401
@pytest.fixture
def nas(monkeypatch, tmp_path):
fake = FakeNas()
monkeypatch.setenv("HERMES_PORTAL_BASE_URL", PORTAL)
monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared-store"))
monkeypatch.setenv("HERMES_GUEST_ONBOARDING", "1")
for var in ("OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "NOUS_API_KEY"):
monkeypatch.delenv(var, raising=False)
from hermes_cli import auth_nous
def _client(timeout_seconds, verify):
return httpx.Client(transport=httpx.MockTransport(fake.handler), base_url=PORTAL)
monkeypatch.setattr(auth_nous, "_nous_http_client", _client)
# The runtime resolver builds its own client; route it through the fake too.
real_client = httpx.Client
class _RoutedClient(real_client):
def __init__(self, *a, **kw):
kw.pop("verify", None)
kw["transport"] = httpx.MockTransport(fake.handler)
super().__init__(*a, **kw)
monkeypatch.setattr(httpx, "Client", _RoutedClient)
monkeypatch.setattr("agent.bedrock_adapter.has_aws_credentials", lambda: False)
anon_auth.reset_mint_memo_for_tests()
free_tier_bootstrap.reset_for_tests()
from hermes_cli import auth as auth_mod
monkeypatch.setattr(auth_mod, "_RESOLVE_TOKEN_CACHE", {})
return fake
return install_portal(monkeypatch, tmp_path)
def _mint_error(nas: FakeNas) -> anon_auth.AuthError:
def _mint_error(nas) -> anon_auth.AuthError:
with pytest.raises(anon_auth.AuthError) as exc:
anon_auth.ensure_portal_identity(explicit=True)
return exc.value
def _exchange_error(nas: FakeNas) -> anon_auth.AuthError:
def _exchange_error(nas) -> anon_auth.AuthError:
"""Mint (the credential is persisted before any exchange), then exchange it at first use."""
from hermes_cli.auth_nous import resolve_nous_runtime_credentials
assert anon_auth.is_guest_state(anon_auth.ensure_portal_identity(explicit=True))
@@ -159,6 +89,21 @@ class TestNasRefusalCodes:
assert "nous" not in _load_auth_store().get("providers", {})
assert nas.creates() == 1
def test_a_locked_account_is_never_replaced_through_connectors_either(self, nas):
from hermes_cli.auth import _auth_store_lock, _save_auth_store
from tests.hermes_cli.anon_portal import make_jwt
from tools import managed_tool_gateway as mtg
anon_auth.ensure_portal_identity(explicit=True)
with _auth_store_lock():
store = _load_auth_store()
store["providers"]["nous"]["expires_at"] = "2000-01-01T00:00:00+00:00"
store["providers"]["nous"]["access_token"] = make_jwt(exp=1)
_save_auth_store(store)
nas.token_response = httpx.Response(403, json={"error": "account_locked"})
assert mtg.read_nous_access_token() is None
assert "nous" not in _load_auth_store().get("providers", {})
assert nas.creates() == 1
def test_unknown_token_is_replaced_once_at_first_use(self, nas):
from hermes_cli.auth_nous import resolve_nous_runtime_credentials
first = anon_auth.ensure_portal_identity(explicit=True)
@@ -293,6 +238,16 @@ class TestBootstrapRecord:
assert nas.creates() == 1 + free_tier_bootstrap.BOOTSTRAP_RETRY_ATTEMPTS
assert free_tier_bootstrap.current_record().error_code == anon_auth.ANON_UNREACHABLE
def test_a_retry_re_inventories_so_a_provider_connected_meanwhile_keeps_inference(self, nas, monkeypatch):
nas.raise_transport = httpx.ConnectTimeout("no route")
free_tier_bootstrap.run_bootstrap(announce=False)
# The user connected their own provider during the cooldown.
monkeypatch.setattr(free_tier_bootstrap, "_inventory_other_providers", lambda: True)
nas.raise_transport = None
record = free_tier_bootstrap.retry_bootstrap_mint(force=True, announce=False)
assert record.has_identity is True and record.other_providers is True
assert _load_auth_store().get("active_provider") != "nous"
def test_the_desktop_retry_refreshes_the_boot_record(self, nas):
nas.raise_transport = httpx.ConnectTimeout("no route")
free_tier_bootstrap.run_bootstrap(announce=False)

View File

@@ -58,6 +58,12 @@ class TestControlSurface:
assert _post(base, "/__reset").json() == {**_post(base, "/__reset").json(), "nas": "ok", "inference": "ok"}
def test_reading_the_log_never_wedges_the_next_request(self, server):
_module, base = server
assert httpx.get(f"{base}/__log", timeout=5.0).status_code == 200
assert httpx.get(f"{base}/__scenario", timeout=5.0).status_code == 200
assert httpx.get(f"{base}/__log", timeout=5.0).json()["requests"][-1]["path"] == "/__scenario"
def test_the_control_surface_is_cors_open_for_a_renderer(self, server):
_module, base = server
preflight = httpx.options(f"{base}/__scenario", timeout=5.0)

View File

@@ -125,16 +125,19 @@ def read_nous_access_token() -> Optional[str]:
from hermes_cli.anon_auth import AnonCredentialDead
if isinstance(exc, AnonCredentialDead):
return _replace_dead_guest_token(nous_provider)
return _replace_dead_guest_token(nous_provider, str(exc.code or "anon_credential_dead"))
logger.debug("Nous access token refresh failed: %s", exc)
return cached_token
def _replace_dead_guest_token(dead_state: dict) -> Optional[str]:
from hermes_cli.anon_auth import clear_dead_guest, ensure_portal_identity
def _replace_dead_guest_token(dead_state: dict, code: str = "anon_credential_dead") -> Optional[str]:
from hermes_cli.anon_auth import ANON_ACCOUNT_LOCKED, clear_dead_guest, ensure_portal_identity
from hermes_cli.auth import resolve_nous_access_token
clear_dead_guest("anon_credential_dead", dead_token=dead_state.get("anon_token"))
clear_dead_guest(code, dead_token=dead_state.get("anon_token"))
# Same rule as inference: a locked account is retired but never silently replaced.
if code == ANON_ACCOUNT_LOCKED:
return None
try:
if ensure_portal_identity(explicit=True) is None:
return None