763646c30d
A relay wrapping an error as {"code": 200, "success": false} must not route to
_by_status as a 200. Also reuse _error_obj for the error.* candidate walk and
point at the string-parsing text-SSE sibling.
1492 lines
82 KiB
Python
1492 lines
82 KiB
Python
"""API error classification for smart failover and recovery.
|
||
|
||
A priority-ordered pipeline maps an API exception to a ``ClassifiedError``
|
||
whose recovery hints (retry, rotate credential, fallback, compress, abort) the
|
||
retry loop in run_agent.py consults instead of re-matching strings itself.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import enum
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Callable, Dict, Iterator, Optional, Sequence
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Synthetic code for the OpenAI SDK rejecting a provider's SSE ``data:`` field
|
||
# before any completion chunk arrives; distinct from generic JSON parse errors.
|
||
PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data"
|
||
|
||
# Same rejection with an EMPTY payload: the frame carried no ``data`` at all (``data:`` /
|
||
# ``event: ping`` / ``id:`` with no content). Per the SSE spec those are legal keepalives /
|
||
# no-ops, not malformed payloads — a degraded gateway answers every streaming request with
|
||
# them, so the session switches to non-streaming instead of re-streaming into the same
|
||
# window. See ``chat_completion_helpers._maybe_disable_streaming``.
|
||
PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE = "provider_stream_empty_frame"
|
||
|
||
|
||
# ── Error taxonomy ──────────────────────────────────────────────────────
|
||
|
||
class FailoverReason(enum.Enum):
|
||
"""Why an API call failed — determines recovery strategy."""
|
||
auth = "auth" # Transient auth (401/403) — refresh/rotate
|
||
auth_permanent = "auth_permanent" # Auth failed after refresh — abort
|
||
billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately
|
||
rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate
|
||
upstream_rate_limit = "upstream_rate_limit" # Aggregator's upstream model 429 — fallback model, key is healthy
|
||
upstream_blocked = "upstream_blocked" # 403 from a WAF/CDN/proxy in front of the provider — key is healthy, fallback
|
||
overloaded = "overloaded" # 503/529 — provider overloaded, backoff
|
||
server_error = "server_error" # 500/502 — internal server error, retry
|
||
timeout = "timeout" # Connection/read timeout — rebuild client + retry
|
||
ssl_cert_verification = "ssl_cert_verification" # Deterministic TLS chain failure — fail fast with guidance
|
||
context_overflow = "context_overflow" # Context too large — compress, not failover
|
||
payload_too_large = "payload_too_large" # 413 — compress payload
|
||
image_too_large = "image_too_large" # Native image part exceeds provider's per-image limit — shrink and retry
|
||
image_corrupt = "image_corrupt" # Provider can't decode image bytes — strip and retry (shrinking won't help)
|
||
model_not_found = "model_not_found" # 404 or invalid model — fallback to different model
|
||
provider_policy_blocked = "provider_policy_blocked" # Aggregator account data/privacy policy excluded the only endpoint
|
||
content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — deterministic per-request, don't retry unchanged
|
||
model_entitlement = "model_entitlement" # This account cannot use the requested model — rotate credential (model-scoped), else fall back
|
||
incomplete_response = "incomplete_response" # Codex/Responses turn stuck emitting reasoning only (no answer, no tool call) after replay + nudge — hand to a different provider
|
||
format_error = "format_error" # 400 bad request — abort or strip + retry
|
||
role_alternation = "role_alternation" # Strict chat template rejected adjacent same-role messages — merge them for this destination and retry
|
||
invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry
|
||
multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry
|
||
reasoning_mandatory = "reasoning_mandatory" # Route rejects reasoning: {enabled: false} — send the disable no more this session and retry
|
||
|
||
# Provider-specific
|
||
thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid
|
||
long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate
|
||
oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth rejects 1M beta — disable beta and retry
|
||
llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar rejects regex `pattern`/`format` — strip from tools and retry
|
||
unknown = "unknown" # Unclassifiable — retry with backoff
|
||
|
||
|
||
@dataclass
|
||
class ClassifiedError:
|
||
"""Structured classification of an API error with recovery hints."""
|
||
|
||
reason: FailoverReason
|
||
status_code: Optional[int] = None
|
||
provider: Optional[str] = None
|
||
model: Optional[str] = None
|
||
message: str = ""
|
||
error_context: Dict[str, Any] = field(default_factory=dict)
|
||
|
||
# Recovery hints — the retry loop checks these instead of re-classifying.
|
||
retryable: bool = True
|
||
should_compress: bool = False
|
||
should_rotate_credential: bool = False
|
||
should_fallback: bool = False
|
||
|
||
@property
|
||
def is_auth(self) -> bool:
|
||
return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
|
||
|
||
@property
|
||
def billing_unverified(self) -> bool:
|
||
"""True when a ``billing`` verdict rests on an ambiguous body (#82154)."""
|
||
return bool(self.error_context.get("billing_unverified"))
|
||
|
||
|
||
# ── Pattern tables (lowercased substrings) ──────────────────────────────
|
||
|
||
# Billing exhaustion (not transient rate limit). "out of extra usage" is the
|
||
# Anthropic OAuth Pro/Max overage bucket depleted (HTTP 400).
|
||
# The Nous gateway's own words for "the free tier will not serve this" — a billing wall for a
|
||
# named account, the tier refusing for an anonymous one (see ``_WELCOME_403_NAMED_PATTERNS``).
|
||
_FREE_TIER_REFUSAL_PATTERNS = ("model_not_supported_on_free_tier", "not available on the free tier")
|
||
_BILLING_PATTERNS = (
|
||
"insufficient credits", "insufficient_quota", "insufficient balance", "credit balance",
|
||
"credits exhausted", "credits have been exhausted", "requires available credits",
|
||
"account balance is too low", "no usable credits", "top up your credits", "payment required",
|
||
"billing hard limit", "exceeded your current quota", "account is deactivated", "plan does not include",
|
||
"out of extra usage", "out of funds", "run out of funds", "balance_depleted",
|
||
# OpenRouter org-level monthly cap arrives as 403 "Budget limit exceeded (monthly limit)" (#107166):
|
||
# account exhaustion, not a credential problem.
|
||
"budget limit exceeded",
|
||
*_FREE_TIER_REFUSAL_PATTERNS,
|
||
# LiteLLM proxies word a hard cap as "hard billing limit" (structured twin:
|
||
# ``terminal_quota_exhausted`` in _BILLING_ERROR_CODES). "terminal billing
|
||
# limit" free text is NOT matched: substring rules can't negate the
|
||
# "non-terminal billing limit" wording, and the structured code covers it.
|
||
"hard billing limit",
|
||
)
|
||
|
||
# Not proof of exhaustion: Anthropic returns the same "out of extra usage" body
|
||
# for a content-filter rejection (#82154). Verdict stays ``billing`` but is
|
||
# marked unverified so surfaces hedge and the pool uses a short cooldown.
|
||
_UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",)
|
||
|
||
# xAI's Grok credit-exhaustion code arrives as HTTP 403, not 402. Provider-
|
||
# scoped on purpose: other providers' billing codes on a 403 stay auth failures.
|
||
_XAI_SPENDING_LIMIT_ERROR_CODE = "personal-team-blocked:spending-limit"
|
||
|
||
# Structured codes meaning the account cannot serve paid traffic.
|
||
_BILLING_ERROR_CODES = frozenset({
|
||
"insufficient_quota", "billing_not_active", "payment_required", "insufficient_credits",
|
||
"no_usable_credits", "balance_depleted", "model_not_supported_on_free_tier",
|
||
"member_spend_cap_exceeded", "terminal_quota_exhausted", _XAI_SPENDING_LIMIT_ERROR_CODE,
|
||
# OpenAI (and OpenAI-compatible aggregators) spend/usage-limit family:
|
||
# a credit balance or an org/project spend or usage cap is exhausted —
|
||
# terminal for this credential until limits are raised.
|
||
"credit_balance_exhausted", "organization_spend_limit_exceeded",
|
||
"organization_usage_limit_exceeded", "project_spend_limit_exceeded",
|
||
# Nous paid model behind an empty credit balance arrives as a 404 (#115702).
|
||
"insufficient_credits_for_paid_model",
|
||
})
|
||
|
||
# Transient rate limiting. Bedrock "Throttling error: Too many tokens" also
|
||
# contains an overflow phrase; rate limit is matched first so throttle wins.
|
||
_RATE_LIMIT_PATTERNS = (
|
||
"rate limit", "rate_limit", "too many requests", "throttled", "requests per minute",
|
||
"tokens per minute", "requests per day", "try again in", "please retry after",
|
||
"resource exhausted", "resource_exhausted", "resource-exhausted", "resourceexhausted",
|
||
"rate increased too quickly", "throttlingexception", "too many concurrent requests",
|
||
"servicequotaexceededexception", "throttling",
|
||
)
|
||
|
||
# Server busy, credential fine: back off on the same key, never rotate. Z.AI/
|
||
# Zhipu reuse HTTP 429 for this, so the 429 path checks these first. Kept narrow
|
||
# so a plain "you have been rate-limited" doesn't land here. (#14038, #15297)
|
||
_OVERLOADED_PATTERNS = (
|
||
"overloaded", "temporarily overloaded", "service is temporarily overloaded",
|
||
"service may be temporarily overloaded", "server is overloaded", "server overloaded",
|
||
"server overload", "server_overload",
|
||
"service overloaded", "service is overloaded", "upstream overloaded", "currently overloaded",
|
||
# CommandCode's 429 body for an unavailable upstream model — the key is healthy (#117111).
|
||
"upstream model provider is temporarily unavailable. please try again in a moment.",
|
||
"at capacity", "over capacity",
|
||
)
|
||
|
||
# Usage-limit patterns that need disambiguation (billing OR rate_limit), and
|
||
# the signals that mark such a limit as transient (periodic quota, not billing).
|
||
_USAGE_LIMIT_PATTERNS = ("usage limit", "quota", "limit exceeded", "key limit exceeded")
|
||
_USAGE_LIMIT_TRANSIENT_SIGNALS = (
|
||
"try again", "retry", "resets at", "reset in", "resets in", "reset after", "available in",
|
||
"wait", "requests remaining", "periodic", "window", "per minute", "per second",
|
||
)
|
||
|
||
# 413 detected from message text (proxies embed the status or re-wrap
|
||
# Anthropic's "request_too_large" type without one).
|
||
_PAYLOAD_TOO_LARGE_PATTERNS = (
|
||
"request entity too large", "payload too large", "error code: 413", "request_too_large",
|
||
# Normally arrives with an HTTP 413 status (handled by the status path), but aggregators/proxies can
|
||
# re-wrap it into a plain message with no status attribute — route it to the same compression recovery.
|
||
# (port of anomalyco/opencode#37848)
|
||
"request exceeds the maximum size",
|
||
)
|
||
|
||
# Per-image size/dimension 400s (Anthropic 5 MB / 8000 px; MiniMax "media
|
||
# exceeds size limit" #76039) — a specific 400 before the request hits 413. A
|
||
# non-image media hit is harmless: the shrink pass finds no image parts.
|
||
# "patches after processing": OpenAI Codex Responses rejects an image whose
|
||
# tile-patch budget (ceil(w/32)×ceil(h/32)) exceeds its 30000-patch ceiling
|
||
# with wording that names no image-size vocabulary — without this pattern it
|
||
# fell to format_error (non-retryable), bypassing the shrink recovery (#106337).
|
||
# Byte caps enforced with a 400 instead of a 413 (#112473): NVIDIA NIM caps the whole
|
||
# payload ("Please make sure your payload is below 26214400 bytes in size"); Alibaba
|
||
# DashScope caps the base64 image string via Jackson ("String value length (N) exceeds the
|
||
# maximum allowed (M, from `StreamReadConstraints.getMaxStringLength()`)"). Only an inline
|
||
# image reaches those sizes, so shrinking is the recovery; the method-scoped Jackson token
|
||
# is used because the bare class name also appears when Jackson caps a *token* length.
|
||
_IMAGE_TOO_LARGE_PATTERNS = (
|
||
"image exceeds", "image too large", "image_too_large", "image size exceeds", "image dimensions exceed",
|
||
"dimensions exceed max allowed size", "max allowed size: 8000", "media exceeds", "media too large",
|
||
"patches after processing", "make sure your payload is below", "streamreadconstraints.getmaxstringlength",
|
||
)
|
||
|
||
# Undecodable image bytes → strip-and-retry, never shrink. xAI wordings
|
||
# (#69078); the last is the full sentence because shorter fragments also match
|
||
# non-image download failures.
|
||
_IMAGE_CORRUPT_PATTERNS = (
|
||
"invalid png image", "invalid jpeg image", "base64 string of provided image cannot be decoded",
|
||
"downloaded response does not contain a valid jpg, png, webp, or ico image",
|
||
)
|
||
|
||
# 400s rejecting list-type ``content`` in tool messages (Xiaomi MiMo "text is
|
||
# not set", Alibaba, OpenAI-compat long tail). Recovery: strip image parts from
|
||
# tool messages, remember (provider, model), retry. (#27344)
|
||
# NVIDIA NIM's Rust gateway never names the field: its serde rejection says the
|
||
# body "did not match any variant of untagged enum
|
||
# ChatCompletionRequestToolMessageContent", which is the same list-type tool
|
||
# content that every other wording here describes (#111231).
|
||
_MULTIMODAL_TOOL_CONTENT_PATTERNS = (
|
||
"text is not set", "tool message content must be a string", "tool content must be a string",
|
||
"tool message must be a string", "expected string, got list", "expected string, got array",
|
||
# Console Go / pydantic-v2 relays behind opencode-go (422, param ``messages.N.tool.content.str``, #104731).
|
||
"tool_call.content must be string", "tool.content.str", "input should be a valid string",
|
||
"chatcompletionrequesttoolmessagecontent",
|
||
)
|
||
|
||
# Local-inference memory/resource-ceiling rejections (oMLX/MLX memory guard,
|
||
# llama.cpp/vLLM OOM, Metal/CUDA allocation ceilings). The server aborts on a
|
||
# prefill memory PEAK, not a window limit, yet its remediation tail says
|
||
# "reduce context length" — so without this list the request routes into
|
||
# compression, which cannot lower a prefill peak: it burns the compression
|
||
# budget, re-hits the wedged server each attempt and ends in a session reset.
|
||
# Every token names memory/allocation in BYTES, never a token count, so the
|
||
# list is disjoint from _CONTEXT_OVERFLOW_PATTERNS. Must be checked BEFORE
|
||
# both overflow AND the usage-limit disambiguation ("memory limit exceeded"
|
||
# contains "limit exceeded", which would otherwise read as billing). oMLX
|
||
# reworded the accounting sentence in 0.5.7 ("predicted peak would require /
|
||
# exceed"); the 0.5.6 wording is still in the field, so both stay. (#52261)
|
||
_MEMORY_CEILING_PATTERNS = (
|
||
"memory guard", "memory limit exceeded", "memory_guard_tier", "dynamic ceiling",
|
||
"memory ceiling", "available memory", "out of memory", "insufficient memory",
|
||
"prefill would require", "predicted peak would", "prefill safety cap", "metal_cap",
|
||
)
|
||
|
||
# Structured codes identifying the same rejection at the source, before an
|
||
# OpenAI-compatible proxy flattens the body and drops the wording.
|
||
_MEMORY_CEILING_ERROR_CODES = frozenset({
|
||
"prefill_memory_exceeded", "prefill_memory_aborted", "omlx_prefill_memory_exceeded",
|
||
})
|
||
|
||
# Bare "max_tokens" is load-bearing: the output-cap-retry path keys off it;
|
||
# empty-response advisories mentioning it are intercepted earlier. Groups:
|
||
# generic; vLLM; Ollama; llama.cpp; Chinese; Z.AI (1210); Bedrock; Together.
|
||
_CONTEXT_OVERFLOW_PATTERNS = (
|
||
"context length", "context size", "maximum context", "token limit", "too many tokens",
|
||
"reduce the length", "exceeds the limit", "context window", "prompt is too long",
|
||
"prompt exceeds max length", "max_tokens", "maximum number of tokens",
|
||
"exceeds the max_model_len", "max_model_len", "prompt length", "input is too long", "maximum model length",
|
||
"context length exceeded", "truncating input",
|
||
"slot context", "n_ctx_slot",
|
||
"超过最大长度", "上下文长度",
|
||
"tokens in request more than max tokens allowed",
|
||
"input is too long", "max input token", "input token", "exceeds the maximum number of input tokens",
|
||
# Together/Fireworks-style: "Input length 131393 exceeds the maximum allowed input length of 131040
|
||
# tokens." No other pattern in this list matches that wording. (port of anomalyco/opencode#37848)
|
||
"maximum allowed input length",
|
||
)
|
||
|
||
# Last entry: OpenRouter 404 when no endpoint supports tool calling —
|
||
# model_not_found triggers fallback instead of burning retries (#58446).
|
||
# Codex ChatGPT-account entitlement 400 — the account can never use the named slug (#71970, #106475).
|
||
CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER = "model is not supported when using codex with a chatgpt account"
|
||
|
||
_MODEL_NOT_FOUND_PATTERNS = (
|
||
"is not a valid model", "invalid model", "model not found", "model_not_found", "does not exist",
|
||
"no such model", "unknown model", "unsupported model", "no endpoints found that support tool use",
|
||
)
|
||
|
||
# Qwen/vLLM chat-template "No user query found". Shared by the invalid-body
|
||
# table (→ format_error) and the llama.cpp grammar guard so they cannot drift.
|
||
_NO_USER_QUERY_SIGNAL = "no user query found"
|
||
|
||
# Deterministic rejections of the *transcript* (e.g. a content-less assistant
|
||
# stub after a dead stream). NOT overflow — input may be tiny and compression
|
||
# cannot invent a missing turn — so fail fast as format_error.
|
||
_INVALID_MESSAGE_BODY_PATTERNS = (
|
||
"must have non-empty content", "messages must have non-empty", "invalid_request_body",
|
||
"text content blocks must be non-empty", "content field is required",
|
||
"messages: at least one message is required", _NO_USER_QUERY_SIGNAL,
|
||
)
|
||
|
||
# Strict-alternation chat templates (llama.cpp / vLLM Jinja templates, Mistral, some
|
||
# OpenRouter routes) 400 when two adjacent messages share a role. Deterministic for the
|
||
# request shape, and the only bad thing is the adjacency, so the caller that produced it
|
||
# (the MoA aggregator appends ``user(guidance)`` after ``user(task)`` on iteration 1 —
|
||
# #112358) merges the pair for THAT destination and retries once. Checked before the
|
||
# request-validation table: the body usually also carries ``invalid_request_error``.
|
||
_ROLE_ALTERNATION_PATTERNS = (
|
||
"roles must alternate", "role must alternate", "must alternate between",
|
||
"consecutive user messages", "consecutive messages with the same role",
|
||
"consecutive messages of the same role", "same role in a row", "multiple user messages in a row",
|
||
"adjacent messages with the same role",
|
||
)
|
||
|
||
# Proxy-side rejection of the model's own tool-call JSON (Ollama "invalid tool call arguments",
|
||
# OpenRouter-wrapped "function_call arguments"). Checked before the generic 400 validation and
|
||
# overflow heuristics: on a large session the bare message would otherwise read as overflow.
|
||
_MALFORMED_TOOL_ARGS_PATTERNS = (
|
||
"invalid tool call arguments", "invalid tool_call arguments", "invalid tool_calls arguments",
|
||
"invalid function call arguments", "invalid function_call arguments",
|
||
"tool call arguments are invalid", "tool_call arguments are invalid",
|
||
"function call arguments are invalid", "function_call arguments are invalid",
|
||
)
|
||
|
||
# Malformed request, identical on every retry. Some gateways (codex.nekos.me)
|
||
# return these as 5xx, so the 5xx path also checks them.
|
||
_REQUEST_VALIDATION_PATTERNS = (
|
||
"unknown parameter", "unsupported parameter", "unrecognized request argument",
|
||
"invalid_request_error", "unknown_parameter", "unsupported_parameter",
|
||
)
|
||
|
||
# Parameters Hermes sends on SOME routes only → hosts where that is deliberate.
|
||
# A rejection from any other host means the provider's gateway injected the
|
||
# field itself: a server-side flake, not our request shape. prompt_cache_retention
|
||
# is only sent for api.meta.ai / bedrock-mantle (agent/transports/codex.py).
|
||
_SERVER_INJECTED_PARAM_SENDERS: Dict[str, tuple] = {
|
||
"prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
|
||
}
|
||
_PARAM_REJECTION_WORDS = ("not supported", "unsupported", "unknown", "unrecognized")
|
||
|
||
# Anthropic thinking-block 400 wordings (see _provider_special_cases).
|
||
_THINKING_MUTATION_WORDS = ("signature", "cannot be modified", "must remain as they were")
|
||
|
||
# Local MoA streaming adapter-shape bugs (see _moa_special_cases).
|
||
_MOA_ADAPTER_SHAPE_BUGS = (
|
||
"'types.SimpleNamespace' object is not iterable", "'types.SimpleNamespace' object has no attribute 'index'",
|
||
)
|
||
|
||
# OpenRouter 404 when the account data policy excludes the only endpoint. Not
|
||
# model_not_found: the model exists, fallback can't help, body has the fix URL.
|
||
_PROVIDER_POLICY_BLOCKED_PATTERNS = (
|
||
"no endpoints available matching your guardrail", "no endpoints available matching your data policy",
|
||
"no endpoints found matching your data policy",
|
||
)
|
||
|
||
# Upstream account ban relayed by an aggregator, often as HTTP 200 + an SSE error
|
||
# event (no status): permanent for this account, so never the transient retry ladder.
|
||
_ACCOUNT_POLICY_BLOCK_PATTERNS = ("blocked for a previous policy violation",)
|
||
|
||
# Per-prompt safety-filter blocks: deterministic for the unchanged request, so
|
||
# fallback immediately. Each phrase is verbatim from one provider (Codex cyber
|
||
# flags #18028, OpenAI moderation, Anthropic safety, Azure token, MiniMax
|
||
# #32421, CommandCode gateway moderation #115218) — never a generic word like
|
||
# "policy" that collides with billing/auth.
|
||
# "content_filter" deliberately excludes the space variant seen in echoed config.
|
||
_CONTENT_POLICY_BLOCKED_PATTERNS = (
|
||
"flagged for possible cybersecurity risk", "trusted access for cyber",
|
||
"violates our usage policies", "violates openai's usage policies", "your request was flagged by",
|
||
"prompt was flagged by our safety", "responses cannot be generated due to safety",
|
||
"content_filter", "responsibleaipolicyviolation", "new_sensitive",
|
||
"content exists risk",
|
||
)
|
||
|
||
# Auth patterns (non-status-code signals).
|
||
_AUTH_PATTERNS = (
|
||
"invalid api key", "invalid_api_key", "gateway_auth_failed", "authentication", "unauthorized",
|
||
"forbidden", "invalid token", "token expired", "token revoked", "access denied",
|
||
# Codex backend rejecting an OAuth access token without a usable
|
||
# ``chatgpt_account_id`` claim; arrives as a bare ``detail`` string.
|
||
"failed to extract accountid from token",
|
||
)
|
||
|
||
# Empty-response advisories (OpenRouter / nano-gpt). Checked before overflow
|
||
# because the text often mentions "max_tokens" (caused compression spirals).
|
||
_EMPTY_PROVIDER_RESPONSE_PATTERNS = (
|
||
"returned an empty response", "empty response despite retries", "provider returned an empty response",
|
||
"model returning empty responses", "empty response stream",
|
||
)
|
||
|
||
# Timeout wording from generic exception types the type heuristics would miss.
|
||
_TIMEOUT_MESSAGE_PATTERNS = (
|
||
"timed out", "turn timed out", "request timed out", "deadline exceeded", "operation timed out",
|
||
"upstream timed out",
|
||
)
|
||
|
||
# Connect/DNS failures from generic exception types with no status. EXCLUDES
|
||
# mid-stream disconnects (_SERVER_DISCONNECT_PATTERNS may route large sessions
|
||
# to compression; a never-established connection cannot be an overflow).
|
||
# Groups: TCP connect; DNS (Python/glibc/macOS/Node); undici bridge; Envoy.
|
||
_CONNECTION_MESSAGE_PATTERNS = (
|
||
"connection refused", "econnrefused", "no route to host", "network is unreachable", "network unreachable",
|
||
"name or service not known", "temporary failure in name resolution", "nodename nor servname provided",
|
||
"getaddrinfo failed", "getaddrinfo enotfound", "eai_again",
|
||
"fetch failed", "failed to fetch",
|
||
"upstream connect error",
|
||
)
|
||
|
||
# SSL names keep provider-wrapped SSL errors (chain lost) as transport, not
|
||
# unknown; OpenAI SDK errors are not subclasses of Python builtins.
|
||
_TRANSPORT_ERROR_TYPES = frozenset({
|
||
"ReadTimeout", "ConnectTimeout", "PoolTimeout", "ConnectError", "RemoteProtocolError",
|
||
"ConnectionError", "ConnectionResetError", "ConnectionAbortedError", "BrokenPipeError",
|
||
"TimeoutError", "ReadError", "ServerDisconnectedError",
|
||
"SSLError", "SSLZeroReturnError", "SSLWantReadError", "SSLWantWriteError", "SSLEOFError", "SSLSyscallError",
|
||
"APIConnectionError", "APITimeoutError",
|
||
})
|
||
|
||
# Ambiguous disconnects (no status): transient hiccup OR a gateway dropping an
|
||
# oversized request. A large session + one of these → context-overflow path.
|
||
_SERVER_DISCONNECT_PATTERNS = (
|
||
"server disconnected", "peer closed connection", "connection reset by peer", "connection was closed",
|
||
"network connection lost", "unexpected eof", "incomplete chunked read",
|
||
)
|
||
|
||
# Deterministic cert failures (proxy, missing CA, expired/self-signed) — fail
|
||
# fast. Checked BEFORE _SSL_TRANSIENT_PATTERNS: these also contain "[SSL:".
|
||
_SSL_CERT_VERIFY_PATTERNS = (
|
||
"certificate verify failed", "certificate_verify_failed", "unable to get local issuer certificate",
|
||
"self-signed certificate", "self signed certificate", "certificate has expired",
|
||
"hostname mismatch, certificate is not valid", "unable to verify the first certificate",
|
||
)
|
||
|
||
# Transient SSL alerts: retry but NOT compression (kept apart from disconnects).
|
||
# Both space and underscore forms because OpenSSL 3 changed token separators
|
||
# (SSLV3_ALERT_... → SSL/TLS_ALERT_...); "[ssl:" is the Python ssl prefix.
|
||
_SSL_TRANSIENT_PATTERNS = (
|
||
"bad record mac", "ssl alert", "tls alert", "ssl handshake failure", "tlsv1 alert", "sslv3 alert",
|
||
"bad_record_mac", "ssl_alert", "tls_alert", "tls_alert_internal_error", "[ssl:",
|
||
)
|
||
|
||
|
||
# A 403 body written by a WAF/CDN/proxy rather than the provider's API: Cloudflare's browser
|
||
# challenge and block pages, plus the plain-text block relays return when they reject the SDK
|
||
# User-Agent (#53099). Matched only on 403 (see ``_status_403``); a bare "access denied" or
|
||
# "forbidden" stays auth because providers word real permission errors that way too.
|
||
_UPSTREAM_BLOCKED_PATTERNS = (
|
||
"your request was blocked", "request blocked", "sorry, you have been blocked",
|
||
"enable javascript and cookies to continue", "cdn-cgi/challenge-platform", "cf-browser-verification",
|
||
"challenge-error-text", "__cf_chl", "cf-error-details", "attention required! | cloudflare",
|
||
)
|
||
|
||
|
||
# ── Verdicts and rule tables ────────────────────────────────────────────
|
||
# A verdict is the ClassifiedError kwargs a stage decided on: ``reason`` plus
|
||
# hint overrides (unlisted hints keep dataclass defaults). Rule tables are
|
||
# ordered ``(patterns, verdict)`` pairs matched first-hit; ``verdict`` may be
|
||
# a callable of the error message.
|
||
|
||
Verdict = Dict[str, Any]
|
||
|
||
|
||
def _v(reason: FailoverReason, **hints: Any) -> Verdict:
|
||
return {"reason": reason, **hints}
|
||
|
||
|
||
_ROTATE_FALLBACK = {"should_rotate_credential": True, "should_fallback": True}
|
||
_ABORT_FALLBACK = {"retryable": False, "should_fallback": True}
|
||
_R = FailoverReason
|
||
|
||
_V_BILLING = _v(_R.billing, retryable=False, **_ROTATE_FALLBACK)
|
||
_V_RATE_LIMIT = _v(_R.rate_limit, **_ROTATE_FALLBACK)
|
||
_V_AUTH_ROTATE = _v(_R.auth, retryable=False, **_ROTATE_FALLBACK)
|
||
_V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK)
|
||
_V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK)
|
||
_V_UPSTREAM_BLOCKED = _v(_R.upstream_blocked, **_ABORT_FALLBACK)
|
||
_V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK)
|
||
# Another account in the same pool may hold the entitlement; the credential itself is healthy.
|
||
_V_MODEL_ENTITLEMENT = _v(_R.model_entitlement, retryable=False, **_ROTATE_FALLBACK)
|
||
_V_FORMAT_ERROR = _v(_R.format_error, **_ABORT_FALLBACK)
|
||
# A different provider (direct instead of the aggregator; another host's TLS chain) can fix these.
|
||
_V_POLICY_BLOCKED = _v(_R.provider_policy_blocked, **_ABORT_FALLBACK)
|
||
_V_SSL_CERT = _v(_R.ssl_cert_verification, **_ABORT_FALLBACK)
|
||
_V_CONTEXT_OVERFLOW = _v(_R.context_overflow, should_compress=True)
|
||
_V_PAYLOAD_TOO_LARGE = _v(_R.payload_too_large, should_compress=True)
|
||
_V_OVERLOADED, _V_SERVER_ERROR, _V_TIMEOUT, _V_UNKNOWN = map(_v, (_R.overloaded, _R.server_error, _R.timeout, _R.unknown))
|
||
_V_IMAGE_TOO_LARGE, _V_IMAGE_CORRUPT = _v(_R.image_too_large), _v(_R.image_corrupt)
|
||
_V_MULTIMODAL, _V_INVALID_ENCRYPTED = _v(_R.multimodal_tool_content_unsupported), _v(_R.invalid_encrypted_content)
|
||
_V_REASONING_MANDATORY = _v(_R.reasoning_mandatory, should_compress=False, should_fallback=False)
|
||
# Same recovery hints as format_error: consumers without a merge-and-retry step (the main loop
|
||
# already merges adjacent users before the call) keep aborting to the fallback chain.
|
||
_V_ROLE_ALTERNATION = _v(_R.role_alternation, **_ABORT_FALLBACK)
|
||
# The MODEL emitted unparseable tool-call JSON and the proxy (Ollama, OpenRouter) rejected it: no
|
||
# other provider can fix that output, so falling back only replays the same broken turn 4-5 times
|
||
# (20-60s per occurrence, #12770). Abort this call; the loop's argument repair handles the retry.
|
||
_V_MALFORMED_TOOL_ARGS = _v(_R.format_error, retryable=False, should_fallback=False)
|
||
# A reasoning-mandatory route answering ``reasoning: {enabled: false}`` (Nous Portal + OpenRouter wording).
|
||
_REASONING_MANDATORY_PATTERN = "reasoning is mandatory"
|
||
|
||
# Generic markers a provider 400 puts next to the offending parameter name. Bedrock Converse
|
||
# rejects sampling params for reasoning-first models with the contraction ("This model doesn't
|
||
# support the temperature field", xAI Grok) and inference-profile Claude with "`temperature` is
|
||
# deprecated for this model" (#111043); strict pydantic gateways (Fireworks) name the unknown
|
||
# field as "extra inputs are not permitted" (#109774). Enum-rejecting aggregators (commandcode.ai)
|
||
# say "Invalid option: expected one of ..." with no "unsupported" anywhere, naming the field only
|
||
# in the structured 'param' tail (#115277). Shared with the auxiliary retry ladder
|
||
# (``agent.auxiliary_client._is_unsupported_parameter_error``).
|
||
UNSUPPORTED_PARAM_MARKERS = (
|
||
"unsupported parameter", "unsupported_parameter", "not supported", "does not support",
|
||
"doesn't support", "is deprecated for this model",
|
||
"unknown parameter", "unrecognized request argument", "unrecognized parameter",
|
||
"invalid parameter", "extra inputs are not permitted",
|
||
"invalid option: expected one of",
|
||
)
|
||
|
||
# Reasoning wire-field names (the profile reasoning controls minus ``verbosity``), longest first.
|
||
# Standalone only: never a model-id segment ("The model kimi-k2-thinking is not supported when
|
||
# using this account" is route gating for the provider-fallback rung) nor the adjective in
|
||
# "... not supported with reasoning models".
|
||
_REASONING_FIELD_TOKEN = re.compile(
|
||
r"(?<![\w\-/])(?:reasoning_effort|thinking_config|thinking_budget|enable_thinking|thinkingconfig"
|
||
r"|thinkingbudget|reasoning|thinking|think)(?![\w\-/])(?!\s+models?\b)"
|
||
)
|
||
|
||
# Structured rejection of a reasoning field, read from the stringified body: OpenAI-style
|
||
# ``param`` naming a reasoning field (``reasoning_effort`` on chat, ``reasoning.effort`` on
|
||
# Responses) or an ``invalid_reasoning_effort`` code. Custom Responses relays send this with NO
|
||
# message at all (#100536), so no wording rule can match it — and without a match the message-less
|
||
# 400 fell through to the generic large-session overflow heuristic and started compression.
|
||
_REASONING_PARAM_REJECTION = re.compile(
|
||
r"""['"]param['"]\s*:\s*['"](?:reasoning(?:[._]effort)?|thinking(?:_config|_budget)?|enable_thinking)['"]"""
|
||
r"""|invalid_reasoning_effort"""
|
||
)
|
||
|
||
|
||
_REASONING_REQUIRED_MARKERS = (
|
||
"mandatory", "cannot be disabled", "can't be disabled", "must be enabled", "is required",
|
||
"always enabled", "cannot be turned off",
|
||
)
|
||
|
||
|
||
def is_reasoning_required_rejection(error_msg: str) -> bool:
|
||
"""Provider 400 saying the model's reasoning cannot be switched OFF ("Reasoning is mandatory for
|
||
this endpoint and cannot be disabled", the Nous Portal on gpt-6-astra). The opposite of
|
||
``is_reasoning_field_rejection``: the field is understood, the *disable* is refused, so the right
|
||
reaction is to step the effort up to the lowest level rather than drop the field (a dropped field
|
||
also works, but tells the caller nothing about the next call)."""
|
||
msg = (error_msg or "").lower()
|
||
token = _REASONING_FIELD_TOKEN.search(msg)
|
||
if token is None:
|
||
return False
|
||
near = msg[max(0, token.start() - 48):token.end() + 96]
|
||
return any(m in near for m in _REASONING_REQUIRED_MARKERS)
|
||
|
||
|
||
def is_reasoning_field_rejection(error_msg: str) -> bool:
|
||
"""Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
|
||
``thinking``/``think``): the field token plus either a generic unsupported marker ("Unrecognized
|
||
request argument supplied: reasoning_effort", #112781) or a standalone "unsupported" next to the
|
||
field in either word order ("unsupported reasoning_effort"; "reasoning_effort 'none' unsupported;
|
||
use minimal|low|medium|high|xhigh", #114460). The route default is the right answer for such a
|
||
model, so both the main loop and the auxiliary ladder retry once without the disable. A body
|
||
whose structured ``param``/code names the reasoning field (``'param': 'reasoning.effort'``,
|
||
``invalid_reasoning_effort``, #100536) is a rejection whatever the message says — even none.
|
||
|
||
Known trade-off: a 400 about a thinking *state* ("Function calling is not supported when
|
||
thinking is enabled") also matches — the marker sits right next to the token, so no proximity
|
||
rule separates it from the forward wordings. Cost is one dropped-disable retry before the
|
||
spent path takes the fallback chain; the auxiliary ladder already treated it this way."""
|
||
msg = (error_msg or "").lower()
|
||
if _REASONING_PARAM_REJECTION.search(msg):
|
||
return True
|
||
token = _REASONING_FIELD_TOKEN.search(msg)
|
||
if token is None:
|
||
return False
|
||
near = msg[max(0, token.start() - 32):token.end() + 32]
|
||
return "unsupported" in near or any(m in msg for m in UNSUPPORTED_PARAM_MARKERS)
|
||
|
||
|
||
def _billing_hints(error_msg: str) -> Verdict:
|
||
"""Billing verdict carrying the #82154 ambiguity marker when applicable."""
|
||
ctx: Dict[str, Any] = {}
|
||
if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS):
|
||
ctx = {"billing_unverified": True, "possible_content_filter": True}
|
||
return {**_V_BILLING, "error_context": ctx}
|
||
|
||
|
||
def _first_match(error_msg: str, rules: Sequence[tuple[Sequence[str], Any]]) -> Optional[Verdict]:
|
||
"""Verdict of the first rule whose pattern list hits ``error_msg``."""
|
||
for patterns, verdict in rules:
|
||
if any(p in error_msg for p in patterns):
|
||
return verdict(error_msg) if callable(verdict) else verdict
|
||
return None
|
||
|
||
|
||
# Image/tool-content 400s, ordered: multimodal recovery ≠ image shrink; corrupt
|
||
# bytes need strip not shrink; image-shrink is cheaper than context compression.
|
||
_IMAGE_TOOL_RULES = (
|
||
(_MULTIMODAL_TOOL_CONTENT_PATTERNS, _V_MULTIMODAL), (_IMAGE_CORRUPT_PATTERNS, _V_IMAGE_CORRUPT),
|
||
(_IMAGE_TOO_LARGE_PATTERNS, _V_IMAGE_TOO_LARGE),
|
||
)
|
||
|
||
# Overflow signals arriving as 5xx (llama.cpp reports overflow as 500; busy /
|
||
# model-load OOM as 503). Empty-response advisories must not enter compression.
|
||
_OVERFLOW_AS_5XX_RULES = (
|
||
(_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR), (_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
|
||
(_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW),
|
||
)
|
||
|
||
# 404: Nous API surfaces credit depletion as a paid model vanishing from the
|
||
# Free Tier (billing, not missing model); policy block before model_not_found.
|
||
_404_RULES = (
|
||
(_BILLING_PATTERNS, _V_BILLING), (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED),
|
||
(_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
)
|
||
|
||
# 400 tail after the deterministic request-shape checks. Some providers return
|
||
# model-not-found / rate-limit / billing as 400 instead of 404/429/402.
|
||
_400_TAIL_RULES = _OVERFLOW_AS_5XX_RULES + (
|
||
(_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
(_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_BILLING_PATTERNS, _billing_hints),
|
||
)
|
||
|
||
# LM Studio / llama.cpp raise a bare status-less ``APIError`` when chat-template Jinja rendering
|
||
# fails mid-stream (#62662). Deterministic for the request, so fall back instead of retrying.
|
||
_STREAM_RENDER_ERROR_PATTERNS = ("error rendering", "rendering prompt", "jinja template", "jinja render")
|
||
|
||
# Status-less message path, head (before usage-limit disambiguation).
|
||
_MESSAGE_HEAD_RULES = ((_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
|
||
(_PAYLOAD_TOO_LARGE_PATTERNS, _V_PAYLOAD_TOO_LARGE),
|
||
(_ROLE_ALTERNATION_PATTERNS, _V_ROLE_ALTERNATION),
|
||
(_STREAM_RENDER_ERROR_PATTERNS, _V_FORMAT_ERROR)) + _IMAGE_TOOL_RULES
|
||
|
||
# Status-less tail. Overload before rate_limit/billing so "overloaded" backs off
|
||
# instead of rotating; policy block before model_not_found; timeout/connection
|
||
# wording last, classified as transport (never compression).
|
||
_MESSAGE_TAIL_RULES = (
|
||
(_OVERLOADED_PATTERNS, _V_OVERLOADED), (_BILLING_PATTERNS, _billing_hints),
|
||
(_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR),
|
||
(_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW), (_AUTH_PATTERNS, _V_AUTH_ROTATE),
|
||
(_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
(_TIMEOUT_MESSAGE_PATTERNS, _V_TIMEOUT), (_CONNECTION_MESSAGE_PATTERNS, _V_TIMEOUT),
|
||
)
|
||
|
||
# Structured error code → verdict. The error-code rate_limit verdict rotates
|
||
# but does not set should_fallback (unlike the message/status paths).
|
||
_ERROR_CODE_VERDICTS: Dict[str, Verdict] = {
|
||
**dict.fromkeys(("resource_exhausted", "throttled", "rate_limit_exceeded"),
|
||
_v(_R.rate_limit, should_rotate_credential=True)),
|
||
**dict.fromkeys(_BILLING_ERROR_CODES, _V_BILLING),
|
||
**dict.fromkeys(("model_not_found", "model_not_available", "invalid_model"), _V_MODEL_NOT_FOUND),
|
||
**dict.fromkeys(("context_length_exceeded", "max_tokens_exceeded"), _V_CONTEXT_OVERFLOW),
|
||
**dict.fromkeys(_MEMORY_CEILING_ERROR_CODES, _V_OVERLOADED),
|
||
"invalid_encrypted_content": _V_INVALID_ENCRYPTED,
|
||
}
|
||
|
||
# Provider-native status codes that arrive as a bare ``{"error": {"code": …}}`` body
|
||
# (no HTTP status, no prose): gRPC canonical names from Gemini, Anthropic error
|
||
# types, OpenAI's ``server_error``. Scoped per provider so a coincidentally named
|
||
# code from another backend stays ``unknown`` (#70414). Provider aliases collapse
|
||
# to the family key before lookup.
|
||
_PROVIDER_CODE_FAMILIES = {"openai-codex": "openai", "google": "gemini", "google-gemini": "gemini",
|
||
"google-ai-studio": "gemini", "vertex": "gemini", "google-vertex": "gemini"}
|
||
_PROVIDER_CODE_VERDICTS: Dict[str, Dict[str, Verdict]] = {
|
||
"openai": {"server_error": _V_SERVER_ERROR},
|
||
"gemini": {"unavailable": _V_OVERLOADED, "deadline_exceeded": _V_TIMEOUT, "internal": _V_SERVER_ERROR},
|
||
"anthropic": {"api_error": _V_SERVER_ERROR, "rate_limit_error": _V_RATE_LIMIT},
|
||
}
|
||
|
||
# Generic ``invalid_request_error`` is deliberately NOT a 400 validation
|
||
# signal — OpenAI stamps it on genuine overflow 400s too.
|
||
_400_VALIDATION_CODES = {"unknown_parameter", "unsupported_parameter"}
|
||
_5XX_VALIDATION_CODES = _400_VALIDATION_CODES | {"invalid_request_error"}
|
||
_400_VALIDATION_PATTERNS = tuple(p for p in _REQUEST_VALIDATION_PATTERNS if p != "invalid_request_error")
|
||
|
||
|
||
# ── Classification pipeline ─────────────────────────────────────────────
|
||
|
||
@dataclass
|
||
class _Ctx:
|
||
"""Everything the classifier stages need about one failed call."""
|
||
|
||
error: Exception
|
||
status_code: Optional[int]
|
||
body: dict
|
||
msg: str # lowercased str(error) + body message(s)
|
||
provider: str # as passed by the caller
|
||
model: str
|
||
approx_tokens: int
|
||
context_length: int
|
||
num_messages: int
|
||
base_url: str = "" # the route the call went to; "" when the caller did not say
|
||
anonymous: bool = False
|
||
|
||
def __post_init__(self) -> None:
|
||
self.error_type = type(self.error).__name__
|
||
self.error_code = _extract_error_code(self.body)
|
||
self.code = self.error_code.lower()
|
||
self.headers = _from_cause_chain(self.error, _headers_of, {})
|
||
self.provider_slug = (self.provider or "").strip().lower()
|
||
self.model_slug = (self.model or "").strip().lower()
|
||
|
||
def large_session(self, frac: float, tokens: int, messages: int) -> bool:
|
||
"""Absolute thresholds only proxy for smaller context windows."""
|
||
return self.approx_tokens > self.context_length * frac or (
|
||
self.context_length <= 256000 and (self.approx_tokens > tokens or self.num_messages > messages)
|
||
)
|
||
|
||
|
||
def _plugin_verdict(c: _Ctx) -> Optional[Verdict]:
|
||
"""First valid plugin classification (runs before the built-in pipeline so a
|
||
provider plugin can add or correct verdicts). invoke_hook isolates callback
|
||
failures; this guard only covers import/dispatch failure."""
|
||
try:
|
||
from hermes_cli.plugins import get_plugin_error_classification
|
||
verdict = get_plugin_error_classification(
|
||
provider=c.provider, model=c.model, status_code=c.status_code, error_type=c.error_type,
|
||
error_code=c.error_code, error_message=c.msg, error_body=c.body, error=c.error,
|
||
approx_tokens=c.approx_tokens, context_length=c.context_length, num_messages=c.num_messages,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("Plugin error classification unavailable: %s", exc)
|
||
return None
|
||
if verdict is not None:
|
||
logger.info("API error classified by plugin hook: %s (provider=%s, status=%s)",
|
||
verdict["reason"].value, c.provider, c.status_code)
|
||
return verdict
|
||
|
||
|
||
def _profile_verdict(c: _Ctx) -> Optional[Verdict]:
|
||
"""The current provider's own ``ProviderProfile.classify_api_error`` verdict, or None.
|
||
|
||
A ``kind: model-provider`` plugin never enters the PluginManager hook lifecycle, so without this a
|
||
vendor-specific body (a 403 ``quota_exhausted`` that is billing, not auth) could only be corrected by
|
||
shipping a second plugin component. Scoped to the provider that produced the error; no name table."""
|
||
if not c.provider_slug:
|
||
return None
|
||
try:
|
||
from providers import get_provider_profile
|
||
hook = getattr(get_provider_profile(c.provider_slug), "classify_api_error", None)
|
||
if not callable(hook):
|
||
return None
|
||
result = hook(c.error, status_code=c.status_code, error_code=c.error_code, message=c.msg,
|
||
body=c.body, model=c.model)
|
||
except Exception as exc:
|
||
logger.debug("Provider profile error classification failed for %s: %s", c.provider_slug, exc)
|
||
return None
|
||
if not isinstance(result, dict):
|
||
return None
|
||
reason = result.get("reason")
|
||
if isinstance(reason, str):
|
||
try:
|
||
reason = FailoverReason(reason.strip().lower())
|
||
except ValueError:
|
||
return None
|
||
if not isinstance(reason, FailoverReason):
|
||
return None
|
||
hints = {k: bool(result[k]) for k in _HINT_FLAGS if k in result}
|
||
# turn_api_error walks the fallback chain only for non-retryable verdicts outside
|
||
# RETRYABLE_CLIENT_REASONS; the built-in terminal verdicts (billing, auth, model_not_found …) pin
|
||
# retryable=False, the rate-limit family stays retryable and reaches fallback after backoff. Give
|
||
# a hook that asks for fallback the built-in default for its reason, so it cascades like one.
|
||
if hints.get("should_fallback") and "retryable" not in hints and reason not in RETRYABLE_CLIENT_REASONS:
|
||
hints["retryable"] = False
|
||
verdict = _v(reason, **hints)
|
||
if isinstance(result.get("error_context"), dict):
|
||
verdict["error_context"] = result["error_context"]
|
||
logger.info("API error classified by provider profile: %s (provider=%s, status=%s)",
|
||
reason.value, c.provider, c.status_code)
|
||
return verdict
|
||
|
||
|
||
_HINT_FLAGS = ("retryable", "should_compress", "should_rotate_credential", "should_fallback")
|
||
|
||
# Reasons the retry loop keeps retrying (with backoff) even though the verdict may also carry
|
||
# ``should_fallback``: the cascade for these runs after the backoff budget, not immediately.
|
||
RETRYABLE_CLIENT_REASONS = frozenset({
|
||
FailoverReason.rate_limit, FailoverReason.upstream_rate_limit, FailoverReason.overloaded,
|
||
FailoverReason.context_overflow, FailoverReason.payload_too_large, FailoverReason.long_context_tier,
|
||
FailoverReason.thinking_signature,
|
||
})
|
||
|
||
|
||
# A welcome-host 403 that spells one of these out is a safety block or a billing wall, not the
|
||
# tier refusing. The free-tier refusal phrases are left OUT: on the free route they mean exactly
|
||
# "the tier refused", and an anonymous session has no credits to check.
|
||
_WELCOME_403_NAMED_PATTERNS = _CONTENT_POLICY_BLOCKED_PATTERNS + tuple(
|
||
p for p in _BILLING_PATTERNS if p not in _FREE_TIER_REFUSAL_PATTERNS)
|
||
|
||
|
||
def _nous_welcome_tier(c: _Ctx) -> Optional[Verdict]:
|
||
"""The Nous inference gateway's welcome-tier (free tier) refusals, read from the structured body.
|
||
|
||
A 429 carrying a fairshare ``reason`` is either a tier gate (``model_not_free`` /
|
||
``feature_not_free``: the model or feature is never served on the free tier, so retrying is
|
||
pointless — abort this route and fall back) or capacity (``at_capacity`` / ``admission_closed``
|
||
/ ``rate_limited``: honour ``retry_after``, never rotate the free tier's only credential). A
|
||
400/403 whose message names the wrong host or a dark tier is deterministic for the request.
|
||
The parsed refusal rides ``error_context`` so the terminal copy can say what happened.
|
||
"""
|
||
from hermes_cli.anon_auth import (
|
||
WELCOME_TIER_GATE_REASONS, parse_welcome_refusal, welcome_route_refusal)
|
||
status = c.status_code
|
||
if not c.anonymous:
|
||
# A named credential's fairshare 429 is an ordinary rate limit, whatever its body says. The
|
||
# one welcome refusal it does receive is the gateway's mirror 400 on the welcome host; its
|
||
# reconnect copy stands, only the sign-in card is withheld (``_welcome_surface_kind``).
|
||
if c.provider == "nous" and status == 400 and welcome_route_refusal(status, c.msg) == "named_on_welcome_host":
|
||
return _v(_R.format_error, retryable=False, should_fallback=True,
|
||
error_context={"welcome_route": "named_on_welcome_host"})
|
||
return None
|
||
if status == 429:
|
||
refusal = parse_welcome_refusal(c.body)
|
||
if refusal is None:
|
||
return None
|
||
ctx = {"welcome_refusal": refusal}
|
||
if refusal["reason"] in WELCOME_TIER_GATE_REASONS:
|
||
return _v(_R.model_not_found, retryable=False, should_fallback=True, error_context=ctx)
|
||
if refusal["retry_after"] > 0:
|
||
ctx["reset_at"] = time.time() + refusal["retry_after"]
|
||
return _v(_R.rate_limit, should_fallback=True, error_context=ctx)
|
||
# The route-keyed dark-tier 403 applies only to a 403 that says nothing else: a safety refusal
|
||
# or a billing wall on the welcome host keeps its own classification (and its own recovery).
|
||
plain_403 = not any(p in c.msg for p in _WELCOME_403_NAMED_PATTERNS)
|
||
kind = welcome_route_refusal(status, c.msg, c.base_url if plain_403 else None)
|
||
if kind is None:
|
||
return None
|
||
ctx = {"welcome_route": kind}
|
||
if status == 403:
|
||
return _v(_R.auth_permanent, retryable=False, should_fallback=True, error_context=ctx)
|
||
return _v(_R.format_error, retryable=False, should_fallback=True, error_context=ctx)
|
||
|
||
|
||
def _provider_special_cases(c: _Ctx) -> Optional[Verdict]:
|
||
"""Highest-priority provider-specific shapes that a status code would misroute."""
|
||
msg, status = c.msg, c.status_code
|
||
welcome = _nous_welcome_tier(c)
|
||
if welcome is not None:
|
||
return welcome
|
||
# Safety refusal before status classification so a 400 block isn't downgraded
|
||
# to format_error and a status-less block isn't left retryable (#18028).
|
||
if any(p in msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS):
|
||
return _V_CONTENT_BLOCKED
|
||
# Status-agnostic: the stream-relayed ban has no status, and a 403 variant is not a bad key.
|
||
if any(p in msg for p in _ACCOUNT_POLICY_BLOCK_PATTERNS):
|
||
return _V_POLICY_BLOCKED
|
||
# ChatGPT Codex masks a rejected encrypted-reasoning replay behind the same bare
|
||
# ``invalid_prompt: Request blocked.`` it uses for real blocks (#92353). Exact envelope
|
||
# + provider only. The verdict keeps format_error's abort-and-fallback hints; the one
|
||
# extra thing it buys is turn_recovery's replay strip, which still requires cached
|
||
# ``codex_reasoning_items`` — a genuine block with nothing to strip behaves as before.
|
||
if _is_codex_masked_replay_rejection(c):
|
||
return _v(_R.invalid_encrypted_content, **_ABORT_FALLBACK)
|
||
# OpenAI Responses rejects a stale encrypted-reasoning replay with this code (#70595). It contains
|
||
# both "thinking" and "signature", so it must beat the Anthropic heuristic below: that recovery
|
||
# strips Anthropic thinking blocks and resends the same encrypted item forever.
|
||
if status == 400 and (c.code == "thinking_signature_invalid" or "thinking_signature_invalid" in msg):
|
||
return _V_INVALID_ENCRYPTED
|
||
# Anthropic thinking-block 400s (signature mismatch after transcript
|
||
# mutation). Not gated on provider — OpenRouter proxies Anthropic errors.
|
||
if status == 400 and "thinking" in msg and any(p in msg for p in _THINKING_MUTATION_WORDS):
|
||
return _v(_R.thinking_signature)
|
||
# Anthropic long-context tier gate (429 "extra usage" + "long context").
|
||
if status == 429 and "extra usage" in msg and "long context" in msg:
|
||
return _v(_R.long_context_tier, should_compress=True)
|
||
# Anthropic OAuth rejects the 1M beta header; run_agent retries without it.
|
||
if status == 400 and "long context beta" in msg and "not yet available" in msg:
|
||
return _v(_R.oauth_long_context_beta_forbidden)
|
||
# llama.cpp grammar rejects regex ``pattern``/``format`` in tool schemas; the
|
||
# retry loop strips them. Exclude the Qwen/vLLM "No user query found" error
|
||
# local engines wrap as "Unable to generate parser for this template" —
|
||
# that is a poisoned transcript (→ format_error), not a grammar problem.
|
||
# Strict OpenAI-compatible schema validators reject regex lookaround in ``pattern``
|
||
# with a different sentence ("Invalid JSON schema: regex lookaround is not supported",
|
||
# #42631); same recovery — strip ``pattern``/``format`` and retry once.
|
||
grammar_hit = "error parsing grammar" in msg or "json-schema-to-grammar" in msg or (
|
||
"unable to generate parser" in msg and "template" in msg
|
||
) or ("invalid json schema" in msg and "regex lookaround" in msg and "not supported" in msg)
|
||
if status == 400 and grammar_hit and _NO_USER_QUERY_SIGNAL not in msg:
|
||
return _v(_R.llama_cpp_grammar_pattern)
|
||
# xAI Grok entitlement as an SSE ``type=error`` frame: no status, matches no
|
||
# pattern list, would otherwise burn max_retries as ``unknown``.
|
||
if "do not have an active grok subscription" in msg or ("out of available resources" in msg and "grok" in msg):
|
||
return _V_AUTH_FALLBACK
|
||
return None
|
||
|
||
|
||
def _moa_special_cases(c: _Ctx) -> Optional[Verdict]:
|
||
# Local MoA streaming adapter-shape bugs are not a provider outage; falling
|
||
# back would silently replace the MoA route with a single model (#55933).
|
||
if c.provider_slug == "moa" and any(s in str(c.error) for s in _MOA_ADAPTER_SHAPE_BUGS):
|
||
return _v(_R.format_error, retryable=False)
|
||
# Persisted MoA preset name that was renamed/deleted — deterministic config error.
|
||
from agent.errors import MoAPresetNotFoundError
|
||
return _v(_R.model_not_found, retryable=False) if isinstance(c.error, MoAPresetNotFoundError) else None
|
||
|
||
|
||
def _by_error_code(c: _Ctx) -> Optional[Verdict]:
|
||
"""Structured error codes from the response body."""
|
||
# Request-validation failure as plain-text ``event: error`` SSE data behind
|
||
# HTTP 200: retrying cannot succeed, a configured fallback still may.
|
||
if c.code == PROVIDER_STREAM_NON_JSON_ERROR_CODE and "request validation failed:" in c.msg:
|
||
return _V_FORMAT_ERROR
|
||
verdict = _ERROR_CODE_VERDICTS.get(c.code)
|
||
if verdict is None:
|
||
family = _PROVIDER_CODE_FAMILIES.get(c.provider_slug, c.provider_slug)
|
||
verdict = _PROVIDER_CODE_VERDICTS.get(family, {}).get(c.code)
|
||
return verdict
|
||
|
||
|
||
def _by_message(c: _Ctx) -> Optional[Verdict]:
|
||
"""Message patterns when no status code settled it; status-less usage
|
||
limits get the same disambiguation as 402."""
|
||
head = _first_match(c.msg, _MESSAGE_HEAD_RULES)
|
||
if head is not None:
|
||
return head
|
||
usage_limit = any(p in c.msg for p in _USAGE_LIMIT_PATTERNS)
|
||
return _classify_402(c.msg, dict) if usage_limit else _first_match(c.msg, _MESSAGE_TAIL_RULES)
|
||
|
||
|
||
def _by_transport(c: _Ctx) -> Optional[Verdict]:
|
||
"""SSL, disconnect, circuit-breaker and transport-type heuristics, in that order."""
|
||
msg = c.msg
|
||
# Cert failure → fail fast (checked first: also contains "[ssl:"); transient
|
||
# alert → retry, before disconnects so a flaky handshake never compresses.
|
||
ssl = _first_match(msg, ((_SSL_CERT_VERIFY_PATTERNS, _V_SSL_CERT), (_SSL_TRANSIENT_PATTERNS, _V_TIMEOUT)))
|
||
if ssl is not None:
|
||
return ssl
|
||
# Disconnect + large session → probable overflow rejection, not a hiccup.
|
||
if any(p in msg for p in _SERVER_DISCONNECT_PATTERNS) and not c.status_code:
|
||
# Reasoning models: far more likely the gateway idle-killed a long
|
||
# thinking stream — never compress on a phantom overflow (#52310).
|
||
# Reasoning-model override: a transport disconnect on a reasoning model is much more likely the
|
||
# upstream proxy idle-killing a long thinking stream than a true context overflow — even on large
|
||
# sessions. The default disconnect+large-session routing below would otherwise send the user into
|
||
# the compression branch (should_compress=True) and silently delete conversation history on a
|
||
# phantom context-length error. Reasoning models have multi-minute thinking phases that routinely
|
||
# exceed the cloud gateway's idle window (NVIDIA NIM ~120s — first-party repro at
|
||
# NVIDIA/NemoClaw#4846; OpenAI worker / Anthropic stream-idle similar). The per-reasoning-model
|
||
# stale-timeout floor in agent/reasoning_timeouts.py raises the stale-detector threshold to tolerate
|
||
# long thinking, so a true transport-layer failure here is recoverable via the retry path — not via
|
||
# context compression. Reclassify as timeout. (Part 1 of Fixes #52310.)
|
||
from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
|
||
if get_reasoning_stale_timeout_floor(c.model) is not None:
|
||
return _V_TIMEOUT
|
||
return _V_CONTEXT_OVERFLOW if c.large_session(0.6, 120000, 200) else _V_TIMEOUT
|
||
# Stale-call circuit breaker (_check_stale_giveup RuntimeError before any
|
||
# network call): as ``unknown`` it would burn every retry instantly.
|
||
if c.error_type == "RuntimeError" and "consecutive stale attempts" in msg and "aborting this call" in msg:
|
||
return _v(_R.timeout, **_ABORT_FALLBACK)
|
||
transport = c.error_type in _TRANSPORT_ERROR_TYPES or isinstance(c.error, (TimeoutError, ConnectionError, OSError))
|
||
return _V_TIMEOUT if transport else None
|
||
|
||
|
||
def _by_status(c: _Ctx) -> Optional[Verdict]:
|
||
"""HTTP status code with message-aware refinement (unlisted 4xx/5xx → generic)."""
|
||
status = c.status_code
|
||
if status is None:
|
||
return None
|
||
default = _V_FORMAT_ERROR if 400 <= status < 500 else _V_SERVER_ERROR if 500 <= status < 600 else None
|
||
return _STATUS_HANDLERS[status](c) if status in _STATUS_HANDLERS else default
|
||
|
||
|
||
# Stage order: plugin hooks → the provider's own profile hook → provider-specific special cases →
|
||
# HTTP status → MoA shapes → structured error code → message patterns → SSL → disconnect +
|
||
# large session → transport types → unknown (retryable with backoff).
|
||
_STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = (
|
||
_plugin_verdict, _profile_verdict, _provider_special_cases, _by_status, _moa_special_cases,
|
||
_by_error_code, _by_message, _by_transport,
|
||
)
|
||
|
||
|
||
def classify_api_error(
|
||
error: Exception, *, provider: str = "", model: str = "",
|
||
approx_tokens: int = 0, context_length: int = 200000, num_messages: int = 0,
|
||
base_url: str = "",
|
||
api_key: Any = None,
|
||
) -> ClassifiedError:
|
||
"""Classify an API error into a structured recovery recommendation (see ``_STAGES``).
|
||
|
||
``base_url`` (optional) is the route the call went to; the Nous welcome tier keys its
|
||
dark-tier 403 on it because that refusal carries no distinguishing message.
|
||
``api_key`` identifies an anonymous request; a host or fairshare reason alone does not.
|
||
The credential is never included in the returned context."""
|
||
from hermes_cli.anon_auth import is_anonymous_request
|
||
status_code = _extract_status_code(error)
|
||
# Copilot/GitHub Models RateLimitError may not set .status_code; force 429.
|
||
if status_code is None and type(error).__name__ == "RateLimitError":
|
||
status_code = 429
|
||
body = _extract_error_body(error)
|
||
c = _Ctx(
|
||
error, status_code, body, _build_error_msg(error, body), provider, model,
|
||
approx_tokens, context_length, num_messages, str(base_url or ""),
|
||
anonymous=is_anonymous_request(provider, api_key),
|
||
)
|
||
verdict = next((v for v in (stage(c) for stage in _STAGES) if v is not None), _V_UNKNOWN)
|
||
message = _extract_message(error, body)
|
||
if verdict["reason"] in (_R.auth, _R.auth_permanent):
|
||
# An auth refusal from a non-stock route names the host, so a credential posted to the
|
||
# wrong endpoint (a stale ``model.base_url`` after a provider switch, #113719) reads as
|
||
# such — not as a bad key.
|
||
host = _off_route_host(c)
|
||
if host:
|
||
message = f"{message} (endpoint: {host})"
|
||
base = {"status_code": status_code, "provider": provider, "model": model, "message": message}
|
||
return ClassifiedError(**{**base, **verdict})
|
||
|
||
|
||
def _off_route_host(c: _Ctx) -> str:
|
||
"""The contacted host when ``base_url`` is set and is not the provider's own endpoint; ``""`` otherwise."""
|
||
from hermes_cli.route_identity import provider_owns_route
|
||
from utils import base_url_hostname
|
||
host = base_url_hostname(c.base_url)
|
||
if not host or provider_owns_route(c.provider_slug, c.base_url) is True:
|
||
return ""
|
||
return host
|
||
|
||
|
||
# ── Status code handlers ────────────────────────────────────────────────
|
||
|
||
# Structured codes some gateways put on a 403 that mean "the upstream is down,
|
||
# retry later" — not a credential refusal (#75388). Checked before the auth
|
||
# default so the configured retry budget applies and no credential is benched.
|
||
_403_TRANSIENT_CODES = frozenset({"upstream_unavailable"})
|
||
|
||
|
||
def _status_403(c: _Ctx) -> Verdict:
|
||
if c.code in _403_TRANSIENT_CODES:
|
||
return _V_OVERLOADED
|
||
# OpenRouter 403 "key limit exceeded" and similar plan/credit exhaustion are billing.
|
||
xai_spend = c.provider_slug == "xai-oauth" and c.code == _XAI_SPENDING_LIMIT_ERROR_CODE
|
||
billing = xai_spend or any(p in c.msg for p in ("key limit exceeded", "spending limit") + _BILLING_PATTERNS)
|
||
if billing:
|
||
return _V_BILLING
|
||
# A WAF/CDN in front of the provider answered, not the provider: the credential never
|
||
# reached it, so key guidance and credential rotation are wrong (#53099, #70566). Gated on
|
||
# 403 and on established block/challenge markers; any other 403 stays auth.
|
||
if any(p in c.msg for p in _UPSTREAM_BLOCKED_PATTERNS):
|
||
return _V_UPSTREAM_BLOCKED
|
||
return _V_AUTH_FALLBACK
|
||
|
||
|
||
def _status_404(c: _Ctx) -> Verdict:
|
||
# Structured billing code first, as in _status_429: this handler always returns,
|
||
# so _by_error_code never sees it; a bare "Not Found" message has nothing to match.
|
||
if c.code in _BILLING_ERROR_CODES:
|
||
return _V_BILLING
|
||
verdict = _first_match(c.msg, _404_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Bare id the catalogue only knows prefixed → malformed id (NVIDIA NIM "404
|
||
# page not found", #78796). A generic 404 (wrong path, proxy glitch) stays
|
||
# unknown so the real error surfaces instead of a silent misreported fallback.
|
||
return _V_MODEL_NOT_FOUND if _model_id_missing_known_prefix(c.model_slug, c.provider_slug) else _V_UNKNOWN
|
||
|
||
|
||
def _status_429(c: _Ctx) -> Verdict:
|
||
# A structured billing code is decisive: LiteLLM stamps
|
||
# ``terminal_quota_exhausted`` (a hard cap, not throttling) on 429s, and
|
||
# this handler always returns, so _by_error_code never sees the code.
|
||
if c.code in _BILLING_ERROR_CODES:
|
||
return _V_BILLING
|
||
# Z.AI/Zhipu reuse 429 for server-wide overload: back off on the same
|
||
# key instead of burning the pool (#14038).
|
||
if any(p in c.msg for p in _OVERLOADED_PATTERNS):
|
||
return _V_OVERLOADED
|
||
# OpenRouter-wrapped upstream 429: the key is healthy — fall back, don't bench.
|
||
if _is_openrouter_upstream_error(c.body, c.provider_slug):
|
||
upstream = _extract_upstream_provider_name(c.body)
|
||
ctx = {"upstream_provider": upstream} if upstream else {}
|
||
return _v(_R.upstream_rate_limit, should_fallback=True, error_context=ctx)
|
||
# Quota walls as 429 (Anthropic ``usage_limit_reached``, "quota", billing
|
||
# phrases) are billing ONLY when the body is not itself a rate-limit phrase
|
||
# ("Rate limit exceeded" contains "limit exceeded") and carries no reset/
|
||
# retry signal (#93419, #39441).
|
||
quota_wall = c.code == "usage_limit_reached" or any(
|
||
p in c.msg for p in ("usage_limit_reached",) + _USAGE_LIMIT_PATTERNS + _BILLING_PATTERNS
|
||
)
|
||
explicit_rate_limit = any(p in c.msg for p in _RATE_LIMIT_PATTERNS)
|
||
if quota_wall and not explicit_rate_limit and not _has_usage_limit_transient_signal(c.msg, c.body, c.headers):
|
||
return _V_BILLING
|
||
# Carry the reset window so the terminal copy can name it instead of "wait a minute" (#89401).
|
||
reset = _rate_limit_reset_seconds(c.msg, c.body, c.headers)
|
||
if reset:
|
||
return _v(_R.rate_limit, **_ROTATE_FALLBACK, error_context={"reset_at": time.time() + reset})
|
||
return _V_RATE_LIMIT
|
||
|
||
|
||
def _status_5xx(c: _Ctx) -> Verdict:
|
||
# Request-validation errors as 5xx (codex.nekos.me) fail fast instead of
|
||
# retry-flooding — unless the parameter was injected server-side.
|
||
validation = any(p in c.msg for p in _REQUEST_VALIDATION_PATTERNS) or c.code in _5XX_VALIDATION_CODES
|
||
if validation and not _is_server_injected_param_rejection(c.msg, c.provider_slug):
|
||
return _V_FORMAT_ERROR
|
||
return _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_SERVER_ERROR
|
||
|
||
|
||
def _classify_402(error_msg: str, result_fn: Callable[..., Any]) -> Any:
|
||
"""Disambiguate 402: "usage limit, try again in 5 minutes" is a periodic quota, not billing."""
|
||
transient = any(p in error_msg for p in _USAGE_LIMIT_PATTERNS) and any(
|
||
p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS
|
||
)
|
||
return result_fn(**(_V_RATE_LIMIT if transient else _V_BILLING))
|
||
|
||
|
||
def _has_large_inline_image(content: Any) -> bool:
|
||
"""True when a rejected ``content`` list carries a ``data:image/`` part the shrink pass would rewrite
|
||
(over ``conversation_compression._IMAGE_SHRINK_TARGET_BYTES``; below it a shrink retry is a no-op)."""
|
||
from agent.conversation_compression import _IMAGE_SHRINK_TARGET_BYTES
|
||
|
||
for part in content if isinstance(content, list) else ():
|
||
image = part.get("image_url") if isinstance(part, dict) else None
|
||
url = image.get("url") if isinstance(image, dict) else image
|
||
if isinstance(url, str) and url.startswith("data:image/") and len(url) > _IMAGE_SHRINK_TARGET_BYTES:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _oversized_message_content_rejection(body: Any) -> bool:
|
||
"""400 rejecting a *message* ``content`` field whose rejected value carries a large inline image.
|
||
|
||
Nebius Token Factory caps a single image at 10 MiB and reports the violation through the field that
|
||
failed to coerce — pydantic ``{"type": "string_type", "loc": ["body","messages",N,"content","str"],
|
||
"msg": "Input should be a valid string", "input": [...]}`` — naming no size vocabulary, so the
|
||
keyword multimodal *tool*-content rule (#104731) claimed it and spent its retry stripping tool images
|
||
that were never there (#112473). The same list-shaped content with a small image succeeds, so the
|
||
image bytes are the trigger. Tool-scoped locs (``messages.N.tool.content.str``) stay with #104731.
|
||
"""
|
||
details = body.get("detail") if isinstance(body, dict) else None
|
||
for detail in details if isinstance(details, list) else ():
|
||
loc = detail.get("loc") if isinstance(detail, dict) else None
|
||
if detail.get("type") != "string_type" or not isinstance(loc, list) or len(loc) < 2:
|
||
continue
|
||
parts = [str(x).lower() for x in loc]
|
||
if parts[:2] == ["body", "messages"] and parts[-2:] == ["content", "str"] and not any(
|
||
x.startswith("tool") for x in parts
|
||
) and _has_large_inline_image(detail.get("input")):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _classify_400(c: _Ctx) -> Verdict:
|
||
"""400 Bad Request — image/tool shapes, request-shape rejections, overflow, or generic."""
|
||
msg, code = c.msg, c.code
|
||
# A size cap reported *through* a message content field must beat the keyword
|
||
# multimodal rule, which would otherwise claim "input should be a valid string".
|
||
if _oversized_message_content_rejection(c.body):
|
||
return _V_IMAGE_TOO_LARGE
|
||
verdict = _first_match(msg, _IMAGE_TOOL_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Codex ChatGPT-account model rejection: exact normalized text only, so arbitrary 400s never
|
||
# rotate. Before request-validation, whose "not supported" wording would abort as format_error (#71970).
|
||
if CODEX_ACCOUNT_MODEL_ENTITLEMENT_MARKER in msg:
|
||
return _V_MODEL_ENTITLEMENT
|
||
# Invalid encrypted reasoning replay blob (OpenAI Responses); before
|
||
# overflow because "encrypted content … could not be verified" trips it.
|
||
if code == "invalid_encrypted_content" or "invalid_encrypted_content" in msg or (
|
||
"encrypted content for item" in msg and "could not be verified" in msg
|
||
) or "could not decrypt the provided encrypted_content" in msg or (
|
||
# Custom Responses endpoints wrap a replay rejection in a generic bad_request (#95834).
|
||
"encrypted content could not be decrypted or parsed" in msg
|
||
) or (
|
||
# OpenCode Zen wraps this OpenAI replay rejection in ``invalid_request_error`` (#111309).
|
||
"encrypted_content" in msg and "was not issued to this caller" in msg
|
||
) or (
|
||
# Azure Foundry (gpt-6-astra) rejects replayed reasoning from several prior responses this way (#105369).
|
||
"conflicting authenticated continuation identities" in msg
|
||
):
|
||
return _V_INVALID_ENCRYPTED
|
||
# Route rejecting a reasoning disable: a reasoning-mandatory route (GLM-5.3 on Nous Portal /
|
||
# OpenRouter) or a chat-only relay that does not accept ``reasoning_effort: none`` at all
|
||
# (#114460). Deterministic for the request shape, but the only bad field is the disable — the
|
||
# loop drops it and retries once. Must precede request-validation, which would abort as format_error.
|
||
if _REASONING_MANDATORY_PATTERN in msg or is_reasoning_field_rejection(msg):
|
||
return _V_REASONING_MANDATORY
|
||
# 400 blaming a field this route never sent (Codex OAuth injects then rejects
|
||
# prompt_cache_retention ~20% of the time): transient, retry identical request.
|
||
if _is_server_injected_param_rejection(msg, c.provider_slug):
|
||
return _V_SERVER_ERROR
|
||
if any(p in msg for p in _MALFORMED_TOOL_ARGS_PATTERNS):
|
||
return _V_MALFORMED_TOOL_ARGS
|
||
if any(p in msg for p in _ROLE_ALTERNATION_PATTERNS):
|
||
return _V_ROLE_ALTERNATION
|
||
# Before overflow: GPT-5's "Unsupported parameter: 'max_tokens'" contains it.
|
||
if any(p in msg for p in _400_VALIDATION_PATTERNS) or code in _400_VALIDATION_CODES:
|
||
return _V_FORMAT_ERROR
|
||
# Malformed message array before overflow: input can be tiny and compression
|
||
# cannot fix it. litellm/Bedrock proxies use errorCode=INVALID_REQUEST_BODY.
|
||
if any(p in msg for p in _INVALID_MESSAGE_BODY_PATTERNS) or code == "invalid_request_body":
|
||
logger.warning(
|
||
"Malformed message array 400 (invalid request body) classified as format_error, NOT context "
|
||
"overflow — failing fast + falling back instead of entering the compression loop. This usually "
|
||
"means an empty-content assistant stub is in the transcript; num_messages=%s approx_tokens=%s. "
|
||
"error=%.200s", c.num_messages, c.approx_tokens, msg,
|
||
)
|
||
return _V_FORMAT_ERROR
|
||
# Memory ceiling by code: _by_status runs before _by_error_code, so a
|
||
# 400 whose wording a proxy stripped would fall through to format_error.
|
||
if code in _MEMORY_CEILING_ERROR_CODES:
|
||
return _V_OVERLOADED
|
||
verdict = _first_match(msg, _400_TAIL_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Generic 400 + large session → probable overflow (Anthropic can return a
|
||
# bare "Error"); proxy shapes are read so a descriptive rejection isn't "bare".
|
||
body_msg = next((m for m in (str(x or "").strip().lower() for x in _body_message_candidates(c.body)) if m), "")
|
||
is_generic = len(body_msg) < 30 or body_msg in {"error", ""}
|
||
if is_generic and c.large_session(0.4, 80000, 80):
|
||
return _V_CONTEXT_OVERFLOW
|
||
return _V_FORMAT_ERROR
|
||
|
||
|
||
def _classify_image_tool_422(c: _Ctx) -> Verdict:
|
||
"""422: pydantic relays report the same content-field shapes as 400 (#104731, #112473)."""
|
||
if _oversized_message_content_rejection(c.body):
|
||
return _V_IMAGE_TOO_LARGE
|
||
return _first_match(c.msg, _IMAGE_TOOL_RULES) or _V_FORMAT_ERROR
|
||
|
||
|
||
# 401 not retryable on its own: rotation/refresh run before the retryability
|
||
# check, then the client-error abort path (fallback first) is correct. 408 is
|
||
# retry-safe (RFC 9110 §15.5.9; proxies emit it when generation outruns the
|
||
# read window). Unlisted 4xx → format_error, 5xx → server_error.
|
||
_STATUS_HANDLERS: Dict[int, Callable[[_Ctx], Verdict]] = {
|
||
400: _classify_400, 401: lambda c: _V_AUTH_ROTATE, 402: lambda c: _classify_402(c.msg, dict),
|
||
403: _status_403, 404: _status_404, 408: lambda c: _V_TIMEOUT, 413: lambda c: _V_PAYLOAD_TOO_LARGE,
|
||
422: lambda c: _classify_image_tool_422(c),
|
||
429: _status_429, 500: _status_5xx, 502: _status_5xx,
|
||
503: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
|
||
529: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
|
||
}
|
||
|
||
|
||
# ── Helpers ─────────────────────────────────────────────────────────────
|
||
|
||
_RESET_FIELDS = ("resets_in_seconds", "resets_at", "reset_at", "retry_after")
|
||
_RESET_HEADERS = ("retry-after", "Retry-After", "x-ratelimit-reset", "X-RateLimit-Reset")
|
||
|
||
|
||
def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_headers) -> bool:
|
||
"""Whether a usage-limit response identifies a reset window (message, body fields, or headers)."""
|
||
if any(pattern in error_msg for pattern in _USAGE_LIMIT_TRANSIENT_SIGNALS):
|
||
return True
|
||
payloads = [p for p in (body, _error_obj(body)) if isinstance(p, dict)]
|
||
if any(payload.get(f) not in (None, "") for payload in payloads for f in _RESET_FIELDS):
|
||
return True
|
||
if response_headers and hasattr(response_headers, "get"):
|
||
return any(response_headers.get(h) not in (None, "") for h in _RESET_HEADERS)
|
||
return False
|
||
|
||
|
||
def _rate_limit_reset_seconds(error_msg: str, body: dict, response_headers) -> Optional[float]:
|
||
"""Seconds until a 429's window reopens, from the body's reset fields, ``Retry-After`` or the
|
||
message grammar (``retry after Ns`` / ``resets in 4hr``); None when the response names none."""
|
||
from agent.retry_utils import parse_retry_after_seconds, reset_delay_from_message
|
||
for payload in (p for p in (body, _error_obj(body)) if isinstance(p, dict)):
|
||
for name in _RESET_FIELDS:
|
||
value = payload.get(name)
|
||
if value in (None, ""):
|
||
continue
|
||
if name.endswith("_at") and isinstance(value, (int, float)):
|
||
return max(0.0, float(value) - time.time())
|
||
if (seconds := parse_retry_after_seconds(value)) is not None:
|
||
return seconds
|
||
if (seconds := parse_retry_after_seconds(response_headers)) is not None:
|
||
return seconds
|
||
return reset_delay_from_message(error_msg)
|
||
|
||
|
||
def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
|
||
"""True when a bare model id is only known to the provider as ``vendor/id``.
|
||
|
||
Never guesses: an id absent from the curated catalogue returns False so real
|
||
endpoint problems keep their retryable ``unknown`` classification.
|
||
"""
|
||
name = (model or "").strip()
|
||
if not name or "/" in name:
|
||
return False
|
||
try:
|
||
from hermes_cli.model_normalize import suggest_prefixed_model_id
|
||
return bool(suggest_prefixed_model_id((provider or "").strip(), name))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
|
||
"""True when a 400 blames a one-route-only parameter this route never sends.
|
||
|
||
Conservative: known parameters only, and only when ``provider`` is not a
|
||
sender, so a genuine bad parameter (``max_tokens`` on GPT-5) stays format_error.
|
||
"""
|
||
provider_slug = (provider or "").strip().lower()
|
||
for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
|
||
if error_msg and param in error_msg and any(w in error_msg for w in _PARAM_REJECTION_WORDS):
|
||
return not any(sender in provider_slug for sender in senders)
|
||
return False
|
||
|
||
|
||
_CODEX_MASKED_REPLAY_MESSAGE = "request blocked."
|
||
_CODEX_UNSUPPORTED_CONTENT_DETAIL = "unsupported content type"
|
||
|
||
|
||
def _is_codex_masked_replay_rejection(c: "_Ctx") -> bool:
|
||
"""HTTP 400 / status-less ``{code: invalid_prompt, message: "Request blocked."}`` from
|
||
``openai-codex`` — as an SDK error body, a Responses ``error`` SSE frame, or the
|
||
``response.failed`` text ``"invalid_prompt: Request blocked."`` — or the bare
|
||
``{"detail": "Unsupported content type"}`` envelope the same backend returns for a rejected
|
||
encrypted-reasoning replay (#51512). Both are exact envelopes, provider-gated."""
|
||
if c.provider_slug != "openai-codex" or c.status_code not in (None, 400):
|
||
return False
|
||
body = c.body if isinstance(c.body, dict) else {}
|
||
if str(body.get("detail") or "").strip().lower() == _CODEX_UNSUPPORTED_CONTENT_DETAIL or (
|
||
not body and _CODEX_UNSUPPORTED_CONTENT_DETAIL in c.msg and "detail" in c.msg
|
||
):
|
||
return True
|
||
# The OpenAI SDK unwraps ``body["error"]`` on status errors; stream frames keep the envelope.
|
||
body_msg = next((str(m).strip().lower() for m in _body_message_candidates(body) if m), "")
|
||
return (c.code == "invalid_prompt" and body_msg == _CODEX_MASKED_REPLAY_MESSAGE) or (
|
||
c.msg.strip() == f"invalid_prompt: {_CODEX_MASKED_REPLAY_MESSAGE}"
|
||
)
|
||
|
||
|
||
def _error_obj(body: Any) -> dict:
|
||
"""``body["error"]`` when it is a dict, else ``{}``."""
|
||
err = body.get("error") if isinstance(body, dict) else None
|
||
return err if isinstance(err, dict) else {}
|
||
|
||
|
||
def _json_dict(text: Any) -> Optional[dict]:
|
||
"""Parse a JSON object string; None for non-strings, blanks, invalid JSON or non-objects."""
|
||
if not (isinstance(text, str) and text.strip()):
|
||
return None
|
||
try:
|
||
inner = json.loads(text)
|
||
except (json.JSONDecodeError, TypeError):
|
||
return None
|
||
return inner if isinstance(inner, dict) else None
|
||
|
||
|
||
def _openrouter_wrapped_message(err_obj: dict) -> str:
|
||
"""Lowercased inner message from OpenRouter's ``error.metadata.raw`` JSON wrapper."""
|
||
metadata = err_obj.get("metadata", {})
|
||
inner = _json_dict(metadata.get("raw")) if isinstance(metadata, dict) else None
|
||
return str(_error_obj(inner).get("message") or "").lower() if inner else ""
|
||
|
||
|
||
def _build_error_msg(error: Exception, body: Any) -> str:
|
||
"""Lowercased str(error) + body message + OpenRouter-wrapped upstream message
|
||
(OpenAI SDK's APIStatusError.__str__ omits the body, so it is appended)."""
|
||
raw_msg = str(error).lower()
|
||
body_msg = metadata_msg = ""
|
||
if isinstance(body, dict):
|
||
err_obj = _error_obj(body)
|
||
body_msg = str(err_obj.get("message") or "").lower() or str(body.get("message") or "").lower()
|
||
metadata_msg = _openrouter_wrapped_message(err_obj) if err_obj else ""
|
||
parts = [raw_msg]
|
||
if body_msg and body_msg not in raw_msg:
|
||
parts.append(body_msg)
|
||
if metadata_msg and metadata_msg not in raw_msg and metadata_msg not in body_msg:
|
||
parts.append(metadata_msg)
|
||
return " ".join(parts)
|
||
|
||
|
||
def _body_message_candidates(body: dict) -> Iterator[Any]:
|
||
"""Body message fields in priority order (OpenAI, flat, litellm/Bedrock proxy, FastAPI shapes)."""
|
||
yield _error_obj(body).get("message")
|
||
yield body.get("message")
|
||
yield body.get("errorMessage")
|
||
args = body.get("errorArgs")
|
||
yield args.get("reason") if isinstance(args, dict) else None
|
||
# FastAPI/Starlette relays and the Codex gateway answer {"detail": "..."} (or a nested
|
||
# OpenAI-ish object); without it a descriptive rejection reads as a bare 400 and the
|
||
# large-session heuristic sends it into compression (#81558). A list here is pydantic's
|
||
# validation shape, read by _oversized_message_content_rejection.
|
||
detail = body.get("detail")
|
||
yield detail.get("message") if isinstance(detail, dict) else detail if isinstance(detail, str) else None
|
||
|
||
|
||
def _from_cause_chain(error: Exception, pick: Callable[[Any], Any], default: Any) -> Any:
|
||
"""First non-None ``pick(exc)`` over the error and its __cause__/__context__ chain (max 5 deep)."""
|
||
current = error
|
||
for _ in range(5):
|
||
found = pick(current)
|
||
if found is not None:
|
||
return found
|
||
cause = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
|
||
if cause is None or cause is current:
|
||
break
|
||
current = cause
|
||
return default
|
||
|
||
|
||
def _status_of(exc: Any) -> Optional[int]:
|
||
code = getattr(exc, "status_code", None)
|
||
if isinstance(code, int):
|
||
return code
|
||
code = getattr(exc, "status", None) # some SDKs use .status
|
||
return code if isinstance(code, int) and 100 <= code < 600 else None
|
||
|
||
|
||
def _body_of(exc: Any) -> Optional[dict]:
|
||
body = getattr(exc, "body", None)
|
||
if isinstance(body, dict):
|
||
return body
|
||
response = getattr(exc, "response", None)
|
||
try:
|
||
json_body = response.json() if response is not None else None
|
||
except Exception:
|
||
return None
|
||
return json_body if isinstance(json_body, dict) else None
|
||
|
||
|
||
def _headers_of(exc: Any) -> Any:
|
||
headers = getattr(getattr(exc, "response", None), "headers", None)
|
||
return headers if headers and hasattr(headers, "get") else None
|
||
|
||
|
||
def _status_code_from_body(body: Any) -> Optional[int]:
|
||
"""Numeric HTTP error status (400-599) from ``error.code``/``code`` in a structured body.
|
||
An aggregator/relay can deliver the upstream failure only this way — as an
|
||
error object inside an HTTP-200 SSE stream — leaving the SDK to raise a
|
||
status-less ``APIError`` whose ``body`` carries the status (#121270). String
|
||
codes stay symbolic (``_code_from_payload``'s ``"400" is not a code``), unlike the
|
||
string-parsing text-SSE sibling ``chat_completion_helpers._status_code_from_payload``."""
|
||
if not isinstance(body, dict):
|
||
return None
|
||
error_obj = _error_obj(body)
|
||
candidates = [error_obj.get(k) for k in ("status_code", "status", "http_status", "code")] + [body.get("code")]
|
||
return next(
|
||
(c for c in candidates if isinstance(c, int) and not isinstance(c, bool) and 400 <= c < 600),
|
||
None,
|
||
)
|
||
|
||
|
||
def _extract_status_code(error: Exception) -> Optional[int]:
|
||
"""HTTP status code from the error or its cause chain; a body-carried numeric
|
||
``code`` counts when the exception itself carries no status (#121270)."""
|
||
status = _from_cause_chain(error, _status_of, None)
|
||
if status is None:
|
||
status = _status_code_from_body(_from_cause_chain(error, _body_of, {}))
|
||
return status
|
||
|
||
|
||
def _extract_error_body(error: Exception) -> dict:
|
||
"""Structured error body from an SDK exception or its cause chain."""
|
||
return _from_cause_chain(error, _body_of, {})
|
||
|
||
|
||
def _code_from_payload(payload: Any, top_keys: Sequence[str], peek_message: bool) -> str:
|
||
"""Code/type from ``payload.error`` or a top-level key; ``"400"`` is not a code.
|
||
``peek_message`` also parses a JSON ``error.message`` for a nested code
|
||
(Responses API surfaces ``invalid_encrypted_content`` this way). Gemini
|
||
puts the HTTP status in ``error.code`` and the symbolic code
|
||
(``UNAVAILABLE``) in ``error.status``, so a numeric code defers to it."""
|
||
if not isinstance(payload, dict):
|
||
return ""
|
||
error_obj = payload.get("error", {})
|
||
if isinstance(error_obj, dict):
|
||
code = error_obj.get("code") or error_obj.get("type") or ""
|
||
if not isinstance(code, str):
|
||
code = error_obj.get("status") or code
|
||
if isinstance(code, str) and code.strip() and code.strip() != "400":
|
||
return code.strip()
|
||
message = error_obj.get("message")
|
||
if peek_message and isinstance(message, str) and message.strip().startswith("{"):
|
||
nested_code = _code_from_payload(_json_dict(message), ("code", "error_code"), False)
|
||
if nested_code:
|
||
return nested_code
|
||
code = next((payload.get(k) for k in top_keys if payload.get(k)), "")
|
||
text = str(code).strip() if isinstance(code, (str, int)) else ""
|
||
return text if text and text != "400" else ""
|
||
|
||
|
||
def _extract_error_code(body: dict) -> str:
|
||
"""Extract an error code string from the response body."""
|
||
return _code_from_payload(body, ("code", "error_code", "errorCode"), True) if body else ""
|
||
|
||
|
||
def _extract_message(error: Exception, body: dict) -> str:
|
||
"""Extract the most informative error message (structured body first)."""
|
||
msg = next((m for m in _body_message_candidates(body or {}) if isinstance(m, str) and m.strip()), None)
|
||
return (msg.strip() if msg else str(error))[:500]
|
||
|
||
|
||
def _is_openrouter_upstream_error(body: Any, provider: str) -> bool:
|
||
"""OpenRouter's "Provider returned error" wrapper: the key is healthy, the
|
||
upstream failed, so credential rotation is the wrong recovery."""
|
||
err = _error_obj(body)
|
||
if str(err.get("message") or "").strip().lower() != "provider returned error":
|
||
return False
|
||
if (provider or "").strip().lower() == "openrouter":
|
||
return True
|
||
# Otherwise require the metadata shape only OpenRouter produces.
|
||
metadata = err.get("metadata")
|
||
return isinstance(metadata, dict) and ("raw" in metadata or "provider_name" in metadata)
|
||
|
||
|
||
def _extract_upstream_provider_name(body: Any) -> Optional[str]:
|
||
"""Pull the upstream provider name out of OpenRouter's error metadata."""
|
||
metadata = _error_obj(body).get("metadata")
|
||
name = metadata.get("provider_name") if isinstance(metadata, dict) else None
|
||
return name.strip() if isinstance(name, str) and name.strip() else None
|