1208 lines
65 KiB
Python
1208 lines
65 KiB
Python
"""API error classification for smart failover and recovery.
|
||
|
||
A priority-ordered pipeline maps an API exception to a ``ClassifiedError``
|
||
whose recovery hints (retry, rotate credential, fallback, compress, abort) the
|
||
retry loop in run_agent.py consults instead of re-matching strings itself.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import enum
|
||
import json
|
||
import logging
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Callable, Dict, Iterator, Optional, Sequence
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Synthetic code for the OpenAI SDK rejecting a provider's SSE ``data:`` field
|
||
# before any completion chunk arrives; distinct from generic JSON parse errors.
|
||
PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data"
|
||
|
||
# Same rejection with an EMPTY payload: the frame carried no ``data`` at all (``data:`` /
|
||
# ``event: ping`` / ``id:`` with no content). Per the SSE spec those are legal keepalives /
|
||
# no-ops, not malformed payloads — a degraded gateway answers every streaming request with
|
||
# them, so the session switches to non-streaming instead of re-streaming into the same
|
||
# window. See ``chat_completion_helpers._maybe_disable_streaming``.
|
||
PROVIDER_STREAM_EMPTY_FRAME_ERROR_CODE = "provider_stream_empty_frame"
|
||
|
||
|
||
# ── Error taxonomy ──────────────────────────────────────────────────────
|
||
|
||
class FailoverReason(enum.Enum):
|
||
"""Why an API call failed — determines recovery strategy."""
|
||
auth = "auth" # Transient auth (401/403) — refresh/rotate
|
||
auth_permanent = "auth_permanent" # Auth failed after refresh — abort
|
||
billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately
|
||
rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate
|
||
upstream_rate_limit = "upstream_rate_limit" # Aggregator's upstream model 429 — fallback model, key is healthy
|
||
overloaded = "overloaded" # 503/529 — provider overloaded, backoff
|
||
server_error = "server_error" # 500/502 — internal server error, retry
|
||
timeout = "timeout" # Connection/read timeout — rebuild client + retry
|
||
ssl_cert_verification = "ssl_cert_verification" # Deterministic TLS chain failure — fail fast with guidance
|
||
context_overflow = "context_overflow" # Context too large — compress, not failover
|
||
payload_too_large = "payload_too_large" # 413 — compress payload
|
||
image_too_large = "image_too_large" # Native image part exceeds provider's per-image limit — shrink and retry
|
||
image_corrupt = "image_corrupt" # Provider can't decode image bytes — strip and retry (shrinking won't help)
|
||
model_not_found = "model_not_found" # 404 or invalid model — fallback to different model
|
||
provider_policy_blocked = "provider_policy_blocked" # Aggregator account data/privacy policy excluded the only endpoint
|
||
content_policy_blocked = "content_policy_blocked" # Provider safety filter rejected this prompt — don't retry unchanged
|
||
format_error = "format_error" # 400 bad request — abort or strip + retry
|
||
invalid_encrypted_content = "invalid_encrypted_content" # Responses replay blob rejected — strip replay state and retry
|
||
multimodal_tool_content_unsupported = "multimodal_tool_content_unsupported" # Provider rejected list-type content in tool messages (e.g. Xiaomi MiMo) — downgrade to text and retry
|
||
reasoning_mandatory = "reasoning_mandatory" # Route rejects reasoning: {enabled: false} — send the disable no more this session and retry
|
||
|
||
# Provider-specific
|
||
thinking_signature = "thinking_signature" # Anthropic thinking block sig invalid
|
||
long_context_tier = "long_context_tier" # Anthropic "extra usage" tier gate
|
||
oauth_long_context_beta_forbidden = "oauth_long_context_beta_forbidden" # Anthropic OAuth rejects 1M beta — disable beta and retry
|
||
llama_cpp_grammar_pattern = "llama_cpp_grammar_pattern" # llama.cpp grammar rejects regex `pattern`/`format` — strip from tools and retry
|
||
unknown = "unknown" # Unclassifiable — retry with backoff
|
||
|
||
|
||
@dataclass
|
||
class ClassifiedError:
|
||
"""Structured classification of an API error with recovery hints."""
|
||
|
||
reason: FailoverReason
|
||
status_code: Optional[int] = None
|
||
provider: Optional[str] = None
|
||
model: Optional[str] = None
|
||
message: str = ""
|
||
error_context: Dict[str, Any] = field(default_factory=dict)
|
||
|
||
# Recovery hints — the retry loop checks these instead of re-classifying.
|
||
retryable: bool = True
|
||
should_compress: bool = False
|
||
should_rotate_credential: bool = False
|
||
should_fallback: bool = False
|
||
|
||
@property
|
||
def is_auth(self) -> bool:
|
||
return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent}
|
||
|
||
@property
|
||
def billing_unverified(self) -> bool:
|
||
"""True when a ``billing`` verdict rests on an ambiguous body (#82154)."""
|
||
return bool(self.error_context.get("billing_unverified"))
|
||
|
||
|
||
# ── Pattern tables (lowercased substrings) ──────────────────────────────
|
||
|
||
# Billing exhaustion (not transient rate limit). "out of extra usage" is the
|
||
# Anthropic OAuth Pro/Max overage bucket depleted (HTTP 400).
|
||
# The Nous gateway's own words for "the free tier will not serve this" — a billing wall for a
|
||
# named account, the tier refusing for an anonymous one (see ``_WELCOME_403_NAMED_PATTERNS``).
|
||
_FREE_TIER_REFUSAL_PATTERNS = ("model_not_supported_on_free_tier", "not available on the free tier")
|
||
_BILLING_PATTERNS = (
|
||
"insufficient credits", "insufficient_quota", "insufficient balance", "credit balance",
|
||
"credits exhausted", "credits have been exhausted", "requires available credits",
|
||
"account balance is too low", "no usable credits", "top up your credits", "payment required",
|
||
"billing hard limit", "exceeded your current quota", "account is deactivated", "plan does not include",
|
||
"out of extra usage", "out of funds", "run out of funds", "balance_depleted",
|
||
# OpenRouter org-level monthly cap arrives as 403 "Budget limit exceeded (monthly limit)" (#107166):
|
||
# account exhaustion, not a credential problem.
|
||
"budget limit exceeded",
|
||
*_FREE_TIER_REFUSAL_PATTERNS,
|
||
# LiteLLM proxies word a hard cap as "hard billing limit" (structured twin:
|
||
# ``terminal_quota_exhausted`` in _BILLING_ERROR_CODES). "terminal billing
|
||
# limit" free text is NOT matched: substring rules can't negate the
|
||
# "non-terminal billing limit" wording, and the structured code covers it.
|
||
"hard billing limit",
|
||
)
|
||
|
||
# Not proof of exhaustion: Anthropic returns the same "out of extra usage" body
|
||
# for a content-filter rejection (#82154). Verdict stays ``billing`` but is
|
||
# marked unverified so surfaces hedge and the pool uses a short cooldown.
|
||
_UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",)
|
||
|
||
# xAI's Grok credit-exhaustion code arrives as HTTP 403, not 402. Provider-
|
||
# scoped on purpose: other providers' billing codes on a 403 stay auth failures.
|
||
_XAI_SPENDING_LIMIT_ERROR_CODE = "personal-team-blocked:spending-limit"
|
||
|
||
# Structured codes meaning the account cannot serve paid traffic.
|
||
_BILLING_ERROR_CODES = frozenset({
|
||
"insufficient_quota", "billing_not_active", "payment_required", "insufficient_credits",
|
||
"no_usable_credits", "balance_depleted", "model_not_supported_on_free_tier",
|
||
"member_spend_cap_exceeded", "terminal_quota_exhausted", _XAI_SPENDING_LIMIT_ERROR_CODE,
|
||
# OpenAI (and OpenAI-compatible aggregators) spend/usage-limit family:
|
||
# a credit balance or an org/project spend or usage cap is exhausted —
|
||
# terminal for this credential until limits are raised.
|
||
"credit_balance_exhausted", "organization_spend_limit_exceeded",
|
||
"organization_usage_limit_exceeded", "project_spend_limit_exceeded",
|
||
})
|
||
|
||
# Transient rate limiting. Bedrock "Throttling error: Too many tokens" also
|
||
# contains an overflow phrase; rate limit is matched first so throttle wins.
|
||
_RATE_LIMIT_PATTERNS = (
|
||
"rate limit", "rate_limit", "too many requests", "throttled", "requests per minute",
|
||
"tokens per minute", "requests per day", "try again in", "please retry after",
|
||
"resource exhausted", "resource_exhausted", "resource-exhausted", "resourceexhausted",
|
||
"rate increased too quickly", "throttlingexception", "too many concurrent requests",
|
||
"servicequotaexceededexception", "throttling",
|
||
)
|
||
|
||
# Server busy, credential fine: back off on the same key, never rotate. Z.AI/
|
||
# Zhipu reuse HTTP 429 for this, so the 429 path checks these first. Kept narrow
|
||
# so a plain "you have been rate-limited" doesn't land here. (#14038, #15297)
|
||
_OVERLOADED_PATTERNS = (
|
||
"overloaded", "temporarily overloaded", "service is temporarily overloaded",
|
||
"service may be temporarily overloaded", "server is overloaded", "server overloaded",
|
||
"server overload", "server_overload",
|
||
"service overloaded", "service is overloaded", "upstream overloaded", "currently overloaded",
|
||
"at capacity", "over capacity",
|
||
)
|
||
|
||
# Usage-limit patterns that need disambiguation (billing OR rate_limit), and
|
||
# the signals that mark such a limit as transient (periodic quota, not billing).
|
||
_USAGE_LIMIT_PATTERNS = ("usage limit", "quota", "limit exceeded", "key limit exceeded")
|
||
_USAGE_LIMIT_TRANSIENT_SIGNALS = (
|
||
"try again", "retry", "resets at", "reset in", "resets in", "reset after", "available in",
|
||
"wait", "requests remaining", "periodic", "window", "per minute", "per second",
|
||
)
|
||
|
||
# 413 detected from message text (proxies embed the status or re-wrap
|
||
# Anthropic's "request_too_large" type without one).
|
||
_PAYLOAD_TOO_LARGE_PATTERNS = (
|
||
"request entity too large", "payload too large", "error code: 413", "request_too_large",
|
||
# Normally arrives with an HTTP 413 status (handled by the status path), but aggregators/proxies can
|
||
# re-wrap it into a plain message with no status attribute — route it to the same compression recovery.
|
||
# (port of anomalyco/opencode#37848)
|
||
"request exceeds the maximum size",
|
||
)
|
||
|
||
# Per-image size/dimension 400s (Anthropic 5 MB / 8000 px; MiniMax "media
|
||
# exceeds size limit" #76039) — a specific 400 before the request hits 413. A
|
||
# non-image media hit is harmless: the shrink pass finds no image parts.
|
||
# "patches after processing": OpenAI Codex Responses rejects an image whose
|
||
# tile-patch budget (ceil(w/32)×ceil(h/32)) exceeds its 30000-patch ceiling
|
||
# with wording that names no image-size vocabulary — without this pattern it
|
||
# fell to format_error (non-retryable), bypassing the shrink recovery (#106337).
|
||
# Byte caps enforced with a 400 instead of a 413 (#112473): NVIDIA NIM caps the whole
|
||
# payload ("Please make sure your payload is below 26214400 bytes in size"); Alibaba
|
||
# DashScope caps the base64 image string via Jackson ("String value length (N) exceeds the
|
||
# maximum allowed (M, from `StreamReadConstraints.getMaxStringLength()`)"). Only an inline
|
||
# image reaches those sizes, so shrinking is the recovery; the method-scoped Jackson token
|
||
# is used because the bare class name also appears when Jackson caps a *token* length.
|
||
_IMAGE_TOO_LARGE_PATTERNS = (
|
||
"image exceeds", "image too large", "image_too_large", "image size exceeds", "image dimensions exceed",
|
||
"dimensions exceed max allowed size", "max allowed size: 8000", "media exceeds", "media too large",
|
||
"patches after processing", "make sure your payload is below", "streamreadconstraints.getmaxstringlength",
|
||
)
|
||
|
||
# Undecodable image bytes → strip-and-retry, never shrink. xAI wordings
|
||
# (#69078); the last is the full sentence because shorter fragments also match
|
||
# non-image download failures.
|
||
_IMAGE_CORRUPT_PATTERNS = (
|
||
"invalid png image", "invalid jpeg image", "base64 string of provided image cannot be decoded",
|
||
"downloaded response does not contain a valid jpg, png, webp, or ico image",
|
||
)
|
||
|
||
# 400s rejecting list-type ``content`` in tool messages (Xiaomi MiMo "text is
|
||
# not set", Alibaba, OpenAI-compat long tail). Recovery: strip image parts from
|
||
# tool messages, remember (provider, model), retry. (#27344)
|
||
# NVIDIA NIM's Rust gateway never names the field: its serde rejection says the
|
||
# body "did not match any variant of untagged enum
|
||
# ChatCompletionRequestToolMessageContent", which is the same list-type tool
|
||
# content that every other wording here describes (#111231).
|
||
_MULTIMODAL_TOOL_CONTENT_PATTERNS = (
|
||
"text is not set", "tool message content must be a string", "tool content must be a string",
|
||
"tool message must be a string", "expected string, got list", "expected string, got array",
|
||
# Console Go / pydantic-v2 relays behind opencode-go (422, param ``messages.N.tool.content.str``, #104731).
|
||
"tool_call.content must be string", "tool.content.str", "input should be a valid string",
|
||
"chatcompletionrequesttoolmessagecontent",
|
||
)
|
||
|
||
# Local-inference memory/resource-ceiling rejections (oMLX/MLX memory guard,
|
||
# llama.cpp/vLLM OOM, Metal/CUDA allocation ceilings). The server aborts on a
|
||
# prefill memory PEAK, not a window limit, yet its remediation tail says
|
||
# "reduce context length" — so without this list the request routes into
|
||
# compression, which cannot lower a prefill peak: it burns the compression
|
||
# budget, re-hits the wedged server each attempt and ends in a session reset.
|
||
# Every token names memory/allocation in BYTES, never a token count, so the
|
||
# list is disjoint from _CONTEXT_OVERFLOW_PATTERNS. Must be checked BEFORE
|
||
# both overflow AND the usage-limit disambiguation ("memory limit exceeded"
|
||
# contains "limit exceeded", which would otherwise read as billing). oMLX
|
||
# reworded the accounting sentence in 0.5.7 ("predicted peak would require /
|
||
# exceed"); the 0.5.6 wording is still in the field, so both stay. (#52261)
|
||
_MEMORY_CEILING_PATTERNS = (
|
||
"memory guard", "memory limit exceeded", "memory_guard_tier", "dynamic ceiling",
|
||
"memory ceiling", "available memory", "out of memory", "insufficient memory",
|
||
"prefill would require", "predicted peak would", "prefill safety cap", "metal_cap",
|
||
)
|
||
|
||
# Structured codes identifying the same rejection at the source, before an
|
||
# OpenAI-compatible proxy flattens the body and drops the wording.
|
||
_MEMORY_CEILING_ERROR_CODES = frozenset({
|
||
"prefill_memory_exceeded", "prefill_memory_aborted", "omlx_prefill_memory_exceeded",
|
||
})
|
||
|
||
# Bare "max_tokens" is load-bearing: the output-cap-retry path keys off it;
|
||
# empty-response advisories mentioning it are intercepted earlier. Groups:
|
||
# generic; vLLM; Ollama; llama.cpp; Chinese; Z.AI (1210); Bedrock; Together.
|
||
_CONTEXT_OVERFLOW_PATTERNS = (
|
||
"context length", "context size", "maximum context", "token limit", "too many tokens",
|
||
"reduce the length", "exceeds the limit", "context window", "prompt is too long",
|
||
"prompt exceeds max length", "max_tokens", "maximum number of tokens",
|
||
"exceeds the max_model_len", "max_model_len", "prompt length", "input is too long", "maximum model length",
|
||
"context length exceeded", "truncating input",
|
||
"slot context", "n_ctx_slot",
|
||
"超过最大长度", "上下文长度",
|
||
"tokens in request more than max tokens allowed",
|
||
"input is too long", "max input token", "input token", "exceeds the maximum number of input tokens",
|
||
# Together/Fireworks-style: "Input length 131393 exceeds the maximum allowed input length of 131040
|
||
# tokens." No other pattern in this list matches that wording. (port of anomalyco/opencode#37848)
|
||
"maximum allowed input length",
|
||
)
|
||
|
||
# Last entry: OpenRouter 404 when no endpoint supports tool calling —
|
||
# model_not_found triggers fallback instead of burning retries (#58446).
|
||
_MODEL_NOT_FOUND_PATTERNS = (
|
||
"is not a valid model", "invalid model", "model not found", "model_not_found", "does not exist",
|
||
"no such model", "unknown model", "unsupported model", "no endpoints found that support tool use",
|
||
)
|
||
|
||
# Qwen/vLLM chat-template "No user query found". Shared by the invalid-body
|
||
# table (→ format_error) and the llama.cpp grammar guard so they cannot drift.
|
||
_NO_USER_QUERY_SIGNAL = "no user query found"
|
||
|
||
# Deterministic rejections of the *transcript* (e.g. a content-less assistant
|
||
# stub after a dead stream). NOT overflow — input may be tiny and compression
|
||
# cannot invent a missing turn — so fail fast as format_error.
|
||
_INVALID_MESSAGE_BODY_PATTERNS = (
|
||
"must have non-empty content", "messages must have non-empty", "invalid_request_body",
|
||
"text content blocks must be non-empty", "content field is required",
|
||
"messages: at least one message is required", _NO_USER_QUERY_SIGNAL,
|
||
)
|
||
|
||
# Proxy-side rejection of the model's own tool-call JSON (Ollama "invalid tool call arguments",
|
||
# OpenRouter-wrapped "function_call arguments"). Checked before the generic 400 validation and
|
||
# overflow heuristics: on a large session the bare message would otherwise read as overflow.
|
||
_MALFORMED_TOOL_ARGS_PATTERNS = (
|
||
"invalid tool call arguments", "invalid tool_call arguments", "invalid tool_calls arguments",
|
||
"invalid function call arguments", "invalid function_call arguments",
|
||
"tool call arguments are invalid", "tool_call arguments are invalid",
|
||
"function call arguments are invalid", "function_call arguments are invalid",
|
||
)
|
||
|
||
# Malformed request, identical on every retry. Some gateways (codex.nekos.me)
|
||
# return these as 5xx, so the 5xx path also checks them.
|
||
_REQUEST_VALIDATION_PATTERNS = (
|
||
"unknown parameter", "unsupported parameter", "unrecognized request argument",
|
||
"invalid_request_error", "unknown_parameter", "unsupported_parameter",
|
||
)
|
||
|
||
# Parameters Hermes sends on SOME routes only → hosts where that is deliberate.
|
||
# A rejection from any other host means the provider's gateway injected the
|
||
# field itself: a server-side flake, not our request shape. prompt_cache_retention
|
||
# is only sent for api.meta.ai / bedrock-mantle (agent/transports/codex.py).
|
||
_SERVER_INJECTED_PARAM_SENDERS: Dict[str, tuple] = {
|
||
"prompt_cache_retention": ("meta", "muse", "msl", "model-api", "bedrock", "mantle"),
|
||
}
|
||
_PARAM_REJECTION_WORDS = ("not supported", "unsupported", "unknown", "unrecognized")
|
||
|
||
# Anthropic thinking-block 400 wordings (see _provider_special_cases).
|
||
_THINKING_MUTATION_WORDS = ("signature", "cannot be modified", "must remain as they were")
|
||
|
||
# Local MoA streaming adapter-shape bugs (see _moa_special_cases).
|
||
_MOA_ADAPTER_SHAPE_BUGS = (
|
||
"'types.SimpleNamespace' object is not iterable", "'types.SimpleNamespace' object has no attribute 'index'",
|
||
)
|
||
|
||
# OpenRouter 404 when the account data policy excludes the only endpoint. Not
|
||
# model_not_found: the model exists, fallback can't help, body has the fix URL.
|
||
_PROVIDER_POLICY_BLOCKED_PATTERNS = (
|
||
"no endpoints available matching your guardrail", "no endpoints available matching your data policy",
|
||
"no endpoints found matching your data policy",
|
||
)
|
||
|
||
# Per-prompt safety-filter blocks: deterministic for the unchanged request, so
|
||
# fallback immediately. Each phrase is verbatim from one provider (Codex cyber
|
||
# flags #18028, OpenAI moderation, Anthropic safety, Azure token, MiniMax
|
||
# #32421) — never a generic word like "policy" that collides with billing/auth.
|
||
# "content_filter" deliberately excludes the space variant seen in echoed config.
|
||
_CONTENT_POLICY_BLOCKED_PATTERNS = (
|
||
"flagged for possible cybersecurity risk", "trusted access for cyber",
|
||
"violates our usage policies", "violates openai's usage policies", "your request was flagged by",
|
||
"prompt was flagged by our safety", "responses cannot be generated due to safety",
|
||
"content_filter", "responsibleaipolicyviolation", "new_sensitive",
|
||
)
|
||
|
||
# Auth patterns (non-status-code signals).
|
||
_AUTH_PATTERNS = (
|
||
"invalid api key", "invalid_api_key", "gateway_auth_failed", "authentication", "unauthorized",
|
||
"forbidden", "invalid token", "token expired", "token revoked", "access denied",
|
||
)
|
||
|
||
# Empty-response advisories (OpenRouter / nano-gpt). Checked before overflow
|
||
# because the text often mentions "max_tokens" (caused compression spirals).
|
||
_EMPTY_PROVIDER_RESPONSE_PATTERNS = (
|
||
"returned an empty response", "empty response despite retries", "provider returned an empty response",
|
||
"model returning empty responses", "empty response stream",
|
||
)
|
||
|
||
# Timeout wording from generic exception types the type heuristics would miss.
|
||
_TIMEOUT_MESSAGE_PATTERNS = (
|
||
"timed out", "turn timed out", "request timed out", "deadline exceeded", "operation timed out",
|
||
"upstream timed out",
|
||
)
|
||
|
||
# Connect/DNS failures from generic exception types with no status. EXCLUDES
|
||
# mid-stream disconnects (_SERVER_DISCONNECT_PATTERNS may route large sessions
|
||
# to compression; a never-established connection cannot be an overflow).
|
||
# Groups: TCP connect; DNS (Python/glibc/macOS/Node); undici bridge; Envoy.
|
||
_CONNECTION_MESSAGE_PATTERNS = (
|
||
"connection refused", "econnrefused", "no route to host", "network is unreachable", "network unreachable",
|
||
"name or service not known", "temporary failure in name resolution", "nodename nor servname provided",
|
||
"getaddrinfo failed", "getaddrinfo enotfound", "eai_again",
|
||
"fetch failed", "failed to fetch",
|
||
"upstream connect error",
|
||
)
|
||
|
||
# SSL names keep provider-wrapped SSL errors (chain lost) as transport, not
|
||
# unknown; OpenAI SDK errors are not subclasses of Python builtins.
|
||
_TRANSPORT_ERROR_TYPES = frozenset({
|
||
"ReadTimeout", "ConnectTimeout", "PoolTimeout", "ConnectError", "RemoteProtocolError",
|
||
"ConnectionError", "ConnectionResetError", "ConnectionAbortedError", "BrokenPipeError",
|
||
"TimeoutError", "ReadError", "ServerDisconnectedError",
|
||
"SSLError", "SSLZeroReturnError", "SSLWantReadError", "SSLWantWriteError", "SSLEOFError", "SSLSyscallError",
|
||
"APIConnectionError", "APITimeoutError",
|
||
})
|
||
|
||
# Ambiguous disconnects (no status): transient hiccup OR a gateway dropping an
|
||
# oversized request. A large session + one of these → context-overflow path.
|
||
_SERVER_DISCONNECT_PATTERNS = (
|
||
"server disconnected", "peer closed connection", "connection reset by peer", "connection was closed",
|
||
"network connection lost", "unexpected eof", "incomplete chunked read",
|
||
)
|
||
|
||
# Deterministic cert failures (proxy, missing CA, expired/self-signed) — fail
|
||
# fast. Checked BEFORE _SSL_TRANSIENT_PATTERNS: these also contain "[SSL:".
|
||
_SSL_CERT_VERIFY_PATTERNS = (
|
||
"certificate verify failed", "certificate_verify_failed", "unable to get local issuer certificate",
|
||
"self-signed certificate", "self signed certificate", "certificate has expired",
|
||
"hostname mismatch, certificate is not valid", "unable to verify the first certificate",
|
||
)
|
||
|
||
# Transient SSL alerts: retry but NOT compression (kept apart from disconnects).
|
||
# Both space and underscore forms because OpenSSL 3 changed token separators
|
||
# (SSLV3_ALERT_... → SSL/TLS_ALERT_...); "[ssl:" is the Python ssl prefix.
|
||
_SSL_TRANSIENT_PATTERNS = (
|
||
"bad record mac", "ssl alert", "tls alert", "ssl handshake failure", "tlsv1 alert", "sslv3 alert",
|
||
"bad_record_mac", "ssl_alert", "tls_alert", "tls_alert_internal_error", "[ssl:",
|
||
)
|
||
|
||
|
||
# ── Verdicts and rule tables ────────────────────────────────────────────
|
||
# A verdict is the ClassifiedError kwargs a stage decided on: ``reason`` plus
|
||
# hint overrides (unlisted hints keep dataclass defaults). Rule tables are
|
||
# ordered ``(patterns, verdict)`` pairs matched first-hit; ``verdict`` may be
|
||
# a callable of the error message.
|
||
|
||
Verdict = Dict[str, Any]
|
||
|
||
|
||
def _v(reason: FailoverReason, **hints: Any) -> Verdict:
|
||
return {"reason": reason, **hints}
|
||
|
||
|
||
_ROTATE_FALLBACK = {"should_rotate_credential": True, "should_fallback": True}
|
||
_ABORT_FALLBACK = {"retryable": False, "should_fallback": True}
|
||
_R = FailoverReason
|
||
|
||
_V_BILLING = _v(_R.billing, retryable=False, **_ROTATE_FALLBACK)
|
||
_V_RATE_LIMIT = _v(_R.rate_limit, **_ROTATE_FALLBACK)
|
||
_V_AUTH_ROTATE = _v(_R.auth, retryable=False, **_ROTATE_FALLBACK)
|
||
_V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK)
|
||
_V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK)
|
||
_V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK)
|
||
_V_FORMAT_ERROR = _v(_R.format_error, **_ABORT_FALLBACK)
|
||
# A different provider (direct instead of the aggregator; another host's TLS chain) can fix these.
|
||
_V_POLICY_BLOCKED = _v(_R.provider_policy_blocked, **_ABORT_FALLBACK)
|
||
_V_SSL_CERT = _v(_R.ssl_cert_verification, **_ABORT_FALLBACK)
|
||
_V_CONTEXT_OVERFLOW = _v(_R.context_overflow, should_compress=True)
|
||
_V_PAYLOAD_TOO_LARGE = _v(_R.payload_too_large, should_compress=True)
|
||
_V_OVERLOADED, _V_SERVER_ERROR, _V_TIMEOUT, _V_UNKNOWN = map(_v, (_R.overloaded, _R.server_error, _R.timeout, _R.unknown))
|
||
_V_IMAGE_TOO_LARGE, _V_IMAGE_CORRUPT = _v(_R.image_too_large), _v(_R.image_corrupt)
|
||
_V_MULTIMODAL, _V_INVALID_ENCRYPTED = _v(_R.multimodal_tool_content_unsupported), _v(_R.invalid_encrypted_content)
|
||
_V_REASONING_MANDATORY = _v(_R.reasoning_mandatory, should_compress=False, should_fallback=False)
|
||
# The MODEL emitted unparseable tool-call JSON and the proxy (Ollama, OpenRouter) rejected it: no
|
||
# other provider can fix that output, so falling back only replays the same broken turn 4-5 times
|
||
# (20-60s per occurrence, #12770). Abort this call; the loop's argument repair handles the retry.
|
||
_V_MALFORMED_TOOL_ARGS = _v(_R.format_error, retryable=False, should_fallback=False)
|
||
# A reasoning-mandatory route answering ``reasoning: {enabled: false}`` (Nous Portal + OpenRouter wording).
|
||
_REASONING_MANDATORY_PATTERN = "reasoning is mandatory"
|
||
|
||
# Reasoning wire-field token shared with the auxiliary retry rung
|
||
# (agent/auxiliary_client._is_reasoning_field_rejection, which cannot be imported
|
||
# here — it imports from this module). Standalone field name only: never a model-id
|
||
# segment ("kimi-k2-thinking") nor the adjective in "... with reasoning models" (#114460).
|
||
_REASONING_DISABLE_TOKEN = re.compile(
|
||
r"(?<![\w\-/])(?:reasoning_effort|thinking_config|thinking_budget|enable_thinking|thinkingconfig"
|
||
r"|thinkingbudget|reasoning|thinking|think)(?![\w\-/])(?!\s+models?\b)"
|
||
)
|
||
|
||
# Generic unsupported/unknown markers mirroring the auxiliary rung's parameter check.
|
||
_UNSUPPORTED_PARAM_MARKERS = (
|
||
"unsupported parameter", "unsupported_parameter", "not supported", "does not support",
|
||
"doesn't support", "is deprecated for this model",
|
||
"unknown parameter", "unrecognized request argument", "unrecognized parameter",
|
||
"invalid parameter", "extra inputs are not permitted",
|
||
)
|
||
|
||
|
||
def is_reasoning_disable_rejected(error_msg: str) -> bool:
|
||
"""True when a provider error rejects a reasoning wire control by name (#114460).
|
||
|
||
Covers the forward markers ("Unrecognized request argument supplied:
|
||
reasoning_effort") and the reversed word order some providers use
|
||
("reasoning_effort 'none' unsupported": field token plus standalone
|
||
"unsupported" within 32 chars). The main loop maps this to
|
||
reasoning_mandatory (drop the disable, retry); the auxiliary ladder maps it
|
||
to its strip-and-retry rung. Same wording class, same remedy.
|
||
"""
|
||
msg = (error_msg or "").lower()
|
||
if "reasoning" not in msg and "think" not in msg:
|
||
return False
|
||
if not any(name in msg and any(m in msg for m in _UNSUPPORTED_PARAM_MARKERS)
|
||
for name in ("reasoning", "think")):
|
||
token = _REASONING_DISABLE_TOKEN.search(msg)
|
||
if token is None or "unsupported" not in msg[token.end():token.end() + 32]:
|
||
return False
|
||
return _REASONING_DISABLE_TOKEN.search(msg) is not None
|
||
|
||
|
||
def _billing_hints(error_msg: str) -> Verdict:
|
||
"""Billing verdict carrying the #82154 ambiguity marker when applicable."""
|
||
ctx: Dict[str, Any] = {}
|
||
if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS):
|
||
ctx = {"billing_unverified": True, "possible_content_filter": True}
|
||
return {**_V_BILLING, "error_context": ctx}
|
||
|
||
|
||
def _first_match(error_msg: str, rules: Sequence[tuple[Sequence[str], Any]]) -> Optional[Verdict]:
|
||
"""Verdict of the first rule whose pattern list hits ``error_msg``."""
|
||
for patterns, verdict in rules:
|
||
if any(p in error_msg for p in patterns):
|
||
return verdict(error_msg) if callable(verdict) else verdict
|
||
return None
|
||
|
||
|
||
# Image/tool-content 400s, ordered: multimodal recovery ≠ image shrink; corrupt
|
||
# bytes need strip not shrink; image-shrink is cheaper than context compression.
|
||
_IMAGE_TOOL_RULES = (
|
||
(_MULTIMODAL_TOOL_CONTENT_PATTERNS, _V_MULTIMODAL), (_IMAGE_CORRUPT_PATTERNS, _V_IMAGE_CORRUPT),
|
||
(_IMAGE_TOO_LARGE_PATTERNS, _V_IMAGE_TOO_LARGE),
|
||
)
|
||
|
||
# Overflow signals arriving as 5xx (llama.cpp reports overflow as 500; busy /
|
||
# model-load OOM as 503). Empty-response advisories must not enter compression.
|
||
_OVERFLOW_AS_5XX_RULES = (
|
||
(_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR), (_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
|
||
(_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW),
|
||
)
|
||
|
||
# 404: Nous API surfaces credit depletion as a paid model vanishing from the
|
||
# Free Tier (billing, not missing model); policy block before model_not_found.
|
||
_404_RULES = (
|
||
(_BILLING_PATTERNS, _V_BILLING), (_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED),
|
||
(_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
)
|
||
|
||
# 400 tail after the deterministic request-shape checks. Some providers return
|
||
# model-not-found / rate-limit / billing as 400 instead of 404/429/402.
|
||
_400_TAIL_RULES = _OVERFLOW_AS_5XX_RULES + (
|
||
(_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
(_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_BILLING_PATTERNS, _billing_hints),
|
||
)
|
||
|
||
# Status-less message path, head (before usage-limit disambiguation).
|
||
_MESSAGE_HEAD_RULES = ((_MEMORY_CEILING_PATTERNS, _V_OVERLOADED),
|
||
(_PAYLOAD_TOO_LARGE_PATTERNS, _V_PAYLOAD_TOO_LARGE)) + _IMAGE_TOOL_RULES
|
||
|
||
# Status-less tail. Overload before rate_limit/billing so "overloaded" backs off
|
||
# instead of rotating; policy block before model_not_found; timeout/connection
|
||
# wording last, classified as transport (never compression).
|
||
_MESSAGE_TAIL_RULES = (
|
||
(_OVERLOADED_PATTERNS, _V_OVERLOADED), (_BILLING_PATTERNS, _billing_hints),
|
||
(_RATE_LIMIT_PATTERNS, _V_RATE_LIMIT), (_EMPTY_PROVIDER_RESPONSE_PATTERNS, _V_SERVER_ERROR),
|
||
(_CONTEXT_OVERFLOW_PATTERNS, _V_CONTEXT_OVERFLOW), (_AUTH_PATTERNS, _V_AUTH_ROTATE),
|
||
(_PROVIDER_POLICY_BLOCKED_PATTERNS, _V_POLICY_BLOCKED), (_MODEL_NOT_FOUND_PATTERNS, _V_MODEL_NOT_FOUND),
|
||
(_TIMEOUT_MESSAGE_PATTERNS, _V_TIMEOUT), (_CONNECTION_MESSAGE_PATTERNS, _V_TIMEOUT),
|
||
)
|
||
|
||
# Structured error code → verdict. The error-code rate_limit verdict rotates
|
||
# but does not set should_fallback (unlike the message/status paths).
|
||
_ERROR_CODE_VERDICTS: Dict[str, Verdict] = {
|
||
**dict.fromkeys(("resource_exhausted", "throttled", "rate_limit_exceeded"),
|
||
_v(_R.rate_limit, should_rotate_credential=True)),
|
||
**dict.fromkeys(_BILLING_ERROR_CODES, _V_BILLING),
|
||
**dict.fromkeys(("model_not_found", "model_not_available", "invalid_model"), _V_MODEL_NOT_FOUND),
|
||
**dict.fromkeys(("context_length_exceeded", "max_tokens_exceeded"), _V_CONTEXT_OVERFLOW),
|
||
**dict.fromkeys(_MEMORY_CEILING_ERROR_CODES, _V_OVERLOADED),
|
||
"invalid_encrypted_content": _V_INVALID_ENCRYPTED,
|
||
}
|
||
|
||
# Generic ``invalid_request_error`` is deliberately NOT a 400 validation
|
||
# signal — OpenAI stamps it on genuine overflow 400s too.
|
||
_400_VALIDATION_CODES = {"unknown_parameter", "unsupported_parameter"}
|
||
_5XX_VALIDATION_CODES = _400_VALIDATION_CODES | {"invalid_request_error"}
|
||
_400_VALIDATION_PATTERNS = tuple(p for p in _REQUEST_VALIDATION_PATTERNS if p != "invalid_request_error")
|
||
|
||
|
||
# ── Classification pipeline ─────────────────────────────────────────────
|
||
|
||
@dataclass
|
||
class _Ctx:
|
||
"""Everything the classifier stages need about one failed call."""
|
||
|
||
error: Exception
|
||
status_code: Optional[int]
|
||
body: dict
|
||
msg: str # lowercased str(error) + body message(s)
|
||
provider: str # as passed by the caller
|
||
model: str
|
||
approx_tokens: int
|
||
context_length: int
|
||
num_messages: int
|
||
base_url: str = "" # the route the call went to; "" when the caller did not say
|
||
anonymous: bool = False
|
||
|
||
def __post_init__(self) -> None:
|
||
self.error_type = type(self.error).__name__
|
||
self.error_code = _extract_error_code(self.body)
|
||
self.code = self.error_code.lower()
|
||
self.headers = _from_cause_chain(self.error, _headers_of, {})
|
||
self.provider_slug = (self.provider or "").strip().lower()
|
||
self.model_slug = (self.model or "").strip().lower()
|
||
|
||
def large_session(self, frac: float, tokens: int, messages: int) -> bool:
|
||
"""Absolute thresholds only proxy for smaller context windows."""
|
||
return self.approx_tokens > self.context_length * frac or (
|
||
self.context_length <= 256000 and (self.approx_tokens > tokens or self.num_messages > messages)
|
||
)
|
||
|
||
|
||
def _plugin_verdict(c: _Ctx) -> Optional[Verdict]:
|
||
"""First valid plugin classification (runs before the built-in pipeline so a
|
||
provider plugin can add or correct verdicts). invoke_hook isolates callback
|
||
failures; this guard only covers import/dispatch failure."""
|
||
try:
|
||
from hermes_cli.plugins import get_plugin_error_classification
|
||
verdict = get_plugin_error_classification(
|
||
provider=c.provider, model=c.model, status_code=c.status_code, error_type=c.error_type,
|
||
error_code=c.error_code, error_message=c.msg, error_body=c.body, error=c.error,
|
||
approx_tokens=c.approx_tokens, context_length=c.context_length, num_messages=c.num_messages,
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("Plugin error classification unavailable: %s", exc)
|
||
return None
|
||
if verdict is not None:
|
||
logger.info("API error classified by plugin hook: %s (provider=%s, status=%s)",
|
||
verdict["reason"].value, c.provider, c.status_code)
|
||
return verdict
|
||
|
||
|
||
# A welcome-host 403 that spells one of these out is a safety block or a billing wall, not the
|
||
# tier refusing. The free-tier refusal phrases are left OUT: on the free route they mean exactly
|
||
# "the tier refused", and an anonymous session has no credits to check.
|
||
_WELCOME_403_NAMED_PATTERNS = _CONTENT_POLICY_BLOCKED_PATTERNS + tuple(
|
||
p for p in _BILLING_PATTERNS if p not in _FREE_TIER_REFUSAL_PATTERNS)
|
||
|
||
|
||
def _nous_welcome_tier(c: _Ctx) -> Optional[Verdict]:
|
||
"""The Nous inference gateway's welcome-tier (free tier) refusals, read from the structured body.
|
||
|
||
A 429 carrying a fairshare ``reason`` is either a tier gate (``model_not_free`` /
|
||
``feature_not_free``: the model or feature is never served on the free tier, so retrying is
|
||
pointless — abort this route and fall back) or capacity (``at_capacity`` / ``admission_closed``
|
||
/ ``rate_limited``: honour ``retry_after``, never rotate the free tier's only credential). A
|
||
400/403 whose message names the wrong host or a dark tier is deterministic for the request.
|
||
The parsed refusal rides ``error_context`` so the terminal copy can say what happened.
|
||
"""
|
||
from hermes_cli.anon_auth import (
|
||
WELCOME_TIER_GATE_REASONS, parse_welcome_refusal, welcome_route_refusal)
|
||
status = c.status_code
|
||
if not c.anonymous:
|
||
# A named credential's fairshare 429 is an ordinary rate limit, whatever its body says. The
|
||
# one welcome refusal it does receive is the gateway's mirror 400 on the welcome host; its
|
||
# reconnect copy stands, only the sign-in card is withheld (``_welcome_surface_kind``).
|
||
if c.provider == "nous" and status == 400 and welcome_route_refusal(status, c.msg) == "named_on_welcome_host":
|
||
return _v(_R.format_error, retryable=False, should_fallback=True,
|
||
error_context={"welcome_route": "named_on_welcome_host"})
|
||
return None
|
||
if status == 429:
|
||
refusal = parse_welcome_refusal(c.body)
|
||
if refusal is None:
|
||
return None
|
||
ctx = {"welcome_refusal": refusal}
|
||
if refusal["reason"] in WELCOME_TIER_GATE_REASONS:
|
||
return _v(_R.model_not_found, retryable=False, should_fallback=True, error_context=ctx)
|
||
if refusal["retry_after"] > 0:
|
||
ctx["reset_at"] = time.time() + refusal["retry_after"]
|
||
return _v(_R.rate_limit, should_fallback=True, error_context=ctx)
|
||
# The route-keyed dark-tier 403 applies only to a 403 that says nothing else: a safety refusal
|
||
# or a billing wall on the welcome host keeps its own classification (and its own recovery).
|
||
plain_403 = not any(p in c.msg for p in _WELCOME_403_NAMED_PATTERNS)
|
||
kind = welcome_route_refusal(status, c.msg, c.base_url if plain_403 else None)
|
||
if kind is None:
|
||
return None
|
||
ctx = {"welcome_route": kind}
|
||
if status == 403:
|
||
return _v(_R.auth_permanent, retryable=False, should_fallback=True, error_context=ctx)
|
||
return _v(_R.format_error, retryable=False, should_fallback=True, error_context=ctx)
|
||
|
||
|
||
def _provider_special_cases(c: _Ctx) -> Optional[Verdict]:
|
||
"""Highest-priority provider-specific shapes that a status code would misroute."""
|
||
msg, status = c.msg, c.status_code
|
||
welcome = _nous_welcome_tier(c)
|
||
if welcome is not None:
|
||
return welcome
|
||
# Safety refusal before status classification so a 400 block isn't downgraded
|
||
# to format_error and a status-less block isn't left retryable (#18028).
|
||
if any(p in msg for p in _CONTENT_POLICY_BLOCKED_PATTERNS):
|
||
return _V_CONTENT_BLOCKED
|
||
# ChatGPT Codex masks a rejected encrypted-reasoning replay behind the same bare
|
||
# ``invalid_prompt: Request blocked.`` it uses for real blocks (#92353). Exact envelope
|
||
# + provider only. The verdict keeps format_error's abort-and-fallback hints; the one
|
||
# extra thing it buys is turn_recovery's replay strip, which still requires cached
|
||
# ``codex_reasoning_items`` — a genuine block with nothing to strip behaves as before.
|
||
if _is_codex_masked_replay_rejection(c):
|
||
return _v(_R.invalid_encrypted_content, **_ABORT_FALLBACK)
|
||
# Anthropic thinking-block 400s (signature mismatch after transcript
|
||
# mutation). Not gated on provider — OpenRouter proxies Anthropic errors.
|
||
if status == 400 and "thinking" in msg and any(p in msg for p in _THINKING_MUTATION_WORDS):
|
||
return _v(_R.thinking_signature)
|
||
# Anthropic long-context tier gate (429 "extra usage" + "long context").
|
||
if status == 429 and "extra usage" in msg and "long context" in msg:
|
||
return _v(_R.long_context_tier, should_compress=True)
|
||
# Anthropic OAuth rejects the 1M beta header; run_agent retries without it.
|
||
if status == 400 and "long context beta" in msg and "not yet available" in msg:
|
||
return _v(_R.oauth_long_context_beta_forbidden)
|
||
# llama.cpp grammar rejects regex ``pattern``/``format`` in tool schemas; the
|
||
# retry loop strips them. Exclude the Qwen/vLLM "No user query found" error
|
||
# local engines wrap as "Unable to generate parser for this template" —
|
||
# that is a poisoned transcript (→ format_error), not a grammar problem.
|
||
grammar_hit = "error parsing grammar" in msg or "json-schema-to-grammar" in msg or (
|
||
"unable to generate parser" in msg and "template" in msg
|
||
)
|
||
if status == 400 and grammar_hit and _NO_USER_QUERY_SIGNAL not in msg:
|
||
return _v(_R.llama_cpp_grammar_pattern)
|
||
# xAI Grok entitlement as an SSE ``type=error`` frame: no status, matches no
|
||
# pattern list, would otherwise burn max_retries as ``unknown``.
|
||
if "do not have an active grok subscription" in msg or ("out of available resources" in msg and "grok" in msg):
|
||
return _V_AUTH_FALLBACK
|
||
return None
|
||
|
||
|
||
def _moa_special_cases(c: _Ctx) -> Optional[Verdict]:
|
||
# Local MoA streaming adapter-shape bugs are not a provider outage; falling
|
||
# back would silently replace the MoA route with a single model (#55933).
|
||
if c.provider_slug == "moa" and any(s in str(c.error) for s in _MOA_ADAPTER_SHAPE_BUGS):
|
||
return _v(_R.format_error, retryable=False)
|
||
# Persisted MoA preset name that was renamed/deleted — deterministic config error.
|
||
from agent.errors import MoAPresetNotFoundError
|
||
return _v(_R.model_not_found, retryable=False) if isinstance(c.error, MoAPresetNotFoundError) else None
|
||
|
||
|
||
def _by_error_code(c: _Ctx) -> Optional[Verdict]:
|
||
"""Structured error codes from the response body."""
|
||
# Request-validation failure as plain-text ``event: error`` SSE data behind
|
||
# HTTP 200: retrying cannot succeed, a configured fallback still may.
|
||
if c.code == PROVIDER_STREAM_NON_JSON_ERROR_CODE and "request validation failed:" in c.msg:
|
||
return _V_FORMAT_ERROR
|
||
return _ERROR_CODE_VERDICTS.get(c.code)
|
||
|
||
|
||
def _by_message(c: _Ctx) -> Optional[Verdict]:
|
||
"""Message patterns when no status code settled it; status-less usage
|
||
limits get the same disambiguation as 402."""
|
||
head = _first_match(c.msg, _MESSAGE_HEAD_RULES)
|
||
if head is not None:
|
||
return head
|
||
usage_limit = any(p in c.msg for p in _USAGE_LIMIT_PATTERNS)
|
||
return _classify_402(c.msg, dict) if usage_limit else _first_match(c.msg, _MESSAGE_TAIL_RULES)
|
||
|
||
|
||
def _by_transport(c: _Ctx) -> Optional[Verdict]:
|
||
"""SSL, disconnect, circuit-breaker and transport-type heuristics, in that order."""
|
||
msg = c.msg
|
||
# Cert failure → fail fast (checked first: also contains "[ssl:"); transient
|
||
# alert → retry, before disconnects so a flaky handshake never compresses.
|
||
ssl = _first_match(msg, ((_SSL_CERT_VERIFY_PATTERNS, _V_SSL_CERT), (_SSL_TRANSIENT_PATTERNS, _V_TIMEOUT)))
|
||
if ssl is not None:
|
||
return ssl
|
||
# Disconnect + large session → probable overflow rejection, not a hiccup.
|
||
if any(p in msg for p in _SERVER_DISCONNECT_PATTERNS) and not c.status_code:
|
||
# Reasoning models: far more likely the gateway idle-killed a long
|
||
# thinking stream — never compress on a phantom overflow (#52310).
|
||
# Reasoning-model override: a transport disconnect on a reasoning model is much more likely the
|
||
# upstream proxy idle-killing a long thinking stream than a true context overflow — even on large
|
||
# sessions. The default disconnect+large-session routing below would otherwise send the user into
|
||
# the compression branch (should_compress=True) and silently delete conversation history on a
|
||
# phantom context-length error. Reasoning models have multi-minute thinking phases that routinely
|
||
# exceed the cloud gateway's idle window (NVIDIA NIM ~120s — first-party repro at
|
||
# NVIDIA/NemoClaw#4846; OpenAI worker / Anthropic stream-idle similar). The per-reasoning-model
|
||
# stale-timeout floor in agent/reasoning_timeouts.py raises the stale-detector threshold to tolerate
|
||
# long thinking, so a true transport-layer failure here is recoverable via the retry path — not via
|
||
# context compression. Reclassify as timeout. (Part 1 of Fixes #52310.)
|
||
from agent.reasoning_timeouts import get_reasoning_stale_timeout_floor
|
||
if get_reasoning_stale_timeout_floor(c.model) is not None:
|
||
return _V_TIMEOUT
|
||
return _V_CONTEXT_OVERFLOW if c.large_session(0.6, 120000, 200) else _V_TIMEOUT
|
||
# Stale-call circuit breaker (_check_stale_giveup RuntimeError before any
|
||
# network call): as ``unknown`` it would burn every retry instantly.
|
||
if c.error_type == "RuntimeError" and "consecutive stale attempts" in msg and "aborting this call" in msg:
|
||
return _v(_R.timeout, **_ABORT_FALLBACK)
|
||
transport = c.error_type in _TRANSPORT_ERROR_TYPES or isinstance(c.error, (TimeoutError, ConnectionError, OSError))
|
||
return _V_TIMEOUT if transport else None
|
||
|
||
|
||
def _by_status(c: _Ctx) -> Optional[Verdict]:
|
||
"""HTTP status code with message-aware refinement (unlisted 4xx/5xx → generic)."""
|
||
status = c.status_code
|
||
if status is None:
|
||
return None
|
||
default = _V_FORMAT_ERROR if 400 <= status < 500 else _V_SERVER_ERROR if 500 <= status < 600 else None
|
||
return _STATUS_HANDLERS[status](c) if status in _STATUS_HANDLERS else default
|
||
|
||
|
||
# Stage order: plugin hooks → provider-specific special cases → HTTP status →
|
||
# MoA shapes → structured error code → message patterns → SSL → disconnect +
|
||
# large session → transport types → unknown (retryable with backoff).
|
||
_STAGES: Sequence[Callable[[_Ctx], Optional[Verdict]]] = (
|
||
_plugin_verdict, _provider_special_cases, _by_status, _moa_special_cases,
|
||
_by_error_code, _by_message, _by_transport,
|
||
)
|
||
|
||
|
||
def classify_api_error(
|
||
error: Exception, *, provider: str = "", model: str = "",
|
||
approx_tokens: int = 0, context_length: int = 200000, num_messages: int = 0,
|
||
base_url: str = "",
|
||
api_key: Any = None,
|
||
) -> ClassifiedError:
|
||
"""Classify an API error into a structured recovery recommendation (see ``_STAGES``).
|
||
|
||
``base_url`` (optional) is the route the call went to; the Nous welcome tier keys its
|
||
dark-tier 403 on it because that refusal carries no distinguishing message.
|
||
``api_key`` identifies an anonymous request; a host or fairshare reason alone does not.
|
||
The credential is never included in the returned context."""
|
||
from hermes_cli.anon_auth import is_anonymous_request
|
||
status_code = _extract_status_code(error)
|
||
# Copilot/GitHub Models RateLimitError may not set .status_code; force 429.
|
||
if status_code is None and type(error).__name__ == "RateLimitError":
|
||
status_code = 429
|
||
body = _extract_error_body(error)
|
||
c = _Ctx(
|
||
error, status_code, body, _build_error_msg(error, body), provider, model,
|
||
approx_tokens, context_length, num_messages, str(base_url or ""),
|
||
anonymous=is_anonymous_request(provider, api_key),
|
||
)
|
||
verdict = next((v for v in (stage(c) for stage in _STAGES) if v is not None), _V_UNKNOWN)
|
||
base = {"status_code": status_code, "provider": provider, "model": model, "message": _extract_message(error, body)}
|
||
return ClassifiedError(**{**base, **verdict})
|
||
|
||
|
||
# ── Status code handlers ────────────────────────────────────────────────
|
||
|
||
def _status_403(c: _Ctx) -> Verdict:
|
||
# OpenRouter 403 "key limit exceeded" and similar plan/credit exhaustion are billing.
|
||
xai_spend = c.provider_slug == "xai-oauth" and c.code == _XAI_SPENDING_LIMIT_ERROR_CODE
|
||
billing = xai_spend or any(p in c.msg for p in ("key limit exceeded", "spending limit") + _BILLING_PATTERNS)
|
||
return _V_BILLING if billing else _V_AUTH_FALLBACK
|
||
|
||
|
||
def _status_404(c: _Ctx) -> Verdict:
|
||
verdict = _first_match(c.msg, _404_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Bare id the catalogue only knows prefixed → malformed id (NVIDIA NIM "404
|
||
# page not found", #78796). A generic 404 (wrong path, proxy glitch) stays
|
||
# unknown so the real error surfaces instead of a silent misreported fallback.
|
||
return _V_MODEL_NOT_FOUND if _model_id_missing_known_prefix(c.model_slug, c.provider_slug) else _V_UNKNOWN
|
||
|
||
|
||
def _status_429(c: _Ctx) -> Verdict:
|
||
# A structured billing code is decisive: LiteLLM stamps
|
||
# ``terminal_quota_exhausted`` (a hard cap, not throttling) on 429s, and
|
||
# this handler always returns, so _by_error_code never sees the code.
|
||
if c.code in _BILLING_ERROR_CODES:
|
||
return _V_BILLING
|
||
# Z.AI/Zhipu reuse 429 for server-wide overload: back off on the same
|
||
# key instead of burning the pool (#14038).
|
||
if any(p in c.msg for p in _OVERLOADED_PATTERNS):
|
||
return _V_OVERLOADED
|
||
# OpenRouter-wrapped upstream 429: the key is healthy — fall back, don't bench.
|
||
if _is_openrouter_upstream_error(c.body, c.provider_slug):
|
||
upstream = _extract_upstream_provider_name(c.body)
|
||
ctx = {"upstream_provider": upstream} if upstream else {}
|
||
return _v(_R.upstream_rate_limit, should_fallback=True, error_context=ctx)
|
||
# Quota walls as 429 (Anthropic ``usage_limit_reached``, "quota", billing
|
||
# phrases) are billing ONLY when the body is not itself a rate-limit phrase
|
||
# ("Rate limit exceeded" contains "limit exceeded") and carries no reset/
|
||
# retry signal (#93419, #39441).
|
||
quota_wall = c.code == "usage_limit_reached" or any(
|
||
p in c.msg for p in ("usage_limit_reached",) + _USAGE_LIMIT_PATTERNS + _BILLING_PATTERNS
|
||
)
|
||
explicit_rate_limit = any(p in c.msg for p in _RATE_LIMIT_PATTERNS)
|
||
if quota_wall and not explicit_rate_limit and not _has_usage_limit_transient_signal(c.msg, c.body, c.headers):
|
||
return _V_BILLING
|
||
return _V_RATE_LIMIT
|
||
|
||
|
||
def _status_5xx(c: _Ctx) -> Verdict:
|
||
# Request-validation errors as 5xx (codex.nekos.me) fail fast instead of
|
||
# retry-flooding — unless the parameter was injected server-side.
|
||
validation = any(p in c.msg for p in _REQUEST_VALIDATION_PATTERNS) or c.code in _5XX_VALIDATION_CODES
|
||
if validation and not _is_server_injected_param_rejection(c.msg, c.provider_slug):
|
||
return _V_FORMAT_ERROR
|
||
return _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_SERVER_ERROR
|
||
|
||
|
||
def _classify_402(error_msg: str, result_fn: Callable[..., Any]) -> Any:
|
||
"""Disambiguate 402: "usage limit, try again in 5 minutes" is a periodic quota, not billing."""
|
||
transient = any(p in error_msg for p in _USAGE_LIMIT_PATTERNS) and any(
|
||
p in error_msg for p in _USAGE_LIMIT_TRANSIENT_SIGNALS
|
||
)
|
||
return result_fn(**(_V_RATE_LIMIT if transient else _V_BILLING))
|
||
|
||
|
||
def _has_large_inline_image(content: Any) -> bool:
|
||
"""True when a rejected ``content`` list carries a ``data:image/`` part the shrink pass would rewrite
|
||
(over ``conversation_compression._IMAGE_SHRINK_TARGET_BYTES``; below it a shrink retry is a no-op)."""
|
||
from agent.conversation_compression import _IMAGE_SHRINK_TARGET_BYTES
|
||
|
||
for part in content if isinstance(content, list) else ():
|
||
image = part.get("image_url") if isinstance(part, dict) else None
|
||
url = image.get("url") if isinstance(image, dict) else image
|
||
if isinstance(url, str) and url.startswith("data:image/") and len(url) > _IMAGE_SHRINK_TARGET_BYTES:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _oversized_message_content_rejection(body: Any) -> bool:
|
||
"""400 rejecting a *message* ``content`` field whose rejected value carries a large inline image.
|
||
|
||
Nebius Token Factory caps a single image at 10 MiB and reports the violation through the field that
|
||
failed to coerce — pydantic ``{"type": "string_type", "loc": ["body","messages",N,"content","str"],
|
||
"msg": "Input should be a valid string", "input": [...]}`` — naming no size vocabulary, so the
|
||
keyword multimodal *tool*-content rule (#104731) claimed it and spent its retry stripping tool images
|
||
that were never there (#112473). The same list-shaped content with a small image succeeds, so the
|
||
image bytes are the trigger. Tool-scoped locs (``messages.N.tool.content.str``) stay with #104731.
|
||
"""
|
||
details = body.get("detail") if isinstance(body, dict) else None
|
||
for detail in details if isinstance(details, list) else ():
|
||
loc = detail.get("loc") if isinstance(detail, dict) else None
|
||
if detail.get("type") != "string_type" or not isinstance(loc, list) or len(loc) < 2:
|
||
continue
|
||
parts = [str(x).lower() for x in loc]
|
||
if parts[:2] == ["body", "messages"] and parts[-2:] == ["content", "str"] and not any(
|
||
x.startswith("tool") for x in parts
|
||
) and _has_large_inline_image(detail.get("input")):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _classify_400(c: _Ctx) -> Verdict:
|
||
"""400 Bad Request — image/tool shapes, request-shape rejections, overflow, or generic."""
|
||
msg, code = c.msg, c.code
|
||
# A size cap reported *through* a message content field must beat the keyword
|
||
# multimodal rule, which would otherwise claim "input should be a valid string".
|
||
if _oversized_message_content_rejection(c.body):
|
||
return _V_IMAGE_TOO_LARGE
|
||
verdict = _first_match(msg, _IMAGE_TOOL_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Invalid encrypted reasoning replay blob (OpenAI Responses); before
|
||
# overflow because "encrypted content … could not be verified" trips it.
|
||
if code == "invalid_encrypted_content" or "invalid_encrypted_content" in msg or (
|
||
"encrypted content for item" in msg and "could not be verified" in msg
|
||
) or "could not decrypt the provided encrypted_content" in msg or (
|
||
# Custom Responses endpoints wrap a replay rejection in a generic bad_request (#95834).
|
||
"encrypted content could not be decrypted or parsed" in msg
|
||
) or (
|
||
# OpenCode Zen wraps this OpenAI replay rejection in ``invalid_request_error`` (#111309).
|
||
"encrypted_content" in msg and "was not issued to this caller" in msg
|
||
) or (
|
||
# Azure Foundry (gpt-6-astra) rejects replayed reasoning from several prior responses this way (#105369).
|
||
"conflicting authenticated continuation identities" in msg
|
||
):
|
||
return _V_INVALID_ENCRYPTED
|
||
# Reasoning-mandatory route rejecting a disable (GLM-5.3 on Nous Portal / OpenRouter). Deterministic
|
||
# for the request shape, but the only bad field is ``reasoning: {enabled: false}`` — the loop drops
|
||
# the disable and retries once. Must precede request-validation, which would abort as format_error.
|
||
if _REASONING_MANDATORY_PATTERN in msg or is_reasoning_disable_rejected(msg):
|
||
return _V_REASONING_MANDATORY
|
||
# 400 blaming a field this route never sent (Codex OAuth injects then rejects
|
||
# prompt_cache_retention ~20% of the time): transient, retry identical request.
|
||
if _is_server_injected_param_rejection(msg, c.provider_slug):
|
||
return _V_SERVER_ERROR
|
||
if any(p in msg for p in _MALFORMED_TOOL_ARGS_PATTERNS):
|
||
return _V_MALFORMED_TOOL_ARGS
|
||
# Before overflow: GPT-5's "Unsupported parameter: 'max_tokens'" contains it.
|
||
if any(p in msg for p in _400_VALIDATION_PATTERNS) or code in _400_VALIDATION_CODES:
|
||
return _V_FORMAT_ERROR
|
||
# Malformed message array before overflow: input can be tiny and compression
|
||
# cannot fix it. litellm/Bedrock proxies use errorCode=INVALID_REQUEST_BODY.
|
||
if any(p in msg for p in _INVALID_MESSAGE_BODY_PATTERNS) or code == "invalid_request_body":
|
||
logger.warning(
|
||
"Malformed message array 400 (invalid request body) classified as format_error, NOT context "
|
||
"overflow — failing fast + falling back instead of entering the compression loop. This usually "
|
||
"means an empty-content assistant stub is in the transcript; num_messages=%s approx_tokens=%s. "
|
||
"error=%.200s", c.num_messages, c.approx_tokens, msg,
|
||
)
|
||
return _V_FORMAT_ERROR
|
||
# Memory ceiling by code: _by_status runs before _by_error_code, so a
|
||
# 400 whose wording a proxy stripped would fall through to format_error.
|
||
if code in _MEMORY_CEILING_ERROR_CODES:
|
||
return _V_OVERLOADED
|
||
verdict = _first_match(msg, _400_TAIL_RULES)
|
||
if verdict is not None:
|
||
return verdict
|
||
# Generic 400 + large session → probable overflow (Anthropic can return a
|
||
# bare "Error"); proxy shapes are read so a descriptive rejection isn't "bare".
|
||
body_msg = next((m for m in (str(x or "").strip().lower() for x in _body_message_candidates(c.body)) if m), "")
|
||
is_generic = len(body_msg) < 30 or body_msg in {"error", ""}
|
||
if is_generic and c.large_session(0.4, 80000, 80):
|
||
return _V_CONTEXT_OVERFLOW
|
||
return _V_FORMAT_ERROR
|
||
|
||
|
||
def _classify_image_tool_422(c: _Ctx) -> Verdict:
|
||
"""422: pydantic relays report the same content-field shapes as 400 (#104731, #112473)."""
|
||
if _oversized_message_content_rejection(c.body):
|
||
return _V_IMAGE_TOO_LARGE
|
||
return _first_match(c.msg, _IMAGE_TOOL_RULES) or _V_FORMAT_ERROR
|
||
|
||
|
||
# 401 not retryable on its own: rotation/refresh run before the retryability
|
||
# check, then the client-error abort path (fallback first) is correct. 408 is
|
||
# retry-safe (RFC 9110 §15.5.9; proxies emit it when generation outruns the
|
||
# read window). Unlisted 4xx → format_error, 5xx → server_error.
|
||
_STATUS_HANDLERS: Dict[int, Callable[[_Ctx], Verdict]] = {
|
||
400: _classify_400, 401: lambda c: _V_AUTH_ROTATE, 402: lambda c: _classify_402(c.msg, dict),
|
||
403: _status_403, 404: _status_404, 408: lambda c: _V_TIMEOUT, 413: lambda c: _V_PAYLOAD_TOO_LARGE,
|
||
422: lambda c: _classify_image_tool_422(c),
|
||
429: _status_429, 500: _status_5xx, 502: _status_5xx,
|
||
503: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
|
||
529: lambda c: _first_match(c.msg, _OVERFLOW_AS_5XX_RULES) or _V_OVERLOADED,
|
||
}
|
||
|
||
|
||
# ── Helpers ─────────────────────────────────────────────────────────────
|
||
|
||
_RESET_FIELDS = ("resets_in_seconds", "resets_at", "reset_at", "retry_after")
|
||
_RESET_HEADERS = ("retry-after", "Retry-After", "x-ratelimit-reset", "X-RateLimit-Reset")
|
||
|
||
|
||
def _has_usage_limit_transient_signal(error_msg: str, body: dict, response_headers) -> bool:
|
||
"""Whether a usage-limit response identifies a reset window (message, body fields, or headers)."""
|
||
if any(pattern in error_msg for pattern in _USAGE_LIMIT_TRANSIENT_SIGNALS):
|
||
return True
|
||
payloads = [p for p in (body, _error_obj(body)) if isinstance(p, dict)]
|
||
if any(payload.get(f) not in (None, "") for payload in payloads for f in _RESET_FIELDS):
|
||
return True
|
||
if response_headers and hasattr(response_headers, "get"):
|
||
return any(response_headers.get(h) not in (None, "") for h in _RESET_HEADERS)
|
||
return False
|
||
|
||
|
||
def _model_id_missing_known_prefix(model: str, provider: str) -> bool:
|
||
"""True when a bare model id is only known to the provider as ``vendor/id``.
|
||
|
||
Never guesses: an id absent from the curated catalogue returns False so real
|
||
endpoint problems keep their retryable ``unknown`` classification.
|
||
"""
|
||
name = (model or "").strip()
|
||
if not name or "/" in name:
|
||
return False
|
||
try:
|
||
from hermes_cli.model_normalize import suggest_prefixed_model_id
|
||
return bool(suggest_prefixed_model_id((provider or "").strip(), name))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _is_server_injected_param_rejection(error_msg: str, provider: str) -> bool:
|
||
"""True when a 400 blames a one-route-only parameter this route never sends.
|
||
|
||
Conservative: known parameters only, and only when ``provider`` is not a
|
||
sender, so a genuine bad parameter (``max_tokens`` on GPT-5) stays format_error.
|
||
"""
|
||
provider_slug = (provider or "").strip().lower()
|
||
for param, senders in _SERVER_INJECTED_PARAM_SENDERS.items():
|
||
if error_msg and param in error_msg and any(w in error_msg for w in _PARAM_REJECTION_WORDS):
|
||
return not any(sender in provider_slug for sender in senders)
|
||
return False
|
||
|
||
|
||
_CODEX_MASKED_REPLAY_MESSAGE = "request blocked."
|
||
|
||
|
||
def _is_codex_masked_replay_rejection(c: "_Ctx") -> bool:
|
||
"""HTTP 400 / status-less ``{code: invalid_prompt, message: "Request blocked."}`` from
|
||
``openai-codex`` — as an SDK error body, a Responses ``error`` SSE frame, or the
|
||
``response.failed`` text ``"invalid_prompt: Request blocked."``."""
|
||
if c.provider_slug != "openai-codex" or c.status_code not in (None, 400):
|
||
return False
|
||
# The OpenAI SDK unwraps ``body["error"]`` on status errors; stream frames keep the envelope.
|
||
body_msg = next((str(m).strip().lower() for m in _body_message_candidates(c.body or {}) if m), "")
|
||
return (c.code == "invalid_prompt" and body_msg == _CODEX_MASKED_REPLAY_MESSAGE) or (
|
||
c.msg.strip() == f"invalid_prompt: {_CODEX_MASKED_REPLAY_MESSAGE}"
|
||
)
|
||
|
||
|
||
def _error_obj(body: Any) -> dict:
|
||
"""``body["error"]`` when it is a dict, else ``{}``."""
|
||
err = body.get("error") if isinstance(body, dict) else None
|
||
return err if isinstance(err, dict) else {}
|
||
|
||
|
||
def _json_dict(text: Any) -> Optional[dict]:
|
||
"""Parse a JSON object string; None for non-strings, blanks, invalid JSON or non-objects."""
|
||
if not (isinstance(text, str) and text.strip()):
|
||
return None
|
||
try:
|
||
inner = json.loads(text)
|
||
except (json.JSONDecodeError, TypeError):
|
||
return None
|
||
return inner if isinstance(inner, dict) else None
|
||
|
||
|
||
def _openrouter_wrapped_message(err_obj: dict) -> str:
|
||
"""Lowercased inner message from OpenRouter's ``error.metadata.raw`` JSON wrapper."""
|
||
metadata = err_obj.get("metadata", {})
|
||
inner = _json_dict(metadata.get("raw")) if isinstance(metadata, dict) else None
|
||
return str(_error_obj(inner).get("message") or "").lower() if inner else ""
|
||
|
||
|
||
def _build_error_msg(error: Exception, body: Any) -> str:
|
||
"""Lowercased str(error) + body message + OpenRouter-wrapped upstream message
|
||
(OpenAI SDK's APIStatusError.__str__ omits the body, so it is appended)."""
|
||
raw_msg = str(error).lower()
|
||
body_msg = metadata_msg = ""
|
||
if isinstance(body, dict):
|
||
err_obj = _error_obj(body)
|
||
body_msg = str(err_obj.get("message") or "").lower() or str(body.get("message") or "").lower()
|
||
metadata_msg = _openrouter_wrapped_message(err_obj) if err_obj else ""
|
||
parts = [raw_msg]
|
||
if body_msg and body_msg not in raw_msg:
|
||
parts.append(body_msg)
|
||
if metadata_msg and metadata_msg not in raw_msg and metadata_msg not in body_msg:
|
||
parts.append(metadata_msg)
|
||
return " ".join(parts)
|
||
|
||
|
||
def _body_message_candidates(body: dict) -> Iterator[Any]:
|
||
"""Body message fields in priority order (OpenAI, flat, litellm/Bedrock proxy shapes)."""
|
||
yield _error_obj(body).get("message")
|
||
yield body.get("message")
|
||
yield body.get("errorMessage")
|
||
args = body.get("errorArgs")
|
||
yield args.get("reason") if isinstance(args, dict) else None
|
||
|
||
|
||
def _from_cause_chain(error: Exception, pick: Callable[[Any], Any], default: Any) -> Any:
|
||
"""First non-None ``pick(exc)`` over the error and its __cause__/__context__ chain (max 5 deep)."""
|
||
current = error
|
||
for _ in range(5):
|
||
found = pick(current)
|
||
if found is not None:
|
||
return found
|
||
cause = getattr(current, "__cause__", None) or getattr(current, "__context__", None)
|
||
if cause is None or cause is current:
|
||
break
|
||
current = cause
|
||
return default
|
||
|
||
|
||
def _status_of(exc: Any) -> Optional[int]:
|
||
code = getattr(exc, "status_code", None)
|
||
if isinstance(code, int):
|
||
return code
|
||
code = getattr(exc, "status", None) # some SDKs use .status
|
||
return code if isinstance(code, int) and 100 <= code < 600 else None
|
||
|
||
|
||
def _body_of(exc: Any) -> Optional[dict]:
|
||
body = getattr(exc, "body", None)
|
||
if isinstance(body, dict):
|
||
return body
|
||
response = getattr(exc, "response", None)
|
||
try:
|
||
json_body = response.json() if response is not None else None
|
||
except Exception:
|
||
return None
|
||
return json_body if isinstance(json_body, dict) else None
|
||
|
||
|
||
def _headers_of(exc: Any) -> Any:
|
||
headers = getattr(getattr(exc, "response", None), "headers", None)
|
||
return headers if headers and hasattr(headers, "get") else None
|
||
|
||
|
||
def _extract_status_code(error: Exception) -> Optional[int]:
|
||
"""HTTP status code from the error or its cause chain."""
|
||
return _from_cause_chain(error, _status_of, None)
|
||
|
||
|
||
def _extract_error_body(error: Exception) -> dict:
|
||
"""Structured error body from an SDK exception or its cause chain."""
|
||
return _from_cause_chain(error, _body_of, {})
|
||
|
||
|
||
def _code_from_payload(payload: Any, top_keys: Sequence[str], peek_message: bool) -> str:
|
||
"""Code/type from ``payload.error`` or a top-level key; ``"400"`` is not a code.
|
||
``peek_message`` also parses a JSON ``error.message`` for a nested code
|
||
(Responses API surfaces ``invalid_encrypted_content`` this way)."""
|
||
if not isinstance(payload, dict):
|
||
return ""
|
||
error_obj = payload.get("error", {})
|
||
if isinstance(error_obj, dict):
|
||
code = error_obj.get("code") or error_obj.get("type") or ""
|
||
if isinstance(code, str) and code.strip() and code.strip() != "400":
|
||
return code.strip()
|
||
message = error_obj.get("message")
|
||
if peek_message and isinstance(message, str) and message.strip().startswith("{"):
|
||
nested_code = _code_from_payload(_json_dict(message), ("code", "error_code"), False)
|
||
if nested_code:
|
||
return nested_code
|
||
code = next((payload.get(k) for k in top_keys if payload.get(k)), "")
|
||
text = str(code).strip() if isinstance(code, (str, int)) else ""
|
||
return text if text and text != "400" else ""
|
||
|
||
|
||
def _extract_error_code(body: dict) -> str:
|
||
"""Extract an error code string from the response body."""
|
||
return _code_from_payload(body, ("code", "error_code", "errorCode"), True) if body else ""
|
||
|
||
|
||
def _extract_message(error: Exception, body: dict) -> str:
|
||
"""Extract the most informative error message (structured body first)."""
|
||
msg = next((m for m in _body_message_candidates(body or {}) if isinstance(m, str) and m.strip()), None)
|
||
return (msg.strip() if msg else str(error))[:500]
|
||
|
||
|
||
def _is_openrouter_upstream_error(body: Any, provider: str) -> bool:
|
||
"""OpenRouter's "Provider returned error" wrapper: the key is healthy, the
|
||
upstream failed, so credential rotation is the wrong recovery."""
|
||
err = _error_obj(body)
|
||
if str(err.get("message") or "").strip().lower() != "provider returned error":
|
||
return False
|
||
if (provider or "").strip().lower() == "openrouter":
|
||
return True
|
||
# Otherwise require the metadata shape only OpenRouter produces.
|
||
metadata = err.get("metadata")
|
||
return isinstance(metadata, dict) and ("raw" in metadata or "provider_name" in metadata)
|
||
|
||
|
||
def _extract_upstream_provider_name(body: Any) -> Optional[str]:
|
||
"""Pull the upstream provider name out of OpenRouter's error metadata."""
|
||
metadata = _error_obj(body).get("metadata")
|
||
name = metadata.get("provider_name") if isinstance(metadata, dict) else None
|
||
return name.strip() if isinstance(name, str) and name.strip() else None
|