43717123ca
Selecting an NVIDIA NIM model whose id reached config without the nvidia/ prefix produced a bare "HTTP 404: 404 page not found" — retried three times, never naming the model. It reads exactly like an outage or an auth failure, which is where the Discord thread spent its time before the id was spotted. normalize_model_for_provider() had no branch for nvidia, so a bare id passed straight through to the API. Repair it from the provider's curated catalogue: a bare name that matches exactly one entry modulo the prefix gets it back. That's a lookup, not a guess — build.nvidia.com also fronts local NIM containers and third-party models, and anything absent from the catalogue is left alone. Because the repair runs on every runtime setup, an already-broken config self-heals on the next turn and prints what it changed. If a bare id still reaches the wire, the 404 now explains itself. The classifier consults the same catalogue: a prefix-less id the provider only serves as vendor/model is a deterministic failure, so it classifies as model_not_found instead of burning three retries on a retryable "unknown", and the error trace names the id to use. Fixes #78796
1086 lines
41 KiB
Python
1086 lines
41 KiB
Python
"""Tests for agent.error_classifier — structured API error classification."""
|
|
|
|
import pytest
|
|
from agent.error_classifier import (
|
|
ClassifiedError,
|
|
FailoverReason,
|
|
classify_api_error,
|
|
_extract_status_code,
|
|
_extract_error_body,
|
|
_extract_error_code,
|
|
_classify_402,
|
|
)
|
|
|
|
|
|
# ── Helper: mock API errors ────────────────────────────────────────────
|
|
|
|
class MockAPIError(Exception):
|
|
"""Simulates an OpenAI SDK APIStatusError."""
|
|
def __init__(self, message, status_code=None, body=None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body or {}
|
|
|
|
|
|
class MockTransportError(Exception):
|
|
"""Simulates a transport-level error with a specific type name."""
|
|
pass
|
|
|
|
|
|
class ReadTimeout(MockTransportError):
|
|
pass
|
|
|
|
|
|
class ConnectError(MockTransportError):
|
|
pass
|
|
|
|
|
|
class RemoteProtocolError(MockTransportError):
|
|
pass
|
|
|
|
|
|
class ServerDisconnectedError(MockTransportError):
|
|
pass
|
|
|
|
|
|
# ── Test: FailoverReason enum ──────────────────────────────────────────
|
|
|
|
class TestFailoverReason:
|
|
def test_all_reasons_have_string_values(self):
|
|
for reason in FailoverReason:
|
|
assert isinstance(reason.value, str)
|
|
|
|
def test_enum_members_exist(self):
|
|
expected = {
|
|
"auth", "auth_permanent", "billing", "rate_limit",
|
|
"upstream_rate_limit",
|
|
"overloaded", "server_error", "timeout",
|
|
"ssl_cert_verification",
|
|
"context_overflow", "payload_too_large", "image_too_large",
|
|
"model_not_found", "format_error",
|
|
"invalid_encrypted_content",
|
|
"multimodal_tool_content_unsupported",
|
|
"provider_policy_blocked",
|
|
"content_policy_blocked",
|
|
"thinking_signature", "long_context_tier",
|
|
"oauth_long_context_beta_forbidden",
|
|
"llama_cpp_grammar_pattern",
|
|
"unknown",
|
|
}
|
|
actual = {r.value for r in FailoverReason}
|
|
assert expected == actual
|
|
|
|
|
|
# ── Test: ClassifiedError ──────────────────────────────────────────────
|
|
|
|
class TestClassifiedError:
|
|
def test_is_auth_property(self):
|
|
e1 = ClassifiedError(reason=FailoverReason.auth)
|
|
assert e1.is_auth is True
|
|
|
|
e2 = ClassifiedError(reason=FailoverReason.auth_permanent)
|
|
assert e2.is_auth is True
|
|
|
|
e3 = ClassifiedError(reason=FailoverReason.billing)
|
|
assert e3.is_auth is False
|
|
|
|
def test_defaults(self):
|
|
e = ClassifiedError(reason=FailoverReason.unknown)
|
|
assert e.retryable is True
|
|
assert e.should_compress is False
|
|
assert e.should_rotate_credential is False
|
|
assert e.should_fallback is False
|
|
assert e.status_code is None
|
|
assert e.message == ""
|
|
|
|
|
|
# ── Test: Status code extraction ───────────────────────────────────────
|
|
|
|
class TestExtractStatusCode:
|
|
|
|
def test_from_status_attr(self):
|
|
class ErrWithStatus(Exception):
|
|
status = 503
|
|
assert _extract_status_code(ErrWithStatus()) == 503
|
|
|
|
def test_from_cause_chain(self):
|
|
inner = MockAPIError("inner", status_code=401)
|
|
outer = Exception("outer")
|
|
outer.__cause__ = inner
|
|
assert _extract_status_code(outer) == 401
|
|
|
|
|
|
|
|
|
|
# ── Test: Error body extraction ────────────────────────────────────────
|
|
|
|
class TestExtractErrorBody:
|
|
def test_from_body_attr(self):
|
|
e = MockAPIError("fail", body={"error": {"message": "bad"}})
|
|
assert _extract_error_body(e) == {"error": {"message": "bad"}}
|
|
|
|
def test_from_cause_chain_body_attr(self):
|
|
inner = MockAPIError(
|
|
"inner",
|
|
status_code=402,
|
|
body={"error": {"message": "Usage limit reached, try again in 5 minutes"}},
|
|
)
|
|
outer = Exception("outer")
|
|
outer.__cause__ = inner
|
|
assert _extract_error_body(outer) == {
|
|
"error": {"message": "Usage limit reached, try again in 5 minutes"},
|
|
}
|
|
|
|
def test_empty_when_no_body(self):
|
|
assert _extract_error_body(Exception("generic")) == {}
|
|
|
|
|
|
# ── Test: Error code extraction ────────────────────────────────────────
|
|
|
|
class TestExtractErrorCode:
|
|
|
|
|
|
def test_from_top_level_code(self):
|
|
body = {"code": "model_not_found"}
|
|
assert _extract_error_code(body) == "model_not_found"
|
|
|
|
|
|
def test_empty_when_no_code(self):
|
|
assert _extract_error_code({}) == ""
|
|
assert _extract_error_code({"error": {"message": "oops"}}) == ""
|
|
|
|
|
|
# ── Test: 402 disambiguation ───────────────────────────────────────────
|
|
|
|
class TestClassify402:
|
|
"""The critical 402 billing vs rate_limit disambiguation."""
|
|
|
|
def test_billing_exhaustion(self):
|
|
"""Plain 402 = billing."""
|
|
result = _classify_402(
|
|
"payment required",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.should_rotate_credential is True
|
|
|
|
|
|
def test_quota_with_retry(self):
|
|
"""402 with 'quota' + 'retry' = rate limit."""
|
|
result = _classify_402(
|
|
"quota exceeded, please retry after the window resets",
|
|
lambda reason, **kw: ClassifiedError(reason=reason, **kw),
|
|
)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
|
|
|
|
|
|
# ── Test: Full classification pipeline ─────────────────────────────────
|
|
|
|
class TestClassifyApiError:
|
|
"""End-to-end classification tests."""
|
|
|
|
# ── Auth errors ──
|
|
|
|
def test_401_classified_as_auth(self):
|
|
e = MockAPIError("Unauthorized", status_code=401)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.auth
|
|
assert result.should_rotate_credential is True
|
|
# 401 is non-retryable on its own — credential rotation runs
|
|
# before the retryability check in the agent loop.
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
def test_403_classified_as_auth(self):
|
|
e = MockAPIError("Forbidden", status_code=403)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.auth
|
|
assert result.should_fallback is True
|
|
|
|
|
|
|
|
|
|
|
|
# ── Billing ──
|
|
|
|
def test_402_plain_billing(self):
|
|
e = MockAPIError("Payment Required", status_code=402)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
|
|
|
|
|
|
|
|
def test_404_free_tier_model_block_is_billing(self):
|
|
e = MockAPIError(
|
|
"Not Found",
|
|
status_code=404,
|
|
body={
|
|
"status": 404,
|
|
"message": (
|
|
"Model 'gpt-5' is not available on the Free Tier. "
|
|
"Upgrade at https://portal.nousresearch.com or pick a free model."
|
|
),
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="nous", model="gpt-5")
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
|
|
|
|
# ── Rate limit ──
|
|
|
|
def test_429_rate_limit(self):
|
|
e = MockAPIError("Too Many Requests", status_code=429)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_fallback is True
|
|
|
|
def test_alibaba_rate_increased_too_quickly(self):
|
|
"""Alibaba/DashScope returns a unique throttling message.
|
|
|
|
Port from anomalyco/opencode#21355.
|
|
"""
|
|
msg = (
|
|
"Upstream error from Alibaba: Request rate increased too quickly. "
|
|
"To ensure system stability, please adjust your client logic to "
|
|
"scale requests more smoothly over time."
|
|
)
|
|
e = MockAPIError(msg, status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is True
|
|
|
|
# ── Server errors ──
|
|
|
|
def test_500_server_error(self):
|
|
e = MockAPIError("Internal Server Error", status_code=500)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
assert result.retryable is True
|
|
|
|
def test_502_server_error(self):
|
|
e = MockAPIError("Bad Gateway", status_code=502)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
|
|
def test_503_overloaded(self):
|
|
e = MockAPIError("Service Unavailable", status_code=503)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.overloaded
|
|
|
|
|
|
def test_408_request_timeout_is_retryable_timeout(self):
|
|
"""HTTP 408 Request Timeout is a transient timing failure the server
|
|
itself flags as safe to retry (RFC 9110 §15.5.9) — commonly emitted by
|
|
reverse proxies in front of self-hosted backends (llama.cpp / Ollama /
|
|
vLLM) when a long generation outruns the proxy's request-read window.
|
|
It must NOT fall into the generic 4xx bucket as a non-retryable
|
|
format_error, which would abort the turn on a retry-safe error."""
|
|
e = MockAPIError("Request Timeout", status_code=408)
|
|
result = classify_api_error(e, provider="vllm")
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_400_bad_request_still_non_retryable_format_error(self):
|
|
"""Guard the boundary: a genuine 400 Bad Request must remain a
|
|
non-retryable format_error and must not be swept up by the 408 branch."""
|
|
e = MockAPIError("Bad Request", status_code=400)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
|
|
def test_message_only_overloaded_without_status_is_overloaded(self):
|
|
"""Some Anthropic-compatible proxies surface 'overloaded' in the
|
|
message with no 503/529 status_code. It must classify as overloaded
|
|
(transient backoff+retry), not unknown / credential rotation. (#14261)"""
|
|
e = MockAPIError(
|
|
"Anthropic API error: Overloaded - the service is temporarily overloaded"
|
|
) # no status_code
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.overloaded
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is False
|
|
|
|
def test_429_with_overloaded_body_is_overloaded_not_rate_limit(self):
|
|
"""Z.AI / Zhipu reuse HTTP 429 for server-wide overload. The credential
|
|
is valid — the server is just busy — so it must classify as overloaded
|
|
(back off + retry the same key), NOT rate_limit (which would rotate and
|
|
exhaust the pool, doing nothing for a single-key user). (#14038)"""
|
|
e = MockAPIError(
|
|
"The service may be temporarily overloaded, please try again later",
|
|
status_code=429,
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.overloaded
|
|
assert result.retryable is True
|
|
assert result.should_rotate_credential is False
|
|
|
|
def test_429_normal_rate_limit_still_rotates(self):
|
|
"""Guard: a genuine 429 rate limit (no overload language) must still
|
|
classify as rate_limit and rotate the credential. (#14038)"""
|
|
e = MockAPIError(
|
|
"Rate limit exceeded: too many requests", status_code=429
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_rotate_credential is True
|
|
|
|
# ── 5xx that are actually request-validation errors ──
|
|
# Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
|
|
# request-validation failures with a 5xx status. These are
|
|
# deterministic, so they must NOT be retried — otherwise the retry
|
|
# loop hammers the identical bad request into a flood.
|
|
|
|
|
|
|
|
|
|
|
|
# ── 5xx that are actually context overflow ──
|
|
# Some local inference servers (llama.cpp / llama-server, and vLLM/Ollama
|
|
# behind a Cloudflare/Tailscale hop) report context overflow with a 5xx
|
|
# status instead of the standard 400/413. These must route into the
|
|
# compression-and-retry path, not the blind server_error/overloaded retry
|
|
# that exhausts and drops the turn.
|
|
|
|
|
|
|
|
|
|
# ── Model not found ──
|
|
|
|
def test_404_model_not_found(self):
|
|
e = MockAPIError("model not found", status_code=404)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
assert result.should_fallback is True
|
|
assert result.retryable is False
|
|
|
|
def test_404_generic(self):
|
|
# Generic 404 with no "model not found" signal — common for local
|
|
# llama.cpp/Ollama/vLLM endpoints with slightly wrong paths. Treat
|
|
# as unknown (retryable) so the real error surfaces, rather than
|
|
# claiming the model is missing and silently falling back.
|
|
e = MockAPIError("Not Found", status_code=404)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
def test_404_bare_model_id_missing_prefix_is_model_not_found(self):
|
|
"""A bare id the provider only serves as ``vendor/id`` is malformed.
|
|
|
|
Regression for #78796: NVIDIA NIM answers a prefix-less
|
|
``nemotron-3-ultra-550b-a55b`` with a naked ``404 page not found``.
|
|
Without the catalogue check this fell into the generic branch and
|
|
burned three retries on a deterministic failure, reporting what
|
|
looked like an outage.
|
|
"""
|
|
e = MockAPIError("404 page not found", status_code=404)
|
|
result = classify_api_error(
|
|
e, provider="nvidia", model="nemotron-3-ultra-550b-a55b"
|
|
)
|
|
assert result.reason == FailoverReason.model_not_found
|
|
assert result.retryable is False
|
|
|
|
def test_404_correctly_prefixed_model_stays_generic(self):
|
|
"""A properly prefixed id hitting a 404 is a real endpoint problem —
|
|
it must keep the retryable generic classification."""
|
|
e = MockAPIError("404 page not found", status_code=404)
|
|
result = classify_api_error(
|
|
e, provider="nvidia", model="nvidia/nemotron-3-ultra-550b-a55b"
|
|
)
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
|
|
def test_404_unknown_bare_model_stays_generic(self):
|
|
"""A local NIM container isn't in the catalogue — no verdict invented."""
|
|
e = MockAPIError("404 page not found", status_code=404)
|
|
result = classify_api_error(e, provider="nvidia", model="my-local-nim")
|
|
assert result.reason == FailoverReason.unknown
|
|
assert result.retryable is True
|
|
|
|
# ── Provider policy-block (OpenRouter privacy/guardrail) ──
|
|
|
|
|
|
|
|
|
|
# ── Provider content-policy block (per-prompt safety filter) ──
|
|
#
|
|
# Distinct from ``provider_policy_blocked`` above — these are upstream
|
|
# model-provider safety refusals for THIS prompt, not OpenRouter
|
|
# account-level data policy. Recovery is fallback model, not config fix.
|
|
# See issue #18028 — OpenAI Codex was burning 3 retries on identical
|
|
# refusals before users saw "API failed after 3 retries" on Telegram.
|
|
|
|
def test_message_only_cyber_content_policy_blocked(self):
|
|
# OpenAI Codex returns this without an HTTP status. Retrying the
|
|
# same prompt three times only repeats the same policy decision, so
|
|
# the classifier must jump straight to fallback / abort instead of
|
|
# leaving it in the retryable ``unknown`` bucket.
|
|
e = Exception(
|
|
"This content was flagged for possible cybersecurity risk. If this "
|
|
"seems wrong, try rephrasing your request. To get authorized for "
|
|
"security work, join the Trusted Access for Cyber program."
|
|
)
|
|
result = classify_api_error(e, provider="openai-codex", model="gpt-5.5")
|
|
assert result.reason == FailoverReason.content_policy_blocked
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
assert result.should_compress is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── Payload too large ──
|
|
|
|
def test_413_payload_too_large(self):
|
|
e = MockAPIError("Request Entity Too Large", status_code=413)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.payload_too_large
|
|
assert result.should_compress is True
|
|
|
|
# ── Context overflow ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── Server disconnect + large session ──
|
|
|
|
|
|
|
|
|
|
# ── Provider-specific: Anthropic thinking signature ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("error_code", ["Invalid_Encrypted_Content", "INVALID_ENCRYPTED_CONTENT"])
|
|
def test_invalid_encrypted_content_code_is_case_insensitive_for_400(self, error_code):
|
|
e = MockAPIError(
|
|
"Error code: 400 - bad request",
|
|
status_code=400,
|
|
body={"error": {"code": error_code, "message": "Bad request"}},
|
|
)
|
|
result = classify_api_error(e, provider="custom", model="gpt-5.4")
|
|
assert result.reason == FailoverReason.invalid_encrypted_content
|
|
assert result.retryable is True
|
|
assert result.should_fallback is False
|
|
|
|
# ── Provider-specific: llama.cpp grammar-parse ──
|
|
|
|
|
|
|
|
|
|
|
|
# ── Provider-specific: Anthropic long-context tier ──
|
|
|
|
def test_anthropic_long_context_tier(self):
|
|
e = MockAPIError(
|
|
"Extra usage is required for long context requests over 200k tokens",
|
|
status_code=429,
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="claude-sonnet-4")
|
|
assert result.reason == FailoverReason.long_context_tier
|
|
assert result.should_compress is True
|
|
|
|
|
|
# ── Provider-specific: Anthropic OAuth 1M-context beta forbidden ──
|
|
|
|
|
|
|
|
|
|
# ── Transport errors ──
|
|
|
|
def test_read_timeout(self):
|
|
e = ReadTimeout("Read timed out")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
def test_connect_error(self):
|
|
e = ConnectError("Connection refused")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_connection_error_builtin(self):
|
|
e = ConnectionError("Connection reset by peer")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
def test_timeout_error_builtin(self):
|
|
e = TimeoutError("timed out")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
|
|
|
|
|
|
# ── Error code classification ──
|
|
|
|
|
|
|
|
|
|
|
|
# ── Message-only patterns (no status code) ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── Message-only usage limit disambiguation (no status code) ──
|
|
|
|
|
|
|
|
|
|
|
|
# ── Unknown / fallback ──
|
|
|
|
|
|
# ── Format error ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_400_litellm_invalid_request_body_shape(self, caplog):
|
|
"""litellm/Bedrock proxy shape (errorMessage/errorCode) → format_error.
|
|
|
|
The proxy in front of Anthropic surfaces the empty-content rejection
|
|
as {"errorMessage": "...non-empty content...", "errorCode":
|
|
"INVALID_REQUEST_BODY", "errorArgs": {"reason": "..."}}. Those keys
|
|
are not the standard error.message / message, so err_body_msg used to
|
|
come back empty → is_generic=True → mis-routed into compression on a
|
|
large session. Both the message pattern and the errorCode must be
|
|
recognized, and a distinct warning must be logged so the condition is
|
|
observable in the field.
|
|
"""
|
|
import logging
|
|
proxy_msg = ("The provided request body is invalid: claude "
|
|
"messages.208: all messages must have non-empty content "
|
|
"except for the optional final assistant message")
|
|
e = MockAPIError(
|
|
proxy_msg,
|
|
status_code=400,
|
|
body={
|
|
"errorMessage": proxy_msg,
|
|
"errorCode": "INVALID_REQUEST_BODY",
|
|
"statusCode": 400,
|
|
"errorArgs": {"reason": "claude messages.208: ..."},
|
|
},
|
|
)
|
|
with caplog.at_level(logging.WARNING, logger="agent.error_classifier"):
|
|
result = classify_api_error(
|
|
e, approx_tokens=66000, context_length=200000, num_messages=219,
|
|
)
|
|
assert result.reason == FailoverReason.format_error
|
|
assert result.retryable is False
|
|
assert result.should_compress is not True
|
|
assert any(
|
|
"Malformed message array 400" in r.getMessage()
|
|
for r in caplog.records
|
|
), "Expected a distinct warning identifying the malformed-body 400"
|
|
|
|
|
|
# ── Peer closed + large session ──
|
|
|
|
|
|
# ── Chinese error messages ──
|
|
|
|
|
|
# ── Z.AI / Zhipu GLM error messages ──
|
|
|
|
def test_zai_glm_token_limit_overflow(self):
|
|
"""Z.AI GLM's 'tokens in request more than max tokens allowed'
|
|
(error code 1210) → context_overflow, so the agent compresses
|
|
instead of blindly retrying. Port of anomalyco/opencode#35671."""
|
|
e = MockAPIError(
|
|
'{"error": {"code": "1210", "message": '
|
|
'"tokens in request more than max tokens allowed"}}',
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="zai")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
# ── vLLM / local inference server error messages ──
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── Result metadata ──
|
|
|
|
def test_provider_and_model_in_result(self):
|
|
e = MockAPIError("fail", status_code=500)
|
|
result = classify_api_error(e, provider="openrouter", model="gpt-5")
|
|
assert result.provider == "openrouter"
|
|
assert result.model == "gpt-5"
|
|
assert result.status_code == 500
|
|
|
|
def test_message_extracted(self):
|
|
e = MockAPIError(
|
|
"outer",
|
|
status_code=500,
|
|
body={"error": {"message": "Internal server error occurred"}},
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.message == "Internal server error occurred"
|
|
|
|
|
|
# ── Test: Adversarial / edge cases (from live testing) ─────────────────
|
|
|
|
class TestAdversarialEdgeCases:
|
|
"""Edge cases discovered during live testing with real SDK objects."""
|
|
|
|
|
|
def test_500_with_none_body(self):
|
|
e = MockAPIError("fail", status_code=500, body=None)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.server_error
|
|
|
|
def test_non_dict_body(self):
|
|
"""Some providers return strings instead of JSON."""
|
|
class StringBodyError(Exception):
|
|
status_code = 400
|
|
body = "just a string"
|
|
result = classify_api_error(StringBodyError("bad"))
|
|
assert result.reason == FailoverReason.format_error
|
|
|
|
|
|
|
|
def test_three_level_cause_chain(self):
|
|
inner = MockAPIError("inner", status_code=429)
|
|
middle = Exception("middle")
|
|
middle.__cause__ = inner
|
|
outer = RuntimeError("outer")
|
|
outer.__cause__ = middle
|
|
result = classify_api_error(outer)
|
|
assert result.status_code == 429
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_400_with_rate_limit_text(self):
|
|
"""Some providers send rate limits as 400 instead of 429."""
|
|
e = MockAPIError(
|
|
"rate limit policy",
|
|
status_code=400,
|
|
body={"error": {"message": "rate limit exceeded on this model"}},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
|
|
def test_400_anthropic_extra_usage_exhausted(self):
|
|
"""Anthropic returns 400 with 'out of extra usage' when the user's
|
|
extra-usage allowance is depleted. Must classify as billing so the
|
|
fallback chain engages (with credential rotation) instead of the
|
|
generic format_error path, which never rotates. (#11736, #13170)"""
|
|
e = MockAPIError(
|
|
"You're out of extra usage. Add more at claude.ai/settings/usage and keep going.",
|
|
status_code=400,
|
|
body={"error": {
|
|
"type": "invalid_request_error",
|
|
"message": "You're out of extra usage. Add more at claude.ai/settings/usage and keep going.",
|
|
}},
|
|
)
|
|
result = classify_api_error(e, provider="anthropic")
|
|
assert result.reason == FailoverReason.billing
|
|
assert result.should_fallback is True
|
|
assert result.retryable is False
|
|
assert result.should_rotate_credential is True
|
|
|
|
def test_200_with_error_body(self):
|
|
"""200 status with error in body — should be unknown, not crash."""
|
|
class WeirdSuccess(Exception):
|
|
status_code = 200
|
|
body = {"error": {"message": "loading"}}
|
|
result = classify_api_error(WeirdSuccess("model loading"))
|
|
assert result.reason == FailoverReason.unknown
|
|
|
|
|
|
def test_connection_refused_error(self):
|
|
e = ConnectionRefusedError("Connection refused: localhost:11434")
|
|
result = classify_api_error(e, provider="ollama")
|
|
assert result.reason == FailoverReason.timeout
|
|
|
|
|
|
def test_disconnect_pattern_ordering(self):
|
|
"""Disconnect + large session must beat generic transport catch."""
|
|
class FakeRemoteProtocol(Exception):
|
|
pass
|
|
# Type name isn't in _TRANSPORT_ERROR_TYPES but message has disconnect pattern
|
|
e = Exception("peer closed connection without sending complete message")
|
|
result = classify_api_error(e, approx_tokens=150000, context_length=200000)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
|
|
def test_deepseek_402_chinese(self):
|
|
"""Chinese billing message should still match billing patterns."""
|
|
# "余额不足" doesn't match English billing patterns, but 402 defaults to billing
|
|
e = MockAPIError("余额不足", status_code=402)
|
|
result = classify_api_error(e, provider="deepseek")
|
|
assert result.reason == FailoverReason.billing
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ── Regression: dict-typed message field (Issue #11233) ──
|
|
|
|
|
|
|
|
|
|
# Broader non-string type guards — defense against other provider quirks.
|
|
|
|
|
|
|
|
|
|
|
|
# ── Test: SSL/TLS transient errors ─────────────────────────────────────
|
|
|
|
class TestSSLTransientPatterns:
|
|
"""SSL/TLS alerts mid-stream should retry as timeout, not unknown, and
|
|
should NOT trigger context compression even on a large session.
|
|
|
|
Motivation: OpenSSL 3.x changed TLS alert error code format
|
|
(`SSLV3_ALERT_BAD_RECORD_MAC` → `SSL/TLS_ALERT_BAD_RECORD_MAC`),
|
|
breaking string-exact matching in downstream retry logic. We match
|
|
stable substrings instead.
|
|
"""
|
|
|
|
def test_bad_record_mac_classifies_as_timeout(self):
|
|
"""OpenSSL 3.x mid-stream bad record mac alert."""
|
|
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac (_ssl.c:2580)")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_plain_disconnect_on_large_session_still_compresses(self):
|
|
"""Regression guard: the context-overflow-via-disconnect path
|
|
(non-SSL disconnects on large sessions) must still trigger
|
|
compression. Only SSL-specific disconnects skip it.
|
|
"""
|
|
e = Exception("Server disconnected without sending a response")
|
|
result = classify_api_error(
|
|
e,
|
|
approx_tokens=180000,
|
|
context_length=200000,
|
|
num_messages=300,
|
|
)
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
|
|
|
|
# ── Test: SSL certificate verification failures (fail fast) ────────────
|
|
|
|
class TestSSLCertVerificationFailFast:
|
|
"""Certificate verification failures are deterministic for the host —
|
|
a TLS-inspecting proxy, missing custom CA, expired or self-signed cert
|
|
fails identically on every retry. They must classify as non-retryable
|
|
``ssl_cert_verification`` so the user sees the fix hint immediately,
|
|
instead of matching the transient "[ssl:" pattern and retrying forever.
|
|
|
|
Inspired by Claude Code v2.1.199 (July 2026).
|
|
"""
|
|
|
|
def test_python_cert_verify_failed_is_non_retryable(self):
|
|
import ssl
|
|
e = ssl.SSLCertVerificationError(
|
|
1,
|
|
"[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: "
|
|
"unable to get local issuer certificate (_ssl.c:1006)",
|
|
)
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.ssl_cert_verification
|
|
assert result.retryable is False
|
|
assert result.should_compress is False
|
|
|
|
|
|
|
|
|
|
|
|
def test_transient_ssl_alert_still_retries(self):
|
|
"""Regression guard: genuine transient alerts keep retrying."""
|
|
e = Exception("[SSL: BAD_RECORD_MAC] sslv3 alert bad record mac")
|
|
result = classify_api_error(e)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
|
|
|
|
# ── Test: RateLimitError without status_code (Copilot/GitHub Models) ──────────
|
|
|
|
class TestRateLimitErrorWithoutStatusCode:
|
|
"""Regression tests for the Copilot/GitHub Models edge case where the
|
|
OpenAI SDK raises RateLimitError but does not populate .status_code."""
|
|
|
|
def _make_rate_limit_error(self, status_code=None):
|
|
"""Create an exception whose class name is 'RateLimitError' with
|
|
an optionally missing status_code, mirroring the OpenAI SDK shape."""
|
|
cls = type("RateLimitError", (Exception,), {})
|
|
e = cls("You have exceeded your rate limit.")
|
|
e.status_code = status_code # None simulates the Copilot case
|
|
return e
|
|
|
|
def test_rate_limit_error_without_status_code_classified_as_rate_limit(self):
|
|
"""RateLimitError with status_code=None must classify as rate_limit."""
|
|
e = self._make_rate_limit_error(status_code=None)
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_rate_limit_error_with_status_code_429_classified_as_rate_limit(self):
|
|
"""RateLimitError that does set status_code=429 still classifies correctly."""
|
|
e = self._make_rate_limit_error(status_code=429)
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
|
|
def test_other_error_without_status_code_not_forced_to_rate_limit(self):
|
|
"""A non-RateLimitError with missing status_code must NOT be forced to 429."""
|
|
cls = type("APIError", (Exception,), {})
|
|
e = cls("something went wrong")
|
|
e.status_code = None
|
|
result = classify_api_error(e, provider="copilot", model="gpt-4o")
|
|
assert result.reason != FailoverReason.rate_limit
|
|
|
|
|
|
|
|
# ── Test: multimodal_tool_content_unsupported pattern ───────────────────
|
|
|
|
class TestMultimodalToolContentUnsupported:
|
|
"""Issue #27344 — providers that reject list-type tool message content
|
|
should be classified as ``multimodal_tool_content_unsupported`` so the
|
|
retry loop can downgrade screenshots to text and try again.
|
|
"""
|
|
|
|
def test_xiaomi_mimo_text_is_not_set_pattern(self):
|
|
"""The actual Xiaomi MiMo 400 wording from the bug report."""
|
|
e = MockAPIError(
|
|
"Error code: 400 - {'error': {'code': '400', 'message': 'Param Incorrect', 'param': 'text is not set', 'type': ''}}",
|
|
status_code=400,
|
|
)
|
|
result = classify_api_error(e, provider="xiaomi", model="mimo-v2.5")
|
|
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
|
|
assert result.retryable is True
|
|
|
|
|
|
|
|
|
|
|
|
def test_unrelated_400_is_not_misclassified(self):
|
|
"""Make sure the patterns don't false-positive on normal 400s."""
|
|
e = MockAPIError("bad request: missing field 'model'", status_code=400)
|
|
result = classify_api_error(e, provider="openrouter", model="anthropic/claude-sonnet-4")
|
|
|
|
|
|
class TestOpenRouterUpstreamRateLimit:
|
|
"""Distinguish upstream-provider 429 from account-level 429 on OpenRouter.
|
|
|
|
When an upstream model (DeepSeek, Anthropic, etc.) rate-limits OpenRouter's
|
|
aggregate traffic, OpenRouter returns 429 with the outer message "Provider
|
|
returned error". The user's key is healthy — we must fall back to a
|
|
different model, NOT mark the credential exhausted.
|
|
"""
|
|
|
|
def test_openrouter_upstream_429_classified_as_upstream_rate_limit(self):
|
|
"""OpenRouter 429 with 'Provider returned error' → upstream_rate_limit."""
|
|
e = MockAPIError(
|
|
"Provider returned error",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Provider returned error",
|
|
"code": 429,
|
|
"metadata": {
|
|
"provider_name": "DeepSeek",
|
|
"raw": '{"error":{"message":"Rate limit exceeded"}}',
|
|
},
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="deepseek/deepseek-v4-flash")
|
|
assert result.reason == FailoverReason.upstream_rate_limit
|
|
assert result.should_rotate_credential is False
|
|
assert result.should_fallback is True
|
|
assert result.error_context.get("upstream_provider") == "DeepSeek"
|
|
|
|
|
|
def test_account_level_429_still_rotates_credential(self):
|
|
"""A real account-level 429 (no upstream wrapper) → rate_limit, rotates."""
|
|
e = MockAPIError(
|
|
"Rate limit exceeded: 200 requests per minute",
|
|
status_code=429,
|
|
body={
|
|
"error": {
|
|
"message": "Rate limit exceeded: 200 requests per minute",
|
|
"code": 429,
|
|
}
|
|
},
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="deepseek/deepseek-v4-flash")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_rotate_credential is True
|
|
|
|
|
|
|
|
|
|
|
|
# ── HTTP 408 request timeout ────────────────────────────────────────────
|
|
|
|
class Test408RequestTimeout:
|
|
"""HTTP 408 must never fall through to the non-retryable 'other 4xx'
|
|
bucket (that abort persists an empty assistant turn — the "disappeared
|
|
conversation" / blank-bubble symptom). ALL 408s are classified as a transient
|
|
``timeout``: retryable, and explicitly NOT should_compress.
|
|
|
|
Design decision (field 2026-07-02): even the GitHub Copilot
|
|
``user_request_timeout`` / "Timed out reading request body ... use a
|
|
smaller request size" case is a plain retry, NOT auto-compression. Real
|
|
data showed the 408 is probabilistic jitter well below the hard prompt
|
|
ceiling — the same ~785k-token request that 408'd once succeeded on the
|
|
next attempt at ~786k — so retrying the same body usually works, and
|
|
auto-compaction would silently delete conversation history for a merely
|
|
transient timeout. Genuine over-window prompts surface as 413 /
|
|
context_overflow (their own compression path); users compact 408-prone
|
|
long sessions deliberately via ``/compress``.
|
|
"""
|
|
|
|
def test_copilot_oversized_body_408_retries_as_timeout_not_compress(self):
|
|
# The exact shape GitHub Copilot returns on a long session. It must
|
|
# retry (timeout), and must NOT auto-compress.
|
|
e = MockAPIError(
|
|
"Error code: 408 - {'error': {'message': 'Timed out reading "
|
|
"request body. Try again, or use a smaller request size.', "
|
|
"'code': 'user_request_timeout'}}",
|
|
status_code=408,
|
|
body={"error": {"message": "Timed out reading request body. "
|
|
"Try again, or use a smaller request size.",
|
|
"code": "user_request_timeout"}},
|
|
)
|
|
result = classify_api_error(e, provider="copilot", model="claude-opus-4.8")
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is True
|
|
assert result.should_compress is False
|
|
|
|
|
|
|
|
|
|
def test_stale_breaker_runtime_error_triggers_fallback_not_retry(self):
|
|
# The cross-turn stale-call circuit breaker (_check_stale_giveup in
|
|
# chat_completion_helpers.py) raises a RuntimeError when the provider
|
|
# has been unresponsive for N consecutive stale attempts. This must
|
|
# be classified as non-retryable + should_fallback so the retry loop
|
|
# activates the fallback provider immediately instead of burning all
|
|
# max_retries against the same dead provider (each retry hitting the
|
|
# circuit breaker instantly with zero network overhead).
|
|
e = RuntimeError(
|
|
"Provider has been unresponsive (no response received) for "
|
|
"6 consecutive stale attempts — aborting this call to "
|
|
"avoid an indefinite stall. Switch models or start a new "
|
|
"session, then retry."
|
|
)
|
|
result = classify_api_error(
|
|
e, provider="openrouter", model="anthropic/claude-fable-5",
|
|
approx_tokens=126327, context_length=200000, num_messages=274,
|
|
)
|
|
assert result.reason == FailoverReason.timeout
|
|
assert result.retryable is False
|
|
assert result.should_fallback is True
|
|
assert result.should_compress is False
|
|
|
|
|
|
# ── Test: throttle vs overflow disambiguation + new overflow shapes ─────
|
|
# Port of anomalyco/opencode#37848 (expand context overflow patterns +
|
|
# rate-limit exclusion guard).
|
|
|
|
class TestThrottleVsOverflowDisambiguation:
|
|
"""Throttle messages that mention tokens must NOT route to compression."""
|
|
|
|
def test_bedrock_throttling_too_many_tokens_is_rate_limit(self):
|
|
# AWS Bedrock (and some proxies) surface throttling as
|
|
# "Throttling error: Too many tokens, please wait before trying
|
|
# again." — the "too many tokens" fragment sits in
|
|
# _CONTEXT_OVERFLOW_PATTERNS, so before the "throttling" rate-limit
|
|
# pattern this compressed a healthy session on every throttle.
|
|
e = Exception(
|
|
"Throttling error: Too many tokens, please wait before trying again."
|
|
)
|
|
result = classify_api_error(e, provider="bedrock", model="claude")
|
|
assert result.reason == FailoverReason.rate_limit
|
|
assert result.should_compress is False
|
|
|
|
def test_plain_too_many_tokens_still_overflow(self):
|
|
# Without any throttle wording, "Too many tokens" remains a
|
|
# context-overflow signal (Z.AI / GLM family wording).
|
|
e = Exception("Too many tokens")
|
|
result = classify_api_error(e, provider="zai", model="glm-5")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
|
|
class TestExpandedOverflowPatterns:
|
|
"""New provider overflow wordings route into compression recovery."""
|
|
|
|
def test_maximum_allowed_input_length_is_overflow(self):
|
|
# Together/Fireworks-style wording — matched no pattern before.
|
|
e = Exception(
|
|
"Input length 131393 exceeds the maximum allowed input length "
|
|
"of 131040 tokens."
|
|
)
|
|
result = classify_api_error(e, provider="together", model="m")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
assert result.should_compress is True
|
|
|
|
def test_request_too_large_message_only_is_payload_too_large(self):
|
|
# Anthropic's structured 413 type re-wrapped by a proxy with no
|
|
# status attribute — was falling through to `unknown`.
|
|
e = Exception(
|
|
'{"error":{"type":"request_too_large",'
|
|
'"message":"Request exceeds the maximum size"}}'
|
|
)
|
|
result = classify_api_error(e, provider="anthropic", model="m")
|
|
assert result.reason == FailoverReason.payload_too_large
|
|
assert result.should_compress is True
|
|
|
|
def test_longer_than_context_length_still_overflow(self):
|
|
# Regression guard for wordings that already matched.
|
|
e = Exception(
|
|
"The input (516368 tokens) is longer than the model's context "
|
|
"length (262144 tokens)."
|
|
)
|
|
result = classify_api_error(e, provider="openrouter", model="m")
|
|
assert result.reason == FailoverReason.context_overflow
|
|
|
|
|
|
|