716bab6967
Review threads left open on three merged PRs, each a gap in what the tests actually guard: - #133740 (relay usage merge): no checked-in test failed on the base. The merged fix keeps message_start's input/cache counts when the SDK's MessageDeltaUsage serializes them as null, but the existing message_delta test only sent output_tokens, so it passed with the old merge too. New test_anthropic_stream_accumulator_null_delta_usage_keeps_message_start_counts sends nulls for input and cache fields; it fails with the old `{**current_usage, **usage}` merge. - #133767 (error-body table): the #49769 cell asserted the 402 would be reclassified as retryable. The fix that landed (#134157) keeps it `billing` and retries in the main loop's recovery chain, so the cell still XFAILed on main and would pass even if the retry broke. It now pins what the recovery depends on (billing + a parseable budget); the retry itself is driven end to end by test_affordable_402_main_loop.py. - #133767 (prompt-cache contract): the marker check needed cache_control in only one request, so a later request that lost its markers passed. It now requires markers on every request. The module docstring claimed native Anthropic layout coverage the cells don't have; it now says envelope layout.
98 lines
5.7 KiB
Python
98 lines
5.7 KiB
Python
"""Provider error bodies observed in real sessions, through the real SDK exception types.
|
|
|
|
Every row is a body Hermes actually received (relay traces + agent.log, scrubbed of request ids
|
|
and account data), raised as the exception class the OpenAI SDK / httpx raises for it, so a
|
|
classifier change is checked against what providers really send rather than hand-written
|
|
strings. Add a row whenever a new production body shows up in a bug report or trace.
|
|
|
|
OpenRouter's "can only afford N tokens" 402 (#49769) stays ``billing`` here; the main loop's
|
|
recovery chain retries it once with the lowered cap (see that test's docstring).
|
|
|
|
The last test is a body main still mishandles, as a run-time xfail keyed on the current wrong
|
|
outcome, so it turns into a passing regression test the moment its fix lands:
|
|
|
|
* #85005 — "<model> is not a multimodal model" (vLLM/text-only endpoints) is not recognised as
|
|
an image rejection, so the image is never stripped and every retry fails identically.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import openai
|
|
import pytest
|
|
|
|
from agent.error_classifier import FailoverReason, classify_api_error
|
|
from agent.message_sanitization import _looks_like_image_content_rejection
|
|
from tests.e2e.core._pending_fixes import known_failure
|
|
|
|
_REQUEST = httpx.Request("POST", "https://example.invalid/v1/chat/completions")
|
|
|
|
|
|
def _status(cls: type, code: int, body: dict) -> Exception:
|
|
return cls(f"Error code: {code} - {body}", response=httpx.Response(code, request=_REQUEST, json=body), body=body)
|
|
|
|
|
|
AFFORD_402 = {"error": {"message": (
|
|
"This request requires more credits, or fewer max_tokens. You requested up to 128000 tokens, but can only "
|
|
"afford 81664. To increase, visit https://openrouter.ai/settings/credits and upgrade to a paid account"),
|
|
"code": 402}}
|
|
NOT_MULTIMODAL_400 = {"error": {"message": "glm-5.2-fp8 is not a multimodal model", "type": "BadRequestError",
|
|
"param": None, "code": 400}}
|
|
|
|
# (id, provider, exception, expected reason, retryable, should_fallback)
|
|
OBSERVED = [
|
|
("connection-error", "custom", openai.APIConnectionError(request=_REQUEST), FailoverReason.timeout, True, False),
|
|
("broken-pipe", "nous", httpx.ReadError("[Errno 32] Broken pipe", request=_REQUEST),
|
|
FailoverReason.timeout, True, False),
|
|
("peer-closed-mid-body", "custom", httpx.RemoteProtocolError(
|
|
"peer closed connection without sending complete message body (incomplete chunked read)", request=_REQUEST),
|
|
FailoverReason.timeout, True, False),
|
|
("request-timeout", "nous", openai.APITimeoutError(request=_REQUEST), FailoverReason.timeout, True, False),
|
|
("nous-404-no-credits", "nous", _status(openai.NotFoundError, 404, {"status": 404, "message": (
|
|
"Model 'anthropic/claude-opus-5' requires available credits. Your account balance is too low to use paid "
|
|
"models — add credits at https://portal.nousresearch.com or pick a free model.")}),
|
|
FailoverReason.billing, False, True),
|
|
("nous-401-invalid-key", "nous", _status(openai.AuthenticationError, 401, {"status": 401, "message": (
|
|
"Your API key is invalid, blocked or out of funds. Please go visit the portal to sort that out: "
|
|
"https://portal.nousresearch.com ")}),
|
|
FailoverReason.auth, False, True),
|
|
("cloudflare-502", "nous", _status(openai.InternalServerError, 502, {
|
|
"type": "https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-5xx-errors/"
|
|
"error-502/", "title": "Error 502: Bad gateway", "status": 502,
|
|
"detail": "The origin web server returned an invalid or incomplete response to Cloudflare.",
|
|
"error_code": 502, "error_name": "origin_bad_gateway", "error_category": "origin"}),
|
|
FailoverReason.server_error, True, False),
|
|
("nous-500-unexpected", "nous", _status(openai.InternalServerError, 500, {"status": 500, "message": (
|
|
"Something unexpected happened while processing your request. Please try again in a moment, or contact us "
|
|
"if the issue persists.")}),
|
|
FailoverReason.server_error, True, False),
|
|
("router-no-workers", "custom", _status(openai.InternalServerError, 500, {"error": {
|
|
"message": "No available workers (all circuits open or unhealthy)"}}),
|
|
FailoverReason.server_error, True, False),
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize("provider,error,reason,retryable,fallback", [r[1:] for r in OBSERVED],
|
|
ids=[r[0] for r in OBSERVED])
|
|
def test_observed_body_routes_to_its_recovery(provider, error, reason, retryable, fallback):
|
|
result = classify_api_error(error, provider=provider, model="test-model")
|
|
assert (result.reason, result.retryable, result.should_fallback) == (reason, retryable, fallback)
|
|
|
|
|
|
def test_affordable_402_classifies_as_billing_with_a_parseable_budget():
|
|
"""#49769 is recovered in the main loop, not by the classifier: the 402 stays ``billing`` and the
|
|
recovery chain retries once with the cap lowered to the budget it names
|
|
(``tests/agent/test_affordable_402_main_loop.py`` drives that retry end to end)."""
|
|
from agent.auxiliary_client import _affordable_max_tokens_from_error
|
|
|
|
error = _status(openai.APIStatusError, 402, AFFORD_402)
|
|
result = classify_api_error(error, provider="openrouter", model="anthropic/claude-opus-5.5")
|
|
assert result.reason == FailoverReason.billing
|
|
assert _affordable_max_tokens_from_error(error) == 81664 - 64
|
|
|
|
|
|
def test_not_a_multimodal_model_is_an_image_rejection():
|
|
body = str(_status(openai.BadRequestError, 400, NOT_MULTIMODAL_400))
|
|
with known_failure(r"not recognised", "#85005: text-only endpoint image rejection not recognised"):
|
|
assert _looks_like_image_content_rejection(body), f"image rejection not recognised: {body[:120]}"
|