diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index e93d068129..1782c72673 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -340,11 +340,13 @@ def _build_anthropic_client_with_bearer_hook( kwargs["http_client"] = build_bearer_http_client(token_provider, timeout=kwargs["timeout"]) kwargs["auth_token"] = "entra-id-bearer-via-http-hook" headers = _beta_header(_common_betas_for_base_url(normalized_base_url, drop_context_1m_beta=drop_context_1m_beta)) - return _new_sdk_client(sdk, kwargs, headers) + return _new_sdk_client(sdk, kwargs, headers, route=base_url) -def _new_sdk_client(sdk, kwargs: Dict[str, Any], headers: Dict[str, str]): +def _new_sdk_client(sdk, kwargs: Dict[str, Any], headers: Dict[str, str], route: str = None): """``sdk.Anthropic(**kwargs)`` with ``headers`` attached, sending exactly ONE credential. + ``route`` is the caller's un-normalized base_url (the ``/v1`` form ``custom_providers`` entries are + keyed by; ``kwargs["base_url"]`` has it stripped) for the per-provider ``extra_headers`` lookup. The SDK fills whichever of ``api_key`` / ``auth_token`` we left unset from ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN in the environment (both loaded from ~/.hermes/.env) and then sends dual @@ -357,11 +359,28 @@ def _new_sdk_client(sdk, kwargs: Dict[str, Any], headers: Dict[str, str]): merged["Authorization"] = sdk.Omit() elif "auth_token" in kwargs and "api_key" not in kwargs: merged["X-Api-Key"] = sdk.Omit() + # Per-provider ``custom_providers[].extra_headers`` last: the most specific config level wins + # over the SDK User-Agent and the attribution/beta sets above, on every builder path (init, + # /model switch, rebuild, auxiliary) — the OpenAI-wire clients already do this (#24293, #9721). + merged.update(_custom_provider_extra_headers(route or kwargs.get("base_url"))) if merged: kwargs["default_headers"] = merged return sdk.Anthropic(**kwargs) +def _custom_provider_extra_headers(base_url) -> Dict[str, str]: + """``extra_headers`` of the ``custom_providers`` entry routed at *base_url*, else ``{}``. + SECURITY: values routinely carry credentials (Cloudflare Access tokens) — never log them.""" + if not base_url: + return {} + try: + from hermes_cli.config import get_custom_provider_extra_headers + return get_custom_provider_extra_headers(str(base_url)) + except Exception: + logger.debug("custom-provider extra_headers skipped for Anthropic client", exc_info=True) + return {} + + def _auth_style(api_key, base_url, normalized_base_url) -> str: """Order-sensitive endpoint/key classification for :func:`build_anthropic_client`. ``kimi``: Kimi's /coding endpoint 403s without a User-Agent (the Kimi team asked for proper attribution). @@ -410,7 +429,7 @@ def build_anthropic_client(api_key, base_url: str = None, timeout: float = None, # get these from profile.default_headers, but this route never sees the profile. for k, v in _attribution_headers().items(): headers.setdefault(k, v) - return _new_sdk_client(sdk, kwargs, headers) + return _new_sdk_client(sdk, kwargs, headers, route=base_url) def build_anthropic_bedrock_client(region: str): diff --git a/agent/error_classifier.py b/agent/error_classifier.py index f951a7458e..7e190cf900 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -38,6 +38,7 @@ class FailoverReason(enum.Enum): billing = "billing" # 402 or confirmed credit exhaustion — rotate immediately rate_limit = "rate_limit" # 429 or quota-based throttling — backoff then rotate upstream_rate_limit = "upstream_rate_limit" # Aggregator's upstream model 429 — fallback model, key is healthy + upstream_blocked = "upstream_blocked" # 403 from a WAF/CDN/proxy in front of the provider — key is healthy, fallback overloaded = "overloaded" # 503/529 — provider overloaded, backoff server_error = "server_error" # 500/502 — internal server error, retry timeout = "timeout" # Connection/read timeout — rebuild client + retry @@ -416,6 +417,17 @@ _SSL_TRANSIENT_PATTERNS = ( ) +# A 403 body written by a WAF/CDN/proxy rather than the provider's API: Cloudflare's browser +# challenge and block pages, plus the plain-text block relays return when they reject the SDK +# User-Agent (#53099). Matched only on 403 (see ``_status_403``); a bare "access denied" or +# "forbidden" stays auth because providers word real permission errors that way too. +_UPSTREAM_BLOCKED_PATTERNS = ( + "your request was blocked", "request blocked", "sorry, you have been blocked", + "enable javascript and cookies to continue", "cdn-cgi/challenge-platform", "cf-browser-verification", + "challenge-error-text", "__cf_chl", "cf-error-details", "attention required! | cloudflare", +) + + # ── Verdicts and rule tables ──────────────────────────────────────────── # A verdict is the ClassifiedError kwargs a stage decided on: ``reason`` plus # hint overrides (unlisted hints keep dataclass defaults). Rule tables are @@ -438,6 +450,7 @@ _V_RATE_LIMIT = _v(_R.rate_limit, **_ROTATE_FALLBACK) _V_AUTH_ROTATE = _v(_R.auth, retryable=False, **_ROTATE_FALLBACK) _V_AUTH_FALLBACK = _v(_R.auth, **_ABORT_FALLBACK) _V_MODEL_NOT_FOUND = _v(_R.model_not_found, **_ABORT_FALLBACK) +_V_UPSTREAM_BLOCKED = _v(_R.upstream_blocked, **_ABORT_FALLBACK) _V_CONTENT_BLOCKED = _v(_R.content_policy_blocked, **_ABORT_FALLBACK) # Another account in the same pool may hold the entitlement; the credential itself is healthy. _V_MODEL_ENTITLEMENT = _v(_R.model_entitlement, retryable=False, **_ROTATE_FALLBACK) @@ -912,7 +925,14 @@ def _status_403(c: _Ctx) -> Verdict: # OpenRouter 403 "key limit exceeded" and similar plan/credit exhaustion are billing. xai_spend = c.provider_slug == "xai-oauth" and c.code == _XAI_SPENDING_LIMIT_ERROR_CODE billing = xai_spend or any(p in c.msg for p in ("key limit exceeded", "spending limit") + _BILLING_PATTERNS) - return _V_BILLING if billing else _V_AUTH_FALLBACK + if billing: + return _V_BILLING + # A WAF/CDN in front of the provider answered, not the provider: the credential never + # reached it, so key guidance and credential rotation are wrong (#53099, #70566). Gated on + # 403 and on established block/challenge markers; any other 403 stays auth. + if any(p in c.msg for p in _UPSTREAM_BLOCKED_PATTERNS): + return _V_UPSTREAM_BLOCKED + return _V_AUTH_FALLBACK def _status_404(c: _Ctx) -> Verdict: diff --git a/agent/turn_failure_copy.py b/agent/turn_failure_copy.py index 7369458406..272888fd8d 100644 --- a/agent/turn_failure_copy.py +++ b/agent/turn_failure_copy.py @@ -168,6 +168,11 @@ _NONRETRYABLE_COPY: Dict[str, str] = { "{label}'s account settings don't allow this model for your request, so it didn't " "answer. Check the provider's data/privacy settings, or switch models with /model." ), + FailoverReason.upstream_blocked.value: ( + "A firewall/CDN in front of {label} blocked the request before it reached the model, so " + "your key is probably fine. Set a custom User-Agent via the provider's extra_headers, check " + "the proxy/WAF rules, or switch providers with /model." + ), } _NONRETRYABLE_DEFAULT_COPY = ( "{label} rejected the request and retrying won't help. Pick another model with /model, " @@ -202,6 +207,7 @@ FAILURE_CAUSE_GLOSS: Dict[str, str] = { "billing_unverified": "the AI model service says the account's usage or credit limit is reached", FailoverReason.auth.value: "the AI model service rejected the sign-in", FailoverReason.auth_permanent.value: "the AI model service rejected the sign-in", + FailoverReason.upstream_blocked.value: "a firewall/CDN in front of the AI model service blocked the request", FailoverReason.model_not_found.value: "the model {subject} uses was not found at the AI model service", FailoverReason.content_policy_blocked.value: "the AI model service's safety filter rejected the request", "context_overflow": "{possessive} request grew too large for the model", diff --git a/agent/turn_recovery.py b/agent/turn_recovery.py index 1c338b12b5..731e54bf4e 100644 --- a/agent/turn_recovery.py +++ b/agent/turn_recovery.py @@ -827,6 +827,7 @@ def _welcome_outage_copy(base_url: Any, classified: Any, *, anonymous: bool = Fa # Terminal status label per non-retryable reason (default names the HTTP status). _NONRETRYABLE_LABELS = { FailoverReason.content_policy_blocked: "The provider's safety filter refused this request", + FailoverReason.upstream_blocked: "A firewall/CDN in front of the provider blocked this request", FailoverReason.ssl_cert_verification: "The provider's security certificate could not be verified", # Only reached after the one-shot image shrink ran (recover_after_classification sets the flag first). FailoverReason.image_too_large: "Request still exceeded the provider's size limit after shrinking images", @@ -889,6 +890,16 @@ def nonretryable_client_error_result( _vlines(agent, f" Did you mean '{_prefix_suggestion}'? It looks like the vendor prefix is missing.") elif classified.reason not in _NONRETRYABLE_LABELS: _vlines(agent, f" 💡 Fix: pick another model (/model), or check `{display_hermes_home()}/logs/agent.log`.") + # A WAF/CDN block (#53099, #70566): the key never reached the provider; the usual cause + # is the SDK User-Agent, which the per-provider extra_headers override. + if classified.reason == FailoverReason.upstream_blocked: + _vlines( + agent, + " 💡 The endpoint's firewall/CDN blocked the request before it reached the model — your key", + " and model access are probably fine. Relays often reject the SDK's default User-Agent:", + " set `extra_headers: {User-Agent: HermesAgent/1.0}` on the custom_providers entry", + " (or `model.default_headers`), or check the proxy/WAF rules and your network.", + ) # Content-policy blocks: the provider refused this prompt, so recovery is a rephrase # or another model, not key/retry advice. if classified.reason == FailoverReason.content_policy_blocked: diff --git a/hermes_cli/cli_chat_error_copy.py b/hermes_cli/cli_chat_error_copy.py index 2327b21616..81ce86c460 100644 --- a/hermes_cli/cli_chat_error_copy.py +++ b/hermes_cli/cli_chat_error_copy.py @@ -17,6 +17,7 @@ _REASON_COPY: dict[str, str] = { "model_not_found": "'{model}' isn't available on {provider}. Run /model to pick a valid model.", "rate_limit": "Rate limited by {provider}; wait a minute or /model to switch.", "upstream_rate_limit": "Rate limited by {provider}; wait a minute or /model to switch.", + "upstream_blocked": "A firewall/CDN in front of {provider} blocked the request (not your key). Set a User-Agent via extra_headers, or /model to switch.", "overloaded": "{provider} is overloaded right now. Send /retry in a moment, or /model to switch.", "server_error": "{provider} had an internal error. Send /retry in a moment, or /model to switch.", "timeout": "{provider} did not answer in time. Send /retry, or /model to switch.", diff --git a/tests/agent/test_anthropic_adapter_extra_headers.py b/tests/agent/test_anthropic_adapter_extra_headers.py new file mode 100644 index 0000000000..bf1aa738dd --- /dev/null +++ b/tests/agent/test_anthropic_adapter_extra_headers.py @@ -0,0 +1,32 @@ +"""Anthropic-messages clients honour ``custom_providers[].extra_headers`` (#24293, #9721). + +The OpenAI-wire clients apply the per-provider headers; ``build_anthropic_client`` used to skip +them, so a relay behind a WAF that rejects the SDK User-Agent kept 403ing in anthropic_messages mode. +""" +from unittest.mock import patch + +from agent.anthropic_adapter import build_anthropic_client + +_ROUTE = "https://proxy.example.com/v1" +_CONFIG = {"custom_providers": [{ + "name": "wafproxy", "base_url": _ROUTE, "api_mode": "anthropic_messages", + "extra_headers": {"User-Agent": "HermesAgent/1.0", "X-Privacy-Tier": "enterprise"}, +}]} + + +def _build(route): + with patch("agent.anthropic_adapter._require_sdk") as sdk, patch("hermes_cli.config.load_config", return_value=_CONFIG): + build_anthropic_client("sk-test", route) + return sdk.return_value.Anthropic.call_args.kwargs["default_headers"] + + +def test_matching_route_merges_extra_headers_after_betas(): + headers = _build(_ROUTE) + assert headers["User-Agent"] == "HermesAgent/1.0" + assert headers["X-Privacy-Tier"] == "enterprise" + assert "anthropic-beta" in headers # provider headers add to, not replace, the beta set + + +def test_other_route_does_not_inherit_extra_headers(): + headers = _build("https://other.example.com/v1") + assert "User-Agent" not in headers and "X-Privacy-Tier" not in headers diff --git a/tests/agent/test_error_classifier.py b/tests/agent/test_error_classifier.py index c5a5e21fe5..059d633b57 100644 --- a/tests/agent/test_error_classifier.py +++ b/tests/agent/test_error_classifier.py @@ -60,7 +60,7 @@ class TestFailoverReason: def test_enum_members_exist(self): expected = { "auth", "auth_permanent", "billing", "rate_limit", - "upstream_rate_limit", + "upstream_rate_limit", "upstream_blocked", "overloaded", "server_error", "timeout", "ssl_cert_verification", "context_overflow", "payload_too_large", "image_too_large", diff --git a/tests/agent/test_error_classifier_upstream_blocked.py b/tests/agent/test_error_classifier_upstream_blocked.py new file mode 100644 index 0000000000..faace46d05 --- /dev/null +++ b/tests/agent/test_error_classifier_upstream_blocked.py @@ -0,0 +1,34 @@ +"""A 403 written by a WAF/CDN in front of the provider is not an API-key rejection (#53099, #70566). + +A relay that blocks the SDK User-Agent answers ``403 Your request was blocked.``; Cloudflare's +browser challenge answers 403 HTML. Both used to classify as ``auth`` and print key guidance. +""" +import pytest + +from agent.error_classifier import FailoverReason, classify_api_error + + +class _APIError(Exception): + def __init__(self, message, status_code): + super().__init__(message) + self.status_code = status_code + + +@pytest.mark.parametrize("body", [ + "Error code: 403 - Your request was blocked.", + "Enable JavaScript and cookies to continue", + "", +]) +def test_403_waf_block_is_upstream_blocked_not_auth(body): + result = classify_api_error(_APIError(body, 403), provider="openai-api") + assert result.reason == FailoverReason.upstream_blocked + assert result.retryable is False and result.should_fallback is True + assert result.should_rotate_credential is False and result.is_auth is False + + +@pytest.mark.parametrize("body, status, reason", [ + ("ForbiddenAccess denied", 403, FailoverReason.auth), + ("Enable JavaScript and cookies to continue", 401, FailoverReason.auth), +]) +def test_generic_403_and_all_401_keep_auth(body, status, reason): + assert classify_api_error(_APIError(body, status), provider="openai-api").reason == reason diff --git a/tui_gateway/user_messages.py b/tui_gateway/user_messages.py index 61afbb8eee..7822c05874 100644 --- a/tui_gateway/user_messages.py +++ b/tui_gateway/user_messages.py @@ -21,6 +21,7 @@ _TURN_ERROR_CODE_COPY: dict[str, tuple[str, str]] = { "billing_unverified": ("The model provider reports no credit left", "Top up the account or switch with /model."), "rate_limit": ("The model provider is rate-limiting requests", "Wait a moment, then /retry."), "upstream_rate_limit": ("The model provider is rate-limiting requests", "Wait a moment, then /retry."), + "upstream_blocked": ("A firewall/CDN in front of the model provider blocked the request", "Set a User-Agent via the provider's extra_headers, or switch with /model."), "overloaded": ("The model provider is overloaded", "Wait a moment, then /retry."), "server_error": ("The model provider had an internal error", "Wait a moment, then /retry."), "timeout": ("The model provider did not answer in time", "Try /retry; if it keeps happening, switch with /model."), diff --git a/ui-tui/src/app/userMessages.ts b/ui-tui/src/app/userMessages.ts index 881620833f..76da58f52c 100644 --- a/ui-tui/src/app/userMessages.ts +++ b/ui-tui/src/app/userMessages.ts @@ -241,6 +241,10 @@ const TURN_CODE_COPY: Record = { "Check the endpoint's certificate, then /retry." ], timeout: ['The model provider did not answer in time', 'Try /retry; if it keeps happening, switch with /model.'], + upstream_blocked: [ + 'A firewall/CDN in front of the model provider blocked the request', + "Set a User-Agent via the provider's extra_headers, or switch with /model." + ], upstream_rate_limit: ['The model provider is rate-limiting requests', 'Wait a moment, then /retry.'] } diff --git a/website/docs/user-guide/configuring-models.md b/website/docs/user-guide/configuring-models.md index 94345b2d61..e830d871b8 100644 --- a/website/docs/user-guide/configuring-models.md +++ b/website/docs/user-guide/configuring-models.md @@ -192,7 +192,7 @@ providers: CF-Access-Client-Secret: "yyyy" ``` -Header values routinely carry credentials — Hermes never logs them. `extra_headers` applies to OpenAI-compatible routes; the `anthropic_messages` and `bedrock_converse` API modes do not use it. +Header values routinely carry credentials — Hermes never logs them. `extra_headers` applies to OpenAI-compatible routes and to `anthropic_messages` routes (the main client, `/model` switches, rebuilds and auxiliary clients alike); `bedrock_converse` does not use it. A relay behind a WAF that rejects the SDK's default `User-Agent` (403 "Your request was blocked" or a browser-challenge page) is the typical reason to set one — Hermes reports such a 403 as a firewall/CDN block rather than an API-key rejection. **`discover_models`** — set to `false` (default `true`) to skip querying the endpoint's `/models` listing and use only the `models` you configured on the entry. Handy for gateways whose model listing is slow, unreliable, or noisy: