From ce6b4d1e16e5bf50e43d3d115bb59688dc767489 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:53:31 -0700 Subject: [PATCH] refactor(run_agent): extract ApiErrorSummaryMixin (agent/api_error_summary.py) --- agent/api_error_summary.py | 240 +++++++++++++++++++++++++++++++++++++ run_agent.py | 234 ++---------------------------------- 2 files changed, 248 insertions(+), 226 deletions(-) create mode 100644 agent/api_error_summary.py diff --git a/agent/api_error_summary.py b/agent/api_error_summary.py new file mode 100644 index 0000000000..27ce350f1e --- /dev/null +++ b/agent/api_error_summary.py @@ -0,0 +1,240 @@ +"""Provider API error summarising for ``AIAgent``. + +Entitlement-failure detection, xAI subscription decoration, structured-detail coercion, and log-safe +redaction of provider error payloads. +Extracted from ``run_agent.py``; every method resolves through ``AIAgent``'s MRO unchanged. +""" +import json +import re +from typing import Any, Dict, Optional + +from agent.redact import redact_sensitive_text + + +class ApiErrorSummaryMixin: + """Provider error -> user/log-safe summary (see module docstring).""" + + @staticmethod + def _is_entitlement_failure( + error_context: Optional[Dict[str, Any]], + status_code: Optional[int], + ) -> bool: + """Detect subscription/entitlement 401/403s that masquerade as auth failures. + + Refreshing a token cannot fix an unsubscribed account, so callers surface the error instead of looping + the + pool. xAI returns the same permission-denied text for BOTH cases; a ``[WKE=unauthenticated:...]`` + suffix (or + "access token could not be validated") means stale token → return False so the refresh path runs. + """ + if status_code not in {401, 403, None}: + return False + if not isinstance(error_context, dict): + return False + # Single lowercase haystack over every field shape (message/reason and raw code/error). + message = str(error_context.get("message") or "").lower() + reason = str(error_context.get("reason") or "").lower() + code = str(error_context.get("code") or "").lower() + err = str(error_context.get("error") or "").lower() + haystack = f"{message} {reason} {code} {err}" + if not haystack.strip(): + return False + # xAI's disambiguator for stale-token vs unsubscribed: same permission-denied text, only one carries + # this suffix. Bail out so a stale OAuth token takes the credential-refresh path (#29344). + if "[wke=unauthenticated:" in haystack: + return False + if "oauth2 access token could not be validated" in haystack: + return False + if "do not have an active grok subscription" in haystack: + return True + if "out of available resources" in haystack and "grok" in haystack: + return True + if "does not have permission" in haystack and "grok" in haystack: + return True + return False + + @staticmethod + def _decorate_xai_entitlement_error(detail: str) -> str: + """Append a neutral hint when xAI's OAuth surface returns the permission-denied 403. + + xAI's ``/v1/responses`` uses one body for several causes (no subscription, tier lacks the model, quota + exhausted). The least obvious: X Premium+ does NOT include API access — only SuperGrok does. Lead with + that, keep the raw text, point at https://grok.com/?_s=usage. Matched once per detail string. + """ + if not detail: + return detail + lower = detail.lower() + is_entitlement = ( + "do not have an active grok subscription" in lower + or ("out of available resources" in lower and "grok" in lower) + or ("does not have permission" in lower and "grok" in lower) + ) + if not is_entitlement: + return detail + hint = ( + " — xAI rejected this OAuth account. NOTE: X Premium+ does NOT " + "include xAI API access — only standalone SuperGrok subscribers " + "can use this provider. Other possible causes: no Grok " + "subscription, your tier doesn't include this model, or your " + "quota is exhausted. Check https://grok.com/?_s=usage to see " + "which, or run `/model` to switch providers." + ) + # Idempotency: detect prior decoration by a substring unique to the + # hint (not present in xAI's own body text). + if "X Premium+ does NOT include" in detail: + return detail + return f"{detail}{hint}" + + @staticmethod + def _coerce_api_error_detail(value: Any) -> str: + """Return a display-safe string for structured provider error fields.""" + if isinstance(value, str): + return value + if isinstance(value, dict): + for key in ("message", "detail", "error", "code", "type"): + nested = value.get(key) + if isinstance(nested, str) and nested.strip(): + return nested + for key in ("message", "detail", "error", "code", "type"): + if key in value: + nested_detail = ApiErrorSummaryMixin._coerce_api_error_detail(value[key]) + if nested_detail: + return nested_detail + try: + return json.dumps(value, ensure_ascii=False, sort_keys=True) + except TypeError: + return str(value) + if isinstance(value, (list, tuple)): + parts = [ + ApiErrorSummaryMixin._coerce_api_error_detail(item) + for item in value + ] + return "; ".join(part for part in parts if part) + if value is None: + return "" + return str(value) + + @staticmethod + def _summarize_api_error(error: Exception) -> str: + """Extract a human-readable one-liner from an API error. + + Cloudflare HTML pages → ````; network/DNS failures (even SDK-wrapped) → offline hint; else + truncated str(error). + """ + raw = str(error) + + # Offline DNS failures are wrapped in a generic "Connection error" by SDKs — inspect the chain. + network_resolution_markers = ( + "temporary failure in name resolution", + "name or service not known", + "nodename nor servname provided, or not known", + "getaddrinfo failed", + "no address associated with hostname", + "network is unreachable", + ) + current: Optional[BaseException] = error + seen: set[int] = set() + while current is not None and id(current) not in seen: + seen.add(id(current)) + if any( + marker in str(current).lower() + for marker in network_resolution_markers + ): + return ( + "Hermes can't reach the model provider. You may be offline. " + "Check your internet connection and try again." + ) + current = current.__cause__ or current.__context__ + + if ( + isinstance(error, ValueError) + and "expected ident at line" in raw.lower() + ): + return f"Malformed provider streaming response: {raw[:300]}" + + # Cloudflare / proxy HTML pages: grab the <title> for a clean summary + if "<!DOCTYPE" in raw or "<html" in raw: + m = re.search(r"<title[^>]*>([^<]+)", raw, re.IGNORECASE) + title = m.group(1).strip() if m else "HTML error page (title not found)" + # Also grab Cloudflare Ray ID if present + ray = re.search(r"Cloudflare Ray ID:\s*]*>([^<]+)", raw) + ray_id = ray.group(1).strip() if ray else None + status_code = getattr(error, "status_code", None) + parts = [] + if status_code: + parts.append(f"HTTP {status_code}") + parts.append(title) + if ray_id: + parts.append(f"Ray {ray_id}") + return " — ".join(parts) + + # GeminiAPIError already composes a clean one-liner with guidance; don't re-extract the raw body. + if type(error).__name__ == "GeminiAPIError": + return redact_sensitive_text(raw[:1000]) + + # JSON body errors from OpenAI/Anthropic SDKs + body = getattr(error, "body", None) + if isinstance(body, dict): + msg = body.get("error", {}).get("message") if isinstance(body.get("error"), dict) else body.get("message") + if msg: + status_code = getattr(error, "status_code", None) + prefix = f"HTTP {status_code}: " if status_code else "" + msg = ApiErrorSummaryMixin._coerce_api_error_detail(msg) + return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}") + + # SDK may leave body empty while httpx has the payload (#36109). Redact: the body is + # attacker-influenced and may echo Authorization / x-api-key / request JSON. + response = getattr(error, "response", None) + if response is not None: + try: + snippet = (getattr(response, "text", None) or "").strip() + except Exception: + snippet = "" + if snippet: + status_code = getattr(error, "status_code", None) + prefix = f"HTTP {status_code}: " if status_code else "" + try: + payload = json.loads(snippet) + except (json.JSONDecodeError, TypeError): + payload = None + if isinstance(payload, dict): + err = payload.get("error") + if isinstance(err, dict) and err.get("message"): + return redact_sensitive_text(f"{prefix}{str(err['message'])[:300]}") + if payload.get("message"): + return redact_sensitive_text(f"{prefix}{str(payload['message'])[:300]}") + return redact_sensitive_text(f"{prefix}{snippet[:300]}") + + # Fallback: truncate the raw string but give more room than 200 chars + status_code = getattr(error, "status_code", None) + prefix = f"HTTP {status_code}: " if status_code else "" + return ApiErrorSummaryMixin._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}") + + def _mask_api_key_for_logs(self, key: Any) -> Optional[str]: + # Azure Foundry Entra ID bearer providers are callables — never + # invoke them in log paths; identify the auth surface instead. + if callable(key) and not isinstance(key, str): + return "" + if not key: + return None + if len(key) <= 12: + return "***" + return f"{key[:8]}...{key[-4:]}" + + def _clean_error_message(self, error_msg: str) -> str: + """Clean up error messages for user display, removing HTML content and truncating.""" + if not error_msg: + return "Unknown error" + + # Remove HTML content (common with CloudFlare and gateway error pages) + if error_msg.strip().startswith(' 150: + cleaned = cleaned[:150] + "..." + + return cleaned diff --git a/run_agent.py b/run_agent.py index 1767af84ea..4bb6ff417b 100644 --- a/run_agent.py +++ b/run_agent.py @@ -176,6 +176,7 @@ from agent.client_lifecycle import ( # noqa: F401 # _routermint_headers/_qwen_ from agent.stream_delivery import StreamDeliveryMixin from agent.status_output import StatusOutputMixin from agent.api_request_hooks import ApiRequestHooksMixin +from agent.api_error_summary import ApiErrorSummaryMixin from agent.lazy_forward import forward as _forward, forward_static as _forward_static from agent.redact import redact_sensitive_text from agent.session_activity import ActivityProvenance @@ -353,7 +354,13 @@ class _StreamErrorEvent(Exception): } -class AIAgent(ClientLifecycleMixin, StreamDeliveryMixin, StatusOutputMixin, ApiRequestHooksMixin): +class AIAgent( + ClientLifecycleMixin, + StreamDeliveryMixin, + StatusOutputMixin, + ApiRequestHooksMixin, + ApiErrorSummaryMixin, +): """AI Agent with tool calling capabilities.""" _TOOL_CALL_ARGUMENTS_CORRUPTION_MARKER = ( @@ -1830,231 +1837,6 @@ class AIAgent(ClientLifecycleMixin, StreamDeliveryMixin, StatusOutputMixin, ApiR trajectory = self._convert_to_trajectory_format(messages, user_query, completed) _save_trajectory_to_file(trajectory, self.model, completed) - @staticmethod - def _is_entitlement_failure( - error_context: Optional[Dict[str, Any]], - status_code: Optional[int], - ) -> bool: - """Detect subscription/entitlement 401/403s that masquerade as auth failures. - - Refreshing a token cannot fix an unsubscribed account, so callers surface the error instead of looping - the - pool. xAI returns the same permission-denied text for BOTH cases; a ``[WKE=unauthenticated:...]`` - suffix (or - "access token could not be validated") means stale token → return False so the refresh path runs. - """ - if status_code not in {401, 403, None}: - return False - if not isinstance(error_context, dict): - return False - # Single lowercase haystack over every field shape (message/reason and raw code/error). - message = str(error_context.get("message") or "").lower() - reason = str(error_context.get("reason") or "").lower() - code = str(error_context.get("code") or "").lower() - err = str(error_context.get("error") or "").lower() - haystack = f"{message} {reason} {code} {err}" - if not haystack.strip(): - return False - # xAI's disambiguator for stale-token vs unsubscribed: same permission-denied text, only one carries - # this suffix. Bail out so a stale OAuth token takes the credential-refresh path (#29344). - if "[wke=unauthenticated:" in haystack: - return False - if "oauth2 access token could not be validated" in haystack: - return False - if "do not have an active grok subscription" in haystack: - return True - if "out of available resources" in haystack and "grok" in haystack: - return True - if "does not have permission" in haystack and "grok" in haystack: - return True - return False - - @staticmethod - def _decorate_xai_entitlement_error(detail: str) -> str: - """Append a neutral hint when xAI's OAuth surface returns the permission-denied 403. - - xAI's ``/v1/responses`` uses one body for several causes (no subscription, tier lacks the model, quota - exhausted). The least obvious: X Premium+ does NOT include API access — only SuperGrok does. Lead with - that, keep the raw text, point at https://grok.com/?_s=usage. Matched once per detail string. - """ - if not detail: - return detail - lower = detail.lower() - is_entitlement = ( - "do not have an active grok subscription" in lower - or ("out of available resources" in lower and "grok" in lower) - or ("does not have permission" in lower and "grok" in lower) - ) - if not is_entitlement: - return detail - hint = ( - " — xAI rejected this OAuth account. NOTE: X Premium+ does NOT " - "include xAI API access — only standalone SuperGrok subscribers " - "can use this provider. Other possible causes: no Grok " - "subscription, your tier doesn't include this model, or your " - "quota is exhausted. Check https://grok.com/?_s=usage to see " - "which, or run `/model` to switch providers." - ) - # Idempotency: detect prior decoration by a substring unique to the - # hint (not present in xAI's own body text). - if "X Premium+ does NOT include" in detail: - return detail - return f"{detail}{hint}" - - @staticmethod - def _coerce_api_error_detail(value: Any) -> str: - """Return a display-safe string for structured provider error fields.""" - if isinstance(value, str): - return value - if isinstance(value, dict): - for key in ("message", "detail", "error", "code", "type"): - nested = value.get(key) - if isinstance(nested, str) and nested.strip(): - return nested - for key in ("message", "detail", "error", "code", "type"): - if key in value: - nested_detail = AIAgent._coerce_api_error_detail(value[key]) - if nested_detail: - return nested_detail - try: - return json.dumps(value, ensure_ascii=False, sort_keys=True) - except TypeError: - return str(value) - if isinstance(value, (list, tuple)): - parts = [ - AIAgent._coerce_api_error_detail(item) - for item in value - ] - return "; ".join(part for part in parts if part) - if value is None: - return "" - return str(value) - - @staticmethod - def _summarize_api_error(error: Exception) -> str: - """Extract a human-readable one-liner from an API error. - - Cloudflare HTML pages → ````; network/DNS failures (even SDK-wrapped) → offline hint; else - truncated str(error). - """ - raw = str(error) - - # Offline DNS failures are wrapped in a generic "Connection error" by SDKs — inspect the chain. - network_resolution_markers = ( - "temporary failure in name resolution", - "name or service not known", - "nodename nor servname provided, or not known", - "getaddrinfo failed", - "no address associated with hostname", - "network is unreachable", - ) - current: Optional[BaseException] = error - seen: set[int] = set() - while current is not None and id(current) not in seen: - seen.add(id(current)) - if any( - marker in str(current).lower() - for marker in network_resolution_markers - ): - return ( - "Hermes can't reach the model provider. You may be offline. " - "Check your internet connection and try again." - ) - current = current.__cause__ or current.__context__ - - if ( - isinstance(error, ValueError) - and "expected ident at line" in raw.lower() - ): - return f"Malformed provider streaming response: {raw[:300]}" - - # Cloudflare / proxy HTML pages: grab the <title> for a clean summary - if "<!DOCTYPE" in raw or "<html" in raw: - m = re.search(r"<title[^>]*>([^<]+)", raw, re.IGNORECASE) - title = m.group(1).strip() if m else "HTML error page (title not found)" - # Also grab Cloudflare Ray ID if present - ray = re.search(r"Cloudflare Ray ID:\s*]*>([^<]+)", raw) - ray_id = ray.group(1).strip() if ray else None - status_code = getattr(error, "status_code", None) - parts = [] - if status_code: - parts.append(f"HTTP {status_code}") - parts.append(title) - if ray_id: - parts.append(f"Ray {ray_id}") - return " — ".join(parts) - - # GeminiAPIError already composes a clean one-liner with guidance; don't re-extract the raw body. - if type(error).__name__ == "GeminiAPIError": - return redact_sensitive_text(raw[:1000]) - - # JSON body errors from OpenAI/Anthropic SDKs - body = getattr(error, "body", None) - if isinstance(body, dict): - msg = body.get("error", {}).get("message") if isinstance(body.get("error"), dict) else body.get("message") - if msg: - status_code = getattr(error, "status_code", None) - prefix = f"HTTP {status_code}: " if status_code else "" - msg = AIAgent._coerce_api_error_detail(msg) - return AIAgent._decorate_xai_entitlement_error(f"{prefix}{msg[:300]}") - - # SDK may leave body empty while httpx has the payload (#36109). Redact: the body is - # attacker-influenced and may echo Authorization / x-api-key / request JSON. - response = getattr(error, "response", None) - if response is not None: - try: - snippet = (getattr(response, "text", None) or "").strip() - except Exception: - snippet = "" - if snippet: - status_code = getattr(error, "status_code", None) - prefix = f"HTTP {status_code}: " if status_code else "" - try: - payload = json.loads(snippet) - except (json.JSONDecodeError, TypeError): - payload = None - if isinstance(payload, dict): - err = payload.get("error") - if isinstance(err, dict) and err.get("message"): - return redact_sensitive_text(f"{prefix}{str(err['message'])[:300]}") - if payload.get("message"): - return redact_sensitive_text(f"{prefix}{str(payload['message'])[:300]}") - return redact_sensitive_text(f"{prefix}{snippet[:300]}") - - # Fallback: truncate the raw string but give more room than 200 chars - status_code = getattr(error, "status_code", None) - prefix = f"HTTP {status_code}: " if status_code else "" - return AIAgent._decorate_xai_entitlement_error(f"{prefix}{raw[:500]}") - - def _mask_api_key_for_logs(self, key: Any) -> Optional[str]: - # Azure Foundry Entra ID bearer providers are callables — never - # invoke them in log paths; identify the auth surface instead. - if callable(key) and not isinstance(key, str): - return "" - if not key: - return None - if len(key) <= 12: - return "***" - return f"{key[:8]}...{key[-4:]}" - - def _clean_error_message(self, error_msg: str) -> str: - """Clean up error messages for user display, removing HTML content and truncating.""" - if not error_msg: - return "Unknown error" - - # Remove HTML content (common with CloudFlare and gateway error pages) - if error_msg.strip().startswith(' 150: - cleaned = cleaned[:150] + "..." - - return cleaned - _extract_api_error_context = _forward_static("agent.agent_runtime_helpers", "extract_api_error_context") _dump_api_request_debug = _forward("agent.agent_runtime_helpers", "dump_api_request_debug")