Files
hermes-agent/tools/mcp_tool_errors.py

441 lines
22 KiB
Python

"""MCP connection/transport error classification: URL validation, TLS client certs, identity
headers, redirect header stripping, exception-group unwrapping, auth/session-expired/
method-not-found detection and connect-error formatting. Split from tools/mcp_tool.py."""
import asyncio
import contextlib
import errno
import importlib
import logging
import os
import re
from typing import Any, List, Optional
from urllib.parse import urlparse
from tools.mcp_tool_common import _sanitize_error, _core
logger = logging.getLogger("tools.mcp_tool")
# Stateless (2026-07-28) servers reject a legacy ``initialize`` with this or plain method-not-found.
_JSONRPC_UNSUPPORTED_PROTOCOL_VERSION = -32022
def _jsonrpc_matches(exc: BaseException, codes: tuple, markers: tuple, code=None) -> bool:
"""Structural ``MCPError.error.code`` (or *code*) in *codes*, else any *marker* in ``str(exc).lower()``. Never
``isinstance`` on SDK exception types: they arrive wrapped in ExceptionGroups and drift across generations."""
code = getattr(getattr(exc, "error", None), "code", None) or code
return code in codes or any(marker in str(exc).lower() for marker in markers)
def _handshake_rejected_as_modern(exc: BaseException) -> bool:
"""True when a failed ``initialize`` signals a stateless-only (2026-07-28) server."""
return _jsonrpc_matches(
exc, (_JSONRPC_UNSUPPORTED_PROTOCOL_VERSION, _core._JSONRPC_METHOD_NOT_FOUND),
("unsupported protocol version", str(_JSONRPC_UNSUPPORTED_PROTOCOL_VERSION)),
code=getattr(exc, "code", None)) or _is_method_not_found_error(exc)
def _is_method_not_found_error(exc: BaseException) -> bool:
"""True if *exc* is a JSON-RPC ``method not found`` (-32601; ``ping`` is optional in MCP). The
substring fallback includes "Unknown method: <name>" — without it the ping→list_tools keepalive
fallback never latches and reconnect-loops.
The substring fallback matters when a server reports method-not-found without a structural ``-32601``
code (e.g. surfaced as a plain exception string). Besides the canonical "method not found", many
JSON-RPC implementations phrase it as "Unknown method: <name>" — agentmemory's MCP server is one such
case (#50028).
"""
return _jsonrpc_matches(
exc, (_core._JSONRPC_METHOD_NOT_FOUND,),
(str(_core._JSONRPC_METHOD_NOT_FOUND), "method not found", "unknown method", "not found: ping"))
class InvalidMcpUrlError(ValueError):
"""A remote MCP server's ``url`` is not parseable http(s):// — validated once at startup to fail fast.
Validated once at startup so we fail fast with a clear message instead of burning through the
reconnect-backoff loop on every attempt. (Ported from anomalyco/opencode#25019.)
"""
class NonMcpEndpointError(ConnectionError):
"""An HTTP MCP URL served a non-MCP 2xx (e.g. ``text/html``). Non-retryable: every attempt gets
the same page, so backoff is skipped and the server fails immediately. Subclasses ConnectionError
so broad catches still see a connection problem."""
# Streamable-HTTP rejection statuses an SSE-only server (or its load balancer) produces for the
# chunked ``initialize`` POST: Bad Request, Method Not Allowed, Not Acceptable, Length Required.
_STREAMABLE_REJECT_STATUSES = (400, 405, 406, 411)
def _is_streamable_http_rejection(exc: BaseException) -> bool:
"""True when a Streamable-HTTP connect failure looks like a transport mismatch rather than a
broken server: a 400-family rejection of the initialize POST, or the SDK's opaque INTERNAL_ERROR
(-32603 ``Server returned an error response``) it maps such rejections to on mcp >= 2.0 (error
class per PR #104363, @RohithPariki). Timeouts and auth errors never qualify — neither carries
these markers — so a slow or 401ing server is not retried on the wrong transport.
"""
root = _unwrap_exception_group(exc)
if getattr(getattr(root, "response", None), "status_code", None) in _STREAMABLE_REJECT_STATUSES:
return True
code = getattr(getattr(root, "error", None), "code", None)
return code == -32603 and "server returned an error response" in str(root).lower()
def _unwrap_exception_group(exc: BaseException) -> BaseException:
"""Root-cause leaf of anyio ``(Base)ExceptionGroup`` wrappers (group ``str()`` is opaque). A
``KeyboardInterrupt``/``SystemExit`` leaf anywhere is re-raised, never flattened into a loggable
error; a non-cancellation leaf is preferred over the ``CancelledError`` noise anyio sprays on siblings."""
while isinstance(exc, BaseExceptionGroup) and exc.exceptions:
leaf: BaseException = exc.split((KeyboardInterrupt, SystemExit))[0]
if leaf is not None:
while isinstance(leaf, BaseExceptionGroup) and leaf.exceptions:
leaf = leaf.exceptions[0]
raise leaf
exc = next((sub for sub in exc.exceptions if not _contains_only_cancellation(sub)), exc.exceptions[0])
return exc
def _contains_only_cancellation(exc: BaseException) -> bool:
"""True if ``exc`` is (or a group containing only) CancelledError."""
if isinstance(exc, BaseExceptionGroup):
return all(_contains_only_cancellation(sub) for sub in exc.exceptions)
return isinstance(exc, asyncio.CancelledError)
def _classify_mcp_failure(exc: BaseException) -> str:
"""``'permanent'`` (``run()`` parks instead of burning the retry ladder: auth 401/403,
NonMcpEndpointError, InvalidMcpUrlError, missing stdio command) or ``'transient'`` (backoff retry)."""
root = _unwrap_exception_group(exc)
permanent = (_is_auth_error(root)
or isinstance(root, (NonMcpEndpointError, InvalidMcpUrlError, FileNotFoundError))
or (isinstance(root, OSError) and getattr(root, "errno", None) == errno.ENOENT)
# 401/403 HTTPStatusError that _is_auth_error's type-gate missed (auth types not importable here)
or getattr(getattr(root, "response", None), "status_code", None) in (401, 403))
return "permanent" if permanent else "transient"
def _validate_remote_mcp_url(server_name: str, url: Any) -> str:
"""The stripped URL if valid http(s); else InvalidMcpUrlError naming the server (non-string, other scheme —
stdio servers use ``command`` — or empty host)."""
def _bad(detail: str) -> InvalidMcpUrlError:
return InvalidMcpUrlError(f"Invalid MCP URL for '{server_name}': {detail}")
if not isinstance(url, str):
raise _bad(f"expected a string, got {type(url).__name__}")
stripped = url.strip()
if not stripped:
raise _bad("empty url")
try:
parsed = urlparse(stripped)
except Exception as exc: # urlparse is very permissive — belt and braces
raise _bad(f"{stripped!r} ({exc})") from exc
if parsed.scheme.lower() not in {"http", "https"}:
raise _bad(f"scheme must be http or https, got {parsed.scheme!r} ({stripped!r})")
if not parsed.netloc:
raise _bad(f"missing host ({stripped!r})")
if not parsed.hostname: # ``urlparse`` accepts ``http://:8080`` (empty host, explicit port)
raise _bad(f"missing hostname ({stripped!r})")
return stripped
def _resolve_client_cert(server_name: str, config: dict):
"""``client_cert`` / ``client_key`` in httpx's ``cert=`` shape: None, a combined-PEM path,
``(cert, key)`` or ``(cert, key, password)``. ``~`` is expanded; missing files raise a
server-scoped FileNotFoundError instead of an opaque TLS handshake error."""
raw_cert = config.get("client_cert")
raw_key = config.get("client_key")
if raw_cert is None and raw_key is None:
return None
prefix = f"MCP server '{server_name}': "
def _expand(path: Any, label: str) -> str:
if not isinstance(path, str) or not path.strip():
raise ValueError(f"{prefix}{label} must be a non-empty string path (got {type(path).__name__})")
expanded = os.path.expanduser(path.strip())
if not os.path.isfile(expanded):
raise FileNotFoundError(f"{prefix}{label} not found at {expanded!r}")
return expanded
if not isinstance(raw_cert, (list, tuple)):
cert_path = _expand(raw_cert, "client_cert")
return (cert_path, _expand(raw_key, "client_key")) if raw_key is not None else cert_path # combined PEM
if raw_key is not None:
raise ValueError(f"{prefix}specify either client_cert as a list [cert, key] OR client_cert + client_key, not both")
if len(raw_cert) not in (2, 3):
raise ValueError(f"{prefix}client_cert list form must have 2 or 3 elements (got {len(raw_cert)})")
pair = (_expand(raw_cert[0], "client_cert[0]"), _expand(raw_cert[1], "client_cert[1]"))
if len(raw_cert) == 2:
return pair
if not isinstance(raw_cert[2], str):
raise ValueError(f"{prefix}client_cert[2] (key passphrase) must be a string")
return (*pair, raw_cert[2])
def _resolve_identity_header(server_name: str, config: dict):
"""``identity_header`` ``{name, value_from: "static"|"profile", value}`` → ``(name, value)`` or
None. Invalid configs warn and are ignored — an identity header must never break the connection.
``profile`` resolves once at connect time."""
raw = config.get("identity_header")
if raw is None:
return None
def _ignore(detail: str, *args):
logger.warning("MCP server '%s': identity_header " + detail + " — ignoring", server_name, *args)
return None
if not isinstance(raw, dict):
return _ignore("must be a mapping with 'name' and 'value'/'value_from' keys (got %s)", type(raw).__name__)
name = raw.get("name")
if not isinstance(name, str) or not name.strip():
return _ignore("requires a non-empty 'name'")
value_from = (raw.get("value_from") or "static").strip().lower()
if value_from == "profile":
from hermes_cli.profiles import get_active_profile_name
return (name.strip(), get_active_profile_name())
if value_from != "static":
return _ignore("value_from must be 'static' or 'profile' (got %r)", value_from)
value = raw.get("value")
if not isinstance(value, str) or not value.strip():
return _ignore("with value_from: static requires a non-empty string 'value'")
return (name.strip(), value)
def _apply_identity_header(server_name: str, config: dict, headers: dict) -> dict:
"""Merge the identity header into ``headers`` in place; an explicit entry of the same name (any
casing) wins — never silently override user config."""
name, value = _resolve_identity_header(server_name, config) or (None, None)
if name is None:
return headers
if any(key.lower() == name.lower() for key in headers):
logger.debug("MCP server '%s': identity_header '%s' already set via explicit "
"headers config — keeping the explicit value", server_name, name)
else:
headers[name] = value
return headers
def _make_redirect_header_stripper(original_url, *, strict: bool = False,
configured_header_names: "set[str] | frozenset[str]" = frozenset()):
"""httpx response hook: strips ``Authorization`` when a redirect leaves the original origin;
with *strict* (Agent Plugins v1 ``strict_redirect_headers``) every configured header (lowercase
names in *configured_header_names*) is stripped too — v1 forbids forwarding them cross-origin."""
origin = (original_url.scheme, original_url.host, original_url.port)
async def _strip_on_cross_origin_redirect(response):
target = response.next_request.url if response.is_redirect and response.next_request else None
if target is None or (target.scheme, target.host, target.port) == origin:
return
headers = response.next_request.headers
headers.pop("authorization", None)
headers.pop("Authorization", None)
for _name in configured_header_names if strict else ():
while _name in headers:
del headers[_name]
return _strip_on_cross_origin_redirect
# Wire-body cap, applied at the httpx transport before the SDK buffers/JSON-parses a response. A
# hostile or misbehaving remote MCP server can stream an unbounded catalog/tool-result body and none
# of the post-parse limits (resource cap, tool-result truncation) run before the parse blows up.
# Finite HTTP bodies are capped at this many bytes (a larger Content-Length is rejected up front);
# each SSE *event* is capped, with the counter reset at completed event boundaries so a long-lived
# stream and its keepalives have no cumulative limit. Violations raise the SDK httpx's ReadError and
# flow through the ordinary transport teardown/reconnect path (#66092).
_MCP_HTTP_MAX_BODY_BYTES = 10 * 1024 * 1024
_SSE_EVENT_BOUNDARIES = (b"\n\n", b"\r\n\r\n")
def _make_mcp_body_cap_transport(httpx_mod, inner_transport, limit: int = _MCP_HTTP_MAX_BODY_BYTES):
"""Wrap ``inner_transport`` so every response body is size-capped. ``httpx_mod`` must be the SDK's
own httpx module (``sdk_httpx()``): the transport is handed to that SDK's ``AsyncClient``."""
class _CappedStream(httpx_mod.AsyncByteStream):
def __init__(self, inner, is_sse: bool, url: str):
self._inner, self._is_sse, self._url = inner, is_sse, url
def _reject(self, kind: str):
return httpx_mod.ReadError(f"MCP {kind} exceeds {limit} bytes (from {self._url})")
async def __aiter__(self):
counted = 0
async for chunk in self._inner:
if self._is_sse:
# Bytes up to the last completed event boundary belong to finished events (they must
# still fit the per-event cap together with the carried prefix); the remainder starts
# the next event's budget.
boundary_end = max(chunk.rfind(sep) + len(sep) if sep in chunk else -1 for sep in _SSE_EVENT_BOUNDARIES)
if boundary_end != -1:
if counted + boundary_end > limit:
raise self._reject("SSE event")
counted = len(chunk) - boundary_end
else:
counted += len(chunk)
else:
counted += len(chunk)
if counted > limit:
raise self._reject("SSE event" if self._is_sse else "HTTP response")
yield chunk
async def aclose(self):
await self._inner.aclose()
class _BodyCapTransport(httpx_mod.AsyncBaseTransport):
def __init__(self, inner):
self._inner = inner
async def handle_async_request(self, request):
response = await self._inner.handle_async_request(request)
declared = response.headers.get("content-length")
with contextlib.suppress(ValueError): # malformed header: the streamed cap still applies
if declared is not None and int(declared) > limit:
await response.aclose()
raise httpx_mod.ReadError(f"MCP HTTP response declares Content-Length {declared} > {limit} "
f"bytes cap (from {request.url})")
is_sse = "text/event-stream" in response.headers.get("content-type", "").lower()
response.stream = _CappedStream(response.stream, is_sse, str(request.url))
return response
async def aclose(self):
await self._inner.aclose()
return _BodyCapTransport(inner_transport)
def _exc_children(exc: BaseException) -> List[BaseException]:
"""Sub-exceptions of a group, else ``__cause__``/``__context__`` when they are exceptions."""
nested = getattr(exc, "exceptions", None)
return list(nested) if nested else [c for c in (exc.__cause__, exc.__context__) if isinstance(c, BaseException)]
def _format_connect_error(exc: BaseException) -> str:
"""Render nested MCP connection errors into an actionable short message."""
def _find_missing() -> Optional[str]:
"""Find a missing executable without recursing through malformed chains."""
stack = [exc]
seen: set[int] = set()
budget = _EXC_TRAVERSAL_MAX_NODES
while stack and budget > 0:
current = stack.pop()
if id(current) in seen:
continue
seen.add(id(current))
budget -= 1
if isinstance(current, FileNotFoundError):
if getattr(current, "filename", None):
return str(current.filename)
match = re.search(r"No such file or directory: '([^']+)'", str(current))
if match:
return match.group(1)
# Reverse preserves the former left-to-right depth-first order.
stack.extend(reversed(_exc_children(current)))
return None
def _flatten_messages() -> List[str]:
"""Collect a short, cycle-safe rendering of an exception chain."""
stack = [exc]
seen: set[int] = set()
messages: List[str] = []
budget = _EXC_TRAVERSAL_MAX_NODES
while stack and budget > 0:
current = stack.pop()
if id(current) in seen:
continue
seen.add(id(current))
budget -= 1
children = _exc_children(current)
# A group's own str() is opaque — only its children speak.
text = "" if getattr(current, "exceptions", None) else str(current).strip()
if text:
messages.append(text)
elif not children:
messages.append(current.__class__.__name__)
stack.extend(reversed(children))
return messages or [exc.__class__.__name__]
missing = _find_missing()
if not missing:
return _sanitize_error("; ".join(list(dict.fromkeys(_flatten_messages()))[:3]))
message = f"missing executable '{missing}'"
if os.path.basename(missing) in {"npx", "npm", "node"}:
message += (" (ensure Node.js is installed and PATH includes its bin directory, "
"or set mcp_servers.<name>.command to an absolute path and include "
"that directory in mcp_servers.<name>.env.PATH)")
return _sanitize_error(message)
def _optional_types(module: str, *names: str) -> list:
"""``[module.name, ...]`` or ``[]`` when the module/attribute is unavailable."""
try:
mod = importlib.import_module(module)
return [getattr(mod, name) for name in names]
except (ImportError, AttributeError):
return []
# Lazily-built ``(auth_types, http_status_types)`` so this module imports without the SDK OAuth module.
_AUTH_ERROR_TYPES: Optional[tuple] = None
def _get_auth_error_types() -> tuple:
"""Cached ``(auth_types, http_status_types)``: SDK ``OAuthFlowError``/``OAuthTokenError`` (+ legacy
``UnauthorizedError``), our ``OAuthNonInteractiveError``, and ``HTTPStatusError`` from both httpx
flavours — a 401 may come from the SDK's own stack (``httpx2`` on mcp >= 2.0) or Hermes' pinned
``httpx``; the classes are unrelated and still need the 401 check in :func:`_is_auth_error`."""
global _AUTH_ERROR_TYPES
if not (_AUTH_ERROR_TYPES and _AUTH_ERROR_TYPES[0]): # retry while empty (SDK may import later)
sdk_mod = _core.sdk_httpx()
http_types = tuple(dict.fromkeys(
([sdk_mod.HTTPStatusError] if sdk_mod is not None else []) + _optional_types("httpx", "HTTPStatusError")))
auth_types = (*_optional_types("mcp.client.auth", "OAuthFlowError", "OAuthTokenError"),
*_optional_types("mcp.client.auth", "UnauthorizedError"), # older SDKs
*_optional_types("tools.mcp_oauth", "OAuthNonInteractiveError"), *http_types)
_AUTH_ERROR_TYPES = (auth_types, http_types)
return _AUTH_ERROR_TYPES
def _is_auth_error(exc: BaseException) -> bool:
"""True if ``exc`` indicates an MCP OAuth failure; ``HTTPStatusError`` counts only with status 401."""
auth_types, http_types = _get_auth_error_types()
if not isinstance(exc, auth_types):
return False
return getattr(exc.response, "status_code", None) == 401 if isinstance(exc, http_types) else True
# Lower-cased substrings meaning the transport session expired / was GC'd (OAuth token still valid).
# Substrings (lower-cased match) that indicate the MCP server rejected the request because its server-side
# transport session expired / was garbage-collected. See #13383.
_SESSION_EXPIRED_MARKERS: tuple = (
"invalid or expired session", "expired session", "session expired", "session not found",
"unknown session", "session terminated", "closedresourceerror", "closed resource",
"transport is closed", "connection closed", "broken pipe", "end of file")
# Node budget for ``_is_session_expired_error`` (the visited set breaks cycles; this bounds acyclic blow-ups).
# Well above ``sys.getrecursionlimit()`` so deep task-group nesting is fully scanned.
_EXC_TRAVERSAL_MAX_NODES = 10_000
def _is_session_expired_error(exc: BaseException) -> bool:
"""True if ``exc`` looks like a transport session expiry (Streamable-HTTP servers GC session state on idle TTL /
restart / pod rotation while the OAuth token stays valid) — the fix is a transport reconnect, not an OAuth
refresh. Iterative walk over ``exceptions`` / ``__cause__`` / ``__context__`` with a visited set AND a node
budget; every reachable node is inspected so an InterruptedError anywhere overrides transport markers, and the
chain walk matters because SDK wrappers raise a generic RuntimeError *from* a message-less ClosedResourceError."""
# AnyIO stream exceptions are often message-less, so type checks complement marker matching.
transport_error_types = tuple(_optional_types("anyio", "BrokenResourceError", "ClosedResourceError", "EndOfStream"))
stack: "list[BaseException | None]" = [exc]
seen: set[int] = set()
found = False
budget = _EXC_TRAVERSAL_MAX_NODES
while stack and budget > 0:
current = stack.pop()
if current is None or id(current) in seen:
continue
seen.add(id(current))
budget -= 1
if isinstance(current, InterruptedError):
return False
# Messages vary across SDK versions/servers: a narrow allow-list of stable substrings avoids false positives.
msg = str(current).lower()
found = found or isinstance(current, transport_error_types) or any(m in msg for m in _SESSION_EXPIRED_MARKERS)
stack.extend((*getattr(current, "exceptions", ()), getattr(current, "__cause__", None),
getattr(current, "__context__", None)))
return found