hermes-agent/tests/tools/test_mcp_failure_classification.py

"""Tests for exception-group unwrapping and failure classification in
``tools/mcp_tool.py`` (#65673, #66092).

The MCP SDK's anyio TaskGroups wrap real errors in ``BaseExceptionGroup``,
whose ``str()`` is "unhandled errors in a TaskGroup (N sub-exceptions)" —
useless in logs. ``_unwrap_exception_group`` digs out the root cause;
``_classify_mcp_failure`` decides whether a failure is worth retrying.
"""

import asyncio
import errno
import logging

import pytest

from tools.mcp_tool_errors import (
    InvalidMcpUrlError, NonMcpEndpointError, _auth_error_detail, _classify_mcp_failure, _mcp_call_failed_message,
    _unwrap_exception_group)
from tools.mcp_tool import MCPServerTask


def _group(*excs, msg="unhandled errors in a TaskGroup") -> BaseExceptionGroup:
    return BaseExceptionGroup(msg, list(excs))


# ── _unwrap_exception_group ──────────────────────────────────────────────────

class TestUnwrapExceptionGroup:
    def test_plain_exception_passes_through(self):
        exc = ConnectionError("boom")
        assert _unwrap_exception_group(exc) is exc

    def test_single_level_group(self):
        inner = BrokenPipeError()
        assert _unwrap_exception_group(_group(inner)) is inner


    def test_system_exit_reraises(self):
        with pytest.raises(SystemExit):
            _unwrap_exception_group(_group(SystemExit(2)))


# ── _classify_mcp_failure ────────────────────────────────────────────────────

class TestClassifyMcpFailure:
    @pytest.mark.parametrize("exc", [
        ConnectionError("connection refused"),
        ConnectionResetError("reset by peer"),
        BrokenPipeError(),
        EOFError(),
        TimeoutError("read timeout"),
        OSError(errno.ECONNRESET, "reset"),
        RuntimeError("something odd"),
    ])
    def test_transient_failures(self, exc):
        assert _classify_mcp_failure(exc) == "transient"


    def test_closed_resource_transient(self):
        anyio = pytest.importorskip("anyio")
        assert _classify_mcp_failure(anyio.ClosedResourceError()) == "transient"


    def test_permanent_inside_taskgroup(self):
        # Classification must apply to the UNWRAPPED root cause.
        g = _group(_group(FileNotFoundError("cmd not found")))
        assert _classify_mcp_failure(g) == "permanent"


# ── _auth_error_detail / _mcp_call_failed_message ────────────────────────────

class TestAuthErrorSurfacing:
    def test_registration_detail_surfaces(self):
        """The actionable registration guidance raised by the DCR guard must
        reach the needs_reauth tool error (GH#78190)."""
        pytest.importorskip("mcp.client.auth")
        from mcp.client.auth import OAuthRegistrationError

        exc = OAuthRegistrationError(
            "MCP OAuth 'gmail': this provider does not support automatic "
            "client registration ... add it under config.yaml "
            "mcp_servers.<name>.oauth (client_id, client_secret), then run "
            "`hermes mcp login gmail`."
        )
        detail = _auth_error_detail(exc)
        assert detail.startswith(" ")
        assert "config.yaml" in detail
        assert "client_id" in detail

    def test_registration_detail_unwraps_group(self):
        """The transport raises the auth error inside a TaskGroup wrapper —
        the detail must still be found."""
        pytest.importorskip("mcp.client.auth")
        from mcp.client.auth import OAuthRegistrationError

        exc = OAuthRegistrationError("config.yaml guidance")
        assert "config.yaml" in _auth_error_detail(_group(exc))

    def test_non_registration_no_detail(self):
        assert _auth_error_detail(ValueError("nope")) == ""
        assert _auth_error_detail(_group(TimeoutError("t"))) == ""

    def test_mcp_call_failed_message_unwraps_group(self):
        """The generic error path must surface the group root cause instead
        of the opaque TaskGroup wrapper text."""
        pytest.importorskip("mcp.client.auth")
        from mcp.client.auth import OAuthRegistrationError

        exc = OAuthRegistrationError("registration guidance text")
        msg = _mcp_call_failed_message(_group(exc))
        assert "OAuthRegistrationError" in msg
        assert "registration guidance text" in msg
        assert "TaskGroup" not in msg

    def test_needs_reauth_tool_error_carries_guidance(self, monkeypatch):
        """Through the real recovery handler: an unrecoverable registration
        failure returns needs_reauth WITH the guard's guidance."""
        pytest.importorskip("mcp.client.auth")
        import json
        from mcp.client.auth import OAuthRegistrationError
        from tools import mcp_tool_handlers as handlers

        monkeypatch.setattr(handlers._loop, "_run_on_mcp_loop", lambda *_a, **_k: False)
        out = json.loads(handlers._handle_auth_error_and_retry(
            "gmail", OAuthRegistrationError("Create an OAuth client (config.yaml)."),
            lambda: None, "call_tool"))
        assert out["needs_reauth"] is True
        assert "requires re-authentication. Create an OAuth client (config.yaml). Run" in out["error"]


# ── Keepalive failure log surfaces the root cause ────────────────────────────

@pytest.mark.no_isolate
def test_keepalive_failure_logs_root_cause(monkeypatch, tmp_path, caplog):
    """A keepalive that dies with a TaskGroup-wrapped BrokenPipeError (empty
    str) must log 'BrokenPipeError', not 'unhandled errors in a TaskGroup'."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path))

    from tools import mcp_tool

    class _Task(MCPServerTask):
        async def _keepalive_probe(self):
            raise _group(BrokenPipeError())

    task = _Task("pipey")
    task._config = {"keepalive_interval": 0.01}
    task.session = object()

    monkeypatch.setattr(mcp_tool, "_MIN_KEEPALIVE_INTERVAL", 0.01)

    async def _scenario():
        with caplog.at_level(logging.WARNING, logger="tools.mcp_tool"):
            reason = await task._wait_for_lifecycle_event()
        assert reason == "reconnect"

    asyncio.run(_scenario())

    keepalive_logs = [
        r.getMessage() for r in caplog.records if "keepalive failed" in r.getMessage()
    ]
    assert keepalive_logs, "keepalive failure was not logged"
    assert any("BrokenPipeError" in m for m in keepalive_logs), keepalive_logs
    assert not any("unhandled errors in a TaskGroup" in m for m in keepalive_logs)


# ── run() parks permanent failures immediately ───────────────────────────────

@pytest.mark.no_isolate
def test_permanent_failure_parks_without_retry_ladder(monkeypatch, tmp_path, caplog):
    """A stdio command that doesn't exist (FileNotFoundError) must park after
    ONE attempt — not burn _MAX_INITIAL_CONNECT_RETRIES identical warnings."""
    monkeypatch.setenv("HERMES_HOME", str(tmp_path))

    from tools import mcp_tool

    _real_sleep = asyncio.sleep

    async def _fast_sleep(_delay, *a, **kw):
        await _real_sleep(0)

    monkeypatch.setattr(mcp_tool.asyncio, "sleep", _fast_sleep)

    state = {"transport_calls": 0, "parked": False}

    async def _scenario():
        class _Task(MCPServerTask):
            def _is_http(self):
                return False

            def _deregister_tools(self):
                state["parked"] = True
                self._registered_tool_names = []

            async def _run_stdio(self, config):
                state["transport_calls"] += 1
                raise FileNotFoundError("nonexistent-mcp-command")

        task = _Task("missing-cmd")

        with caplog.at_level(logging.DEBUG, logger="tools.mcp_tool"):
            run_task = asyncio.ensure_future(task.run({"command": "nope"}))
            for _ in range(500):
                await _real_sleep(0)
                if state["parked"]:
                    break

        assert state["parked"], "permanent failure never parked"
        assert state["transport_calls"] == 1, (
            f"permanent failure burned {state['transport_calls']} attempts — "
            "should park immediately"
        )

        task._shutdown_event.set()
        task._reconnect_event.set()
        try:
            await asyncio.wait_for(run_task, timeout=15)
        except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
            run_task.cancel()

    asyncio.run(_scenario())

    park_warnings = [
        r for r in caplog.records
        if r.levelno == logging.WARNING and "permanent error" in r.getMessage()
    ]
    assert len(park_warnings) == 1
    assert "FileNotFoundError" in park_warnings[0].getMessage()


# ── An initial 401 must stay revivable ───────────────────────────────────────

@pytest.mark.no_isolate
def test_initial_auth_failure_parks_and_revives_after_relogin(
    monkeypatch, tmp_path, caplog,
):
    """A 401 on the FIRST connect must park, not end the run task.

    Ending the task drops the only listener on ``_reconnect_event``, so the
    server stayed dead for the life of the process even after the user
    re-authenticated. Parking keeps it revivable via the self-probe.
    """
    monkeypatch.setenv("HERMES_HOME", str(tmp_path))

    httpx = pytest.importorskip("httpx")

    from tools import mcp_tool

    monkeypatch.setattr(mcp_tool, "_PARKED_RETRY_INTERVAL", 0.05)

    from tools import mcp_tool_config as _config
    monkeypatch.setattr(_config, "_load_mcp_config", lambda: {"figma": {"command": "x"}})

    _real_sleep = asyncio.sleep

    async def _fast_sleep(_delay, *a, **kw):
        await _real_sleep(0)

    monkeypatch.setattr(mcp_tool.asyncio, "sleep", _fast_sleep)

    def _auth_error():
        request = httpx.Request("POST", "https://mcp.example.test/mcp")
        response = httpx.Response(401, request=request)
        return httpx.HTTPStatusError("401", request=request, response=response)

    state = {"transport_calls": 0, "parked": False, "authenticated": False}

    async def _scenario():
        class _Task(MCPServerTask):
            def _is_http(self):
                return False

            def _deregister_tools(self):
                state["parked"] = True
                self._registered_tool_names = []

            async def _run_stdio(self, config):
                state["transport_calls"] += 1
                if not state["authenticated"]:
                    raise _group(_auth_error())
                self.session = object()
                await self._wait_for_lifecycle_event()

        task = _Task("figma")

        with caplog.at_level(logging.DEBUG, logger="tools.mcp_tool"):
            run_task = asyncio.ensure_future(task.run({"command": "x"}))
            for _ in range(500):
                await _real_sleep(0)
                if state["parked"]:
                    break

            assert state["parked"], "auth failure never parked"
            assert state["transport_calls"] == 1, (
                f"auth failure burned {state['transport_calls']} attempts"
            )
            assert not run_task.done(), (
                "run task exited on a 401 — the server is now unrevivable"
            )

            # The user re-authenticates. Nothing sets _reconnect_event:
            # revival must come from the timed self-probe alone.
            state["authenticated"] = True
            for _ in range(200):
                await _real_sleep(0.01)
                if task.session is not None:
                    break

        assert task.session is not None, (
            "parked server never recovered after re-authentication "
            f"(transport_calls={state['transport_calls']})"
        )

        task._shutdown_event.set()
        task._reconnect_event.set()
        try:
            await asyncio.wait_for(run_task, timeout=15)
        except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
            run_task.cancel()

    asyncio.run(_scenario())

    auth_warnings = [
        r for r in caplog.records
        if r.levelno == logging.WARNING
        and "failed initial authentication" in r.getMessage()
    ]
    assert len(auth_warnings) == 1
    assert "hermes mcp login figma" in auth_warnings[0].getMessage()