hermes-agent/tests/gateway/test_local_model_connection_reply.py

"""Regression tests for #86570 / #116323: gateway provider error connection messaging."""

import pytest

from gateway.config import Platform
from gateway.run import (
    _GATEWAY_CONNECTION_ERROR_RE,
    _gateway_provider_error_reply,
    _looks_like_gateway_provider_error,
    _sanitize_gateway_final_response,
)

# Terminal-path envelopes for an ESTABLISHED connection that died mid-transfer. Whether the
# endpoint is up is unknowable from these (the same turn may already have been answered by it).
INTERRUPTED_ENVELOPES = (
    "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer",
    "API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer",
    "API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection "
    "without sending complete message body",
    "API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response",
)

# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis
# here, and it is the case the #86570 wording was written for.
UNREACHABLE_ENVELOPES = (
    "API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused",
    "API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could "
    "be made because the target machine actively refused it",
    "API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host",
)

# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported.
AMBIGUOUS_ENVELOPES = (
    "openai.APIConnectionError: Connection error.",
    "API call failed after 3 retries: openai.APIConnectionError: Connection error.",
    "❌ API failed after 3 retries — openai.APIConnectionError: Connection error.",
)

# Claims that the configured endpoint stopped/never came up. Asserting one of these about a
# mid-transfer reset sends the user to debug a server that is answering.
_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down")


def _claims_the_endpoint_is_down(reply: str) -> bool:
    return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS)


class TestGatewayConnectionErrorReply:
    def test_connection_error_strings_produce_specific_reply(self):
        """A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570.

        A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a
        dropped response from a live endpoint. It stays a recognised provider envelope, but the
        endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test).
        """
        samples = [
            "httpx.ConnectError: connection refused",
            "ConnectionError: [WinError 10061] No connection could be made",
            "Errno 111 Connection refused",
            "All connection attempts failed: Connection refused",
        ]
        for text in samples:
            assert _looks_like_gateway_provider_error(text), text
            reply = _gateway_provider_error_reply(text)
            assert reply == _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text
            assert "/retry" in reply, text

        assert _looks_like_gateway_provider_error("openai.APIConnectionError")

    def test_broad_connection_phrases_still_map_once_classified(self):
        """Reply selector keeps the full phrase set; the gate does not."""
        for text in (
            "cannot connect to http://127.0.0.1:8033/v1",
            "failed to establish a new connection",
        ):
            reply = _gateway_provider_error_reply(text)
            assert reply == _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text

    def test_prose_cannot_connect_is_not_a_provider_error(self):
        text = (
            "cannot connect to the office VPN from this cafe, "
            "so I used the backup notes instead"
        )
        assert not _looks_like_gateway_provider_error(text)

    def test_other_errors_keep_generic_reply(self):
        for text in (
            "RuntimeError: model returned empty content",
            "Exception: unknown provider",
            "HTTP 500 internal server error",
        ):
            if _looks_like_gateway_provider_error(text):
                reply = _gateway_provider_error_reply(text)
                assert reply != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text

    def test_connection_regex_does_not_match_non_connection_error(self):
        assert not _GATEWAY_CONNECTION_ERROR_RE.search("Rate limited after 3 retries")
        assert not _GATEWAY_CONNECTION_ERROR_RE.search("Provider authentication failed")

    def test_auth_and_rate_limit_preserved(self):
        auth_reply = _gateway_provider_error_reply("provider authentication failed")
        assert "sign-in" in auth_reply.lower() and "/login" in auth_reply
        assert "rate-limiting" in _gateway_provider_error_reply(
            "rate limited after 3 retries"
        ).lower()

    def test_every_reply_names_a_slash_command_and_no_jargon(self):
        """Each shaped reply must give the chat user something they can run; 'provider' and
        'gateway logs' are operator words (the log pointer is the `hermes logs` command)."""
        from agent.i18n import t
        from gateway.run import _PROVIDER_ERROR_REPLIES
        replies = [t(reply_key) for _, reply_key in _PROVIDER_ERROR_REPLIES] + [_gateway_provider_error_reply("zzz")]
        for reply in replies:
            assert any(cmd in reply for cmd in ("/login", "/retry", "/model")), reply
            assert "provider" not in reply.lower(), reply


    def test_three_connection_causes_are_three_distinct_categories(self):
        """A dropped response, a refused connect and a cause-free error are different failures.

        Contract, not wording: each cause gets ONE reply, the three replies differ, and only the
        refused/unroutable one may claim the endpoint is down (that claim is pinned to the
        refusal samples by ``test_connection_error_strings_produce_specific_reply`` above).
        """
        interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES}
        unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES}
        ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES}

        assert len(interrupted) == 1, interrupted
        assert len(unreachable) == 1, unreachable
        assert len(ambiguous) == 1, ambiguous
        assert len(interrupted | unreachable | ambiguous) == 3

        for reply in interrupted | ambiguous:
            assert reply.strip()
            assert not _claims_the_endpoint_is_down(reply), reply

    @pytest.mark.parametrize("platform", [Platform.TELEGRAM, "slack", "feishu"])
    def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform):
        """The new category rides the real chat path, and takes nothing from the other rows."""
        raw_reset = (
            "API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset "
            "by peer (Authorization: Bearer sk-ABCDEF0123456789abcdef0123)"
        )

        sanitized = _sanitize_gateway_final_response(platform, raw_reset)

        assert sanitized.strip()
        assert "sk-ABCDEF" not in sanitized
        assert "Errno 104" not in sanitized
        assert not _claims_the_endpoint_is_down(sanitized), sanitized
        assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0])

        # Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own
        # marker and connection wording keeps the category it had before the connection split.
        for tainted, clean in (
            ("API call failed after 3 retries: HTTP 401 incorrect api key provided; "
             "connection reset by peer on the retry", "provider authentication failed"),
            ("API call failed after 3 retries: HTTP 400 request was blocked under the provider "
             "safety policy; connection reset by peer", "request was blocked under the safety policy"),
            ("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; "
             "connection reset by peer", "rate limited after 3 retries"),
        ):
            assert _sanitize_gateway_final_response(platform, tainted) == (
                _gateway_provider_error_reply(clean)), tainted

        # Programmatic consumers still get the bottom exception, byte for byte.
        assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset


class TestQuotaExhaustedIsNotAnAuthFailure:
    """A 429/quota envelope must never send the user to re-authenticate valid credentials, and a
    long reset window must be named instead of "wait a moment" (#89401)."""

    def test_quota_envelope_with_auth_preamble_names_reset_window(self):
        reply = _gateway_provider_error_reply(
            "⚠️ Provider authentication failed: Codex provider quota exhausted (429); "
            "retry after 116168s. Credentials are still valid.")
        assert "/login" not in reply and "resets in ~33h" in reply
        body_reply = _gateway_provider_error_reply(
            "API call failed after 3 retries: Error code: 429 - "
            "{'error': {'type': 'usage_limit_reached', 'resets_in_seconds': 30995}}")
        assert "resets in ~9h" in body_reply
        assert "resets in ~5h" in _gateway_provider_error_reply("429 weekly limit reached. Resets in 4hr 5min")
        # A bare 401 inside a timestamp is not a sign-in failure; every real 401 envelope still is,
        # including the ``HTTP 401: Unauthorized`` shape _summarize_api_error emits (control).
        assert "rate-limiting" in _gateway_provider_error_reply("API call failed at 05:14:15,401 status 429")
        assert "/login" not in _gateway_provider_error_reply("request 05:14:15,401 failed")
        for text in ("HTTP 401: Unauthorized", "returned 401.", "HTTP 401, re-authenticate", "Error code: 401 - token rejected"):
            assert "/login" in _gateway_provider_error_reply(text), text

    def test_turn_runner_resolution_quota_failure_names_reset_window_not_login(self, monkeypatch):
        """Production entry: TurnRunner.run_sync with credential resolution raising the quota
        RuntimeError the gateway wraps around a ``codex_rate_limited`` AuthError."""
        from types import SimpleNamespace
        import gateway.run as gateway_run
        from gateway.config import Platform
        from gateway.run_turn_runner import TurnRunner
        from gateway.session import SessionSource
        from gateway.turn_context import TurnContext
        from hermes_cli.auth import AuthError
        from hermes_cli.auth_constants import CODEX_RATE_LIMITED_CODE

        def _resolve(**_kwargs):
            try:
                raise AuthError("Codex provider quota exhausted (429); retry after 116168s. Credentials are still valid.",
                                code=CODEX_RATE_LIMITED_CODE, relogin_required=False)
            except AuthError as exc:
                raise RuntimeError(str(exc)) from exc

        monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 30)
        runner = SimpleNamespace(_resolve_session_agent_runtime=_resolve,
                                 _get_system_prompt_for_channel=lambda *a, **k: "",
                                 _ephemeral_system_prompt="")
        ctx = TurnContext(source=SessionSource(platform=Platform.SLACK, chat_id="C1", chat_type="dm"),
                          session_key="slack:C1", user_config={}, message="Hi")
        result = TurnRunner(runner, ctx).run_sync()
        reply = result["final_response"]
        assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0