From 993d1b9891ea8b639502b43f9d4943dccb0635ed Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 18 Sep 2026 04:17:37 -0700 Subject: [PATCH] fix(status): dashboard keeps a watchdog-exited gateway `degraded` with its reason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/api/status` mapped a not-running gateway's retained record through `retained_gateway_state`, which only kept `startup_failed`; a watchdog-stamped `degraded` + `exit_reason` of a dead PID became a bare `stopped` with the reason nulled, so the sidebar strip and System page read "Stopped" while `hermes gateway status` said "exited degraded: event loop stopped dispatching". Retain `degraded` under the same rule as `startup_failed` (only while `desired_state` still wants the gateway running, and only for the watchdog reasons in the new `gateway.status.WATCHDOG_EXIT_REASONS`); the resolver already keeps `gateway_exit_reason` for any non-`stopped` verdict. The sidebar strip gains the `degraded` label (warning while live, destructive when the process is gone) and the System page describes a dead `degraded` record as a watchdog exit. `/api/messaging/platforms` keeps yielding `gateway_stopped` for it — the channels really are down. Invariant tests: retained_gateway_state keeps/drops the verdict by desired_state and exit_reason; /api/status carries degraded + exit_reason for the dead PID and stopped/null after `hermes gateway stop`. --- gateway/status.py | 24 +++++++--- hermes_cli/web_routers/status.py | 3 +- tests/gateway/test_status.py | 13 +++++ .../test_web_status_watchdog_surfaces.py | 48 +++++++++++++++++++ web/src/components/SidebarStatusStrip.tsx | 6 +++ web/src/i18n/en.ts | 1 + web/src/i18n/types.ts | 1 + web/src/lib/shared-gateway.ts | 4 ++ website/docs/user-guide/messaging/index.md | 3 +- 9 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 tests/hermes_cli/test_web_status_watchdog_surfaces.py diff --git a/gateway/status.py b/gateway/status.py index 6cd669dcfd..461e955083 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -210,18 +210,30 @@ def normalize_updated_at(value: Any) -> Optional[str]: return None +# ``exit_reason`` values the out-of-loop watchdogs (gateway/shutdown_watchdog.py) stamp together with +# ``gateway_state: degraded`` right before they hard-exit a wedged process (#113372). +WATCHDOG_EXIT_REASONS = frozenset({"loop_liveness_watchdog", "shutdown_watchdog"}) + + def retained_gateway_state(runtime: Any) -> str: """What a NOT-running gateway's retained ``gateway_state.json`` says about it now: - ``"startup_failed"`` only while the operator still wants it running, else ``"stopped"``. + ``"startup_failed"`` (or a watchdog-stamped ``"degraded"``) only while the operator still + wants it running, else ``"stopped"``. ``hermes gateway stop`` keeps the last ``startup_failed`` + ``exit_reason`` on disk for diagnostics and records the durable stop intent as ``desired_state``; a profile the operator - stopped is "stopped", not a current failure. Any other retained state of a dead process - (``running``, ``starting``, missing) is also just "stopped". Shared by ``/api/status`` and - ``/api/messaging/platforms`` so the sidebar strip and the Channels page cannot disagree.""" + stopped is "stopped", not a current failure. A watchdog exit (``degraded`` + an exit_reason in + ``WATCHDOG_EXIT_REASONS``) is the same kind of current failure as ``startup_failed`` and is kept + under the same rule, so the dashboard agrees with ``hermes gateway status``. Any other retained + state of a dead process (``running``, ``starting``, missing) is just "stopped". Shared by + ``/api/status`` and ``/api/messaging/platforms`` so the sidebar strip and the Channels page + cannot disagree.""" rt = runtime if isinstance(runtime, dict) else {} - if rt.get("desired_state") != "stopped" and rt.get("gateway_state") == "startup_failed": - return "startup_failed" + if rt.get("desired_state") != "stopped": + if rt.get("gateway_state") == "startup_failed": + return "startup_failed" + if rt.get("gateway_state") == "degraded" and rt.get("exit_reason") in WATCHDOG_EXIT_REASONS: + return "degraded" return "stopped" diff --git a/hermes_cli/web_routers/status.py b/hermes_cli/web_routers/status.py index 88263c48ce..c8d4adc202 100644 --- a/hermes_cli/web_routers/status.py +++ b/hermes_cli/web_routers/status.py @@ -297,7 +297,8 @@ async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> Di gateway_state = runtime.get("gateway_state") if not gateway_running: # Shared with /api/messaging/platforms: a durable operator stop outranks a retained - # ``startup_failed`` (kept on disk for diagnostics), so the overview does not alarm on it. + # ``startup_failed`` / watchdog ``degraded`` (kept on disk for diagnostics), so the + # overview does not alarm on it. gateway_state = retained_gateway_state(runtime) elif remote_health_body is not None and gateway_state in {None, "stopped"}: # The health probe confirmed the gateway is alive, but the local runtime status diff --git a/tests/gateway/test_status.py b/tests/gateway/test_status.py index ffbcd71ef1..a25890fe58 100644 --- a/tests/gateway/test_status.py +++ b/tests/gateway/test_status.py @@ -1533,3 +1533,16 @@ def test_strict_gateway_identity_rejects_reused_pid(tmp_path, monkeypatch): with pytest.raises(RuntimeError, match="identity changed"): status.get_running_pid_identity_strict(pid_path) + + +def test_retained_gateway_state_keeps_watchdog_degraded_like_startup_failed(): + """A watchdog-stamped ``degraded`` of a dead process is a current failure under the same rule as + ``startup_failed`` (#113372): kept while the operator wants the gateway running, ``stopped`` once + ``hermes gateway stop`` records the intent. The startup-time ``degraded`` (retryable platforms, no + watchdog exit_reason) of a dead process is just ``stopped``.""" + watchdog = {"gateway_state": "degraded", "exit_reason": "loop_liveness_watchdog"} + assert status.retained_gateway_state(watchdog) == "degraded" + assert status.retained_gateway_state({**watchdog, "exit_reason": "shutdown_watchdog"}) == "degraded" + assert status.retained_gateway_state({**watchdog, "desired_state": "stopped"}) == "stopped" + assert status.retained_gateway_state({"gateway_state": "degraded", "exit_reason": None}) == "stopped" + assert status.retained_gateway_state({"gateway_state": "startup_failed", "exit_reason": "x"}) == "startup_failed" diff --git a/tests/hermes_cli/test_web_status_watchdog_surfaces.py b/tests/hermes_cli/test_web_status_watchdog_surfaces.py new file mode 100644 index 0000000000..62a43e92c4 --- /dev/null +++ b/tests/hermes_cli/test_web_status_watchdog_surfaces.py @@ -0,0 +1,48 @@ +"""``/api/status`` agrees with ``hermes gateway status`` on the two #113372 shapes: + +* A watchdog hard-exited the process (``degraded`` + watchdog ``exit_reason``, PID gone) -> the + retained verdict stays ``degraded`` with its ``gateway_exit_reason`` instead of a bare ``stopped``. +""" +from datetime import datetime, timedelta, timezone + +import pytest + +import gateway.status as _gw_status + + +def _iso_age(seconds_ago: float) -> str: + return (datetime.now(timezone.utc) - timedelta(seconds=seconds_ago)).isoformat() + + +@pytest.fixture +def client(monkeypatch): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + + monkeypatch.setattr(_gw_status, "_pid_exists", lambda pid: False) + monkeypatch.setattr(_gw_status, "_get_process_start_time", lambda pid: None) + c = TestClient(app) + c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + return c + + +def test_status_keeps_watchdog_degraded_verdict_and_reason_for_dead_pid(client, monkeypatch): + record = {"gateway_state": "degraded", "exit_reason": "loop_liveness_watchdog", + "pid": 999_999_999, "start_time": 1.0, "updated_at": _iso_age(30), "platforms": {}} + monkeypatch.setattr(_gw_status, "get_running_pid_cached", lambda: None) + monkeypatch.setattr(_gw_status, "read_runtime_status", lambda: record) + + data = client.get("/api/status").json() + assert data["gateway_running"] is False + assert data["gateway_state"] == "degraded" + assert data["gateway_exit_reason"] == "loop_liveness_watchdog" + + # ``hermes gateway stop`` afterwards records the operator's intent: no longer a current failure. + record["desired_state"] = "stopped" + data = client.get("/api/status").json() + assert data["gateway_state"] == "stopped" + assert data["gateway_exit_reason"] is None diff --git a/web/src/components/SidebarStatusStrip.tsx b/web/src/components/SidebarStatusStrip.tsx index fbf05fba56..bb748437a6 100644 --- a/web/src/components/SidebarStatusStrip.tsx +++ b/web/src/components/SidebarStatusStrip.tsx @@ -57,6 +57,12 @@ export function gatewayLine( running: { label: g.running, tone: "text-success" }, starting: { label: g.starting, tone: "text-warning" }, startup_failed: { label: g.failed, tone: "text-destructive" }, + // Live: some channels offline. Retained on a dead PID: a watchdog hard-exited a wedged + // process (gateway_exit_reason names it) — same verdict `hermes gateway status` prints. + degraded: { + label: g.degraded, + tone: status.gateway_running ? "text-warning" : "text-destructive", + }, stopped: { label: g.stopped, tone: "text-muted-foreground" }, }; if (status.gateway_state && byState[status.gateway_state]) { diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 5098f8b198..e406cc4800 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -65,6 +65,7 @@ export const en: Translations = { activeSessionsLabel: "Active Sessions:", gatewayStatusLabel: "Gateway Status:", gatewayStrip: { + degraded: "Degraded", failed: "Start failed", off: "Off", running: "Running", diff --git a/web/src/i18n/types.ts b/web/src/i18n/types.ts index 59378fff17..b36496b1d2 100644 --- a/web/src/i18n/types.ts +++ b/web/src/i18n/types.ts @@ -84,6 +84,7 @@ export interface Translations { activeSessionsLabel: string; gatewayStatusLabel: string; gatewayStrip: { + degraded: string; failed: string; off: string; running: string; diff --git a/web/src/lib/shared-gateway.ts b/web/src/lib/shared-gateway.ts index c4e9873dd9..9619a67a30 100644 --- a/web/src/lib/shared-gateway.ts +++ b/web/src/lib/shared-gateway.ts @@ -33,11 +33,15 @@ const GATEWAY_STATE_COPY: Record = { startup_failed: "Failed to start — see Logs", }; +/** A `degraded` record of a dead process is a watchdog exit, not a live gateway with channels down. */ +const GATEWAY_EXITED_DEGRADED_COPY = "Exited: a watchdog stopped a wedged gateway — see Logs"; + /** Plain description of the gateway's state; null/unknown falls back to running/stopped. */ export function gatewayStateDescription( state: string | null | undefined, running: boolean | undefined, ): string { + if (state === "degraded" && running === false) return GATEWAY_EXITED_DEGRADED_COPY; if (state && GATEWAY_STATE_COPY[state]) return GATEWAY_STATE_COPY[state]; return running ? GATEWAY_STATE_COPY.running : GATEWAY_STATE_COPY.stopped; } diff --git a/website/docs/user-guide/messaging/index.md b/website/docs/user-guide/messaging/index.md index 9a6314e42f..fd73f54838 100644 --- a/website/docs/user-guide/messaging/index.md +++ b/website/docs/user-guide/messaging/index.md @@ -195,7 +195,8 @@ to the log, stamps `gateway_state.json` with `gateway_state: degraded` and `exit_reason: loop_liveness_watchdog`, and exits with code `75` so the service supervisor restarts the process. `hermes gateway status` renders that record as `⚠ Gateway exited degraded: event loop stopped dispatching …` until a new -gateway process overwrites it. Set `gateway.loop_watchdog: false` in +gateway process overwrites it, and the dashboard's gateway badge shows +**Degraded** with the same reason. Set `gateway.loop_watchdog: false` in `config.yaml` to disable the watchdog. ### Optional Linux event-loop watchdog