diff --git a/gateway/run_startup.py b/gateway/run_startup.py index a97622aa1c..a8dc7f05d7 100644 --- a/gateway/run_startup.py +++ b/gateway/run_startup.py @@ -1250,7 +1250,7 @@ class GatewayStartupMixin: self._startup_parked_platforms = True logger.error( "%d configured platform(s) failed to start and are parked (fix the reported error, " - "then `/platform resume `): %s. The gateway is DEGRADED — it serves the " + "then `hermes gateway restart`): %s. The gateway is DEGRADED — it serves the " "remaining platform(s) with those unserved.", len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors), ) diff --git a/gateway/status.py b/gateway/status.py index b351cd03fd..9419b6bf60 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -944,12 +944,13 @@ def parse_active_agents(raw: Any) -> int: return 0 -# Only a live ``running`` gateway is a valid begin-drain target. -_DRAINABLE_GATEWAY_STATES = frozenset({"running"}) +# Live, serving states: a valid begin-drain target. ``degraded`` is a serving gateway with a parked +# platform (a dead watchdog-stamped ``degraded`` is already excluded by ``gateway_running=False``). +_DRAINABLE_GATEWAY_STATES = frozenset({"running", "degraded"}) def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool: - """Busy iff live, ``running``, and ``active_agents > 0`` -- the contract NAS gates on. Liveness + """Busy iff live, serving (``running``/``degraded``), and ``active_agents > 0`` -- the contract NAS gates on. Liveness keys off ``gateway_running``, NEVER ``updated_at`` (a stale heartbeat is a health warning, not death).""" if not derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state): return False @@ -957,7 +958,7 @@ def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_age def derive_gateway_drainable(*, gateway_running: bool, gateway_state: Any) -> bool: - """Drainable iff live and ``running``; independent of ``active_agents`` (idle drains finish).""" + """Drainable iff live and serving; independent of ``active_agents`` (idle drains finish).""" return bool(gateway_running) and gateway_state in _DRAINABLE_GATEWAY_STATES diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index c90580de8d..dcdec3f6b9 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -5173,7 +5173,7 @@ def _runtime_health_lines() -> list[str]: # A live-claiming snapshot can outlive an ungracefully killed gateway (taskkill /F, OOM). Past # the freshness TTL with the recorded PID gone, say so instead of rendering stale live state. - if gateway_state in ("running", "starting", "draining") and runtime_status_is_stale(state): + if gateway_state in ("running", "degraded", "starting", "draining") and runtime_status_is_stale(state): if not runtime_status_pid_is_live(state): lines.append( f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the " diff --git a/hermes_cli/gateway_windows.py b/hermes_cli/gateway_windows.py index e90b4cb981..56eae9a9fd 100644 --- a/hermes_cli/gateway_windows.py +++ b/hermes_cli/gateway_windows.py @@ -1407,7 +1407,7 @@ def _probe_state_file(state_path: Path) -> None: age_str = f" (updated {age_seconds}s ago)" except Exception: pass - _probe(5, gateway_state == "running", f"gateway_state.json state={gateway_state!r}{age_str}") + _probe(5, gateway_state in ("running", "degraded"), f"gateway_state.json state={gateway_state!r}{age_str}") except Exception as exc: _probe(5, False, f"gateway_state.json present but unreadable: {exc}") diff --git a/hermes_cli/web_routers/status.py b/hermes_cli/web_routers/status.py index c41a9f1c89..86994a6f7c 100644 --- a/hermes_cli/web_routers/status.py +++ b/hermes_cli/web_routers/status.py @@ -306,7 +306,7 @@ async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> Di # The health probe confirmed the gateway is alive, but the local runtime status # file may be stale (cross-container): override so the badge is correct. gateway_state = "running" - elif gateway_state in {"running", "starting"} and runtime_status_is_stale(runtime): + elif gateway_state in {"running", "degraded", "starting"} and runtime_status_is_stale(runtime): # Alive PID, but housekeeping stopped re-stamping the heartbeat: the loop or the # housekeeping thread wedged while the file still says 'running' (#113372). Same arm # as ``hermes gateway status`` so the sidebar strip and the CLI agree. diff --git a/tests/gateway/test_status.py b/tests/gateway/test_status.py index a25890fe58..c5438e2b6b 100644 --- a/tests/gateway/test_status.py +++ b/tests/gateway/test_status.py @@ -1224,6 +1224,15 @@ class TestGatewayBusyDerivation: gateway_running=True, gateway_state="running", active_agents=0 ) is False + def test_degraded_gateway_with_work_is_busy_and_drainable(self): + # Serving with a parked platform (#91547): in-flight turns must not look idle to NAS. + assert status.derive_gateway_busy( + gateway_running=True, gateway_state="degraded", active_agents=2 + ) is True + assert status.derive_gateway_drainable( + gateway_running=False, gateway_state="degraded" + ) is False + class TestRespawnStormBreaker: def test_no_storm_under_threshold(self, tmp_path, monkeypatch):