fix(gateway): every live-state check knows a degraded gateway is serving

Gate review: `degraded` became a whole-life serving state but four
sibling predicates only accepted `running`: derive_gateway_busy /
derive_gateway_drainable (the NAS drain gate reported a degraded gateway
with in-flight turns as idle and undrainable), the stale-heartbeat
detectors in `hermes gateway status` and the dashboard, and the Windows
doctor probe. All four accept `degraded`; a dead watchdog-stamped
`degraded` is still excluded by `gateway_running=False`.

The parked-platform ERROR pointed at `/platform resume`, which only
resumes platforms in the retry queue - a non-retryable failure never
enters it. The remedy is `hermes gateway restart`.

Test: a degraded gateway with active agents is busy; not live => not
drainable.
This commit is contained in:
kshitijk4poor
2026-09-19 01:12:13 +05:30
committed by kshitij
parent ed8f8498d3
commit b97b0c6bc9
6 changed files with 18 additions and 8 deletions

View File

@@ -1250,7 +1250,7 @@ class GatewayStartupMixin:
self._startup_parked_platforms = True
logger.error(
"%d configured platform(s) failed to start and are parked (fix the reported error, "
"then `/platform resume <platform>`): %s. The gateway is DEGRADED — it serves the "
"then `hermes gateway restart`): %s. The gateway is DEGRADED — it serves the "
"remaining platform(s) with those unserved.",
len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors),
)

View File

@@ -944,12 +944,13 @@ def parse_active_agents(raw: Any) -> int:
return 0
# Only a live ``running`` gateway is a valid begin-drain target.
_DRAINABLE_GATEWAY_STATES = frozenset({"running"})
# Live, serving states: a valid begin-drain target. ``degraded`` is a serving gateway with a parked
# platform (a dead watchdog-stamped ``degraded`` is already excluded by ``gateway_running=False``).
_DRAINABLE_GATEWAY_STATES = frozenset({"running", "degraded"})
def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
"""Busy iff live, ``running``, and ``active_agents > 0`` -- the contract NAS gates on. Liveness
"""Busy iff live, serving (``running``/``degraded``), and ``active_agents > 0`` -- the contract NAS gates on. Liveness
keys off ``gateway_running``, NEVER ``updated_at`` (a stale heartbeat is a health warning, not death)."""
if not derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state):
return False
@@ -957,7 +958,7 @@ def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_age
def derive_gateway_drainable(*, gateway_running: bool, gateway_state: Any) -> bool:
"""Drainable iff live and ``running``; independent of ``active_agents`` (idle drains finish)."""
"""Drainable iff live and serving; independent of ``active_agents`` (idle drains finish)."""
return bool(gateway_running) and gateway_state in _DRAINABLE_GATEWAY_STATES

View File

@@ -5173,7 +5173,7 @@ def _runtime_health_lines() -> list[str]:
# A live-claiming snapshot can outlive an ungracefully killed gateway (taskkill /F, OOM). Past
# the freshness TTL with the recorded PID gone, say so instead of rendering stale live state.
if gateway_state in ("running", "starting", "draining") and runtime_status_is_stale(state):
if gateway_state in ("running", "degraded", "starting", "draining") and runtime_status_is_stale(state):
if not runtime_status_pid_is_live(state):
lines.append(
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "

View File

@@ -1407,7 +1407,7 @@ def _probe_state_file(state_path: Path) -> None:
age_str = f" (updated {age_seconds}s ago)"
except Exception:
pass
_probe(5, gateway_state == "running", f"gateway_state.json state={gateway_state!r}{age_str}")
_probe(5, gateway_state in ("running", "degraded"), f"gateway_state.json state={gateway_state!r}{age_str}")
except Exception as exc:
_probe(5, False, f"gateway_state.json present but unreadable: {exc}")

View File

@@ -306,7 +306,7 @@ async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> Di
# The health probe confirmed the gateway is alive, but the local runtime status
# file may be stale (cross-container): override so the badge is correct.
gateway_state = "running"
elif gateway_state in {"running", "starting"} and runtime_status_is_stale(runtime):
elif gateway_state in {"running", "degraded", "starting"} and runtime_status_is_stale(runtime):
# Alive PID, but housekeeping stopped re-stamping the heartbeat: the loop or the
# housekeeping thread wedged while the file still says 'running' (#113372). Same arm
# as ``hermes gateway status`` so the sidebar strip and the CLI agree.

View File

@@ -1224,6 +1224,15 @@ class TestGatewayBusyDerivation:
gateway_running=True, gateway_state="running", active_agents=0
) is False
def test_degraded_gateway_with_work_is_busy_and_drainable(self):
# Serving with a parked platform (#91547): in-flight turns must not look idle to NAS.
assert status.derive_gateway_busy(
gateway_running=True, gateway_state="degraded", active_agents=2
) is True
assert status.derive_gateway_drainable(
gateway_running=False, gateway_state="degraded"
) is False
class TestRespawnStormBreaker:
def test_no_storm_under_threshold(self, tmp_path, monkeypatch):