fix(gateway): every live-state check knows a degraded gateway is serving
Gate review: `degraded` became a whole-life serving state but four sibling predicates only accepted `running`: derive_gateway_busy / derive_gateway_drainable (the NAS drain gate reported a degraded gateway with in-flight turns as idle and undrainable), the stale-heartbeat detectors in `hermes gateway status` and the dashboard, and the Windows doctor probe. All four accept `degraded`; a dead watchdog-stamped `degraded` is still excluded by `gateway_running=False`. The parked-platform ERROR pointed at `/platform resume`, which only resumes platforms in the retry queue - a non-retryable failure never enters it. The remedy is `hermes gateway restart`. Test: a degraded gateway with active agents is busy; not live => not drainable.
This commit is contained in:
@@ -1250,7 +1250,7 @@ class GatewayStartupMixin:
|
||||
self._startup_parked_platforms = True
|
||||
logger.error(
|
||||
"%d configured platform(s) failed to start and are parked (fix the reported error, "
|
||||
"then `/platform resume <platform>`): %s. The gateway is DEGRADED — it serves the "
|
||||
"then `hermes gateway restart`): %s. The gateway is DEGRADED — it serves the "
|
||||
"remaining platform(s) with those unserved.",
|
||||
len(startup_nonretryable_errors), "; ".join(startup_nonretryable_errors),
|
||||
)
|
||||
|
||||
@@ -944,12 +944,13 @@ def parse_active_agents(raw: Any) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
# Only a live ``running`` gateway is a valid begin-drain target.
|
||||
_DRAINABLE_GATEWAY_STATES = frozenset({"running"})
|
||||
# Live, serving states: a valid begin-drain target. ``degraded`` is a serving gateway with a parked
|
||||
# platform (a dead watchdog-stamped ``degraded`` is already excluded by ``gateway_running=False``).
|
||||
_DRAINABLE_GATEWAY_STATES = frozenset({"running", "degraded"})
|
||||
|
||||
|
||||
def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
|
||||
"""Busy iff live, ``running``, and ``active_agents > 0`` -- the contract NAS gates on. Liveness
|
||||
"""Busy iff live, serving (``running``/``degraded``), and ``active_agents > 0`` -- the contract NAS gates on. Liveness
|
||||
keys off ``gateway_running``, NEVER ``updated_at`` (a stale heartbeat is a health warning, not death)."""
|
||||
if not derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state):
|
||||
return False
|
||||
@@ -957,7 +958,7 @@ def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_age
|
||||
|
||||
|
||||
def derive_gateway_drainable(*, gateway_running: bool, gateway_state: Any) -> bool:
|
||||
"""Drainable iff live and ``running``; independent of ``active_agents`` (idle drains finish)."""
|
||||
"""Drainable iff live and serving; independent of ``active_agents`` (idle drains finish)."""
|
||||
return bool(gateway_running) and gateway_state in _DRAINABLE_GATEWAY_STATES
|
||||
|
||||
|
||||
|
||||
@@ -5173,7 +5173,7 @@ def _runtime_health_lines() -> list[str]:
|
||||
|
||||
# A live-claiming snapshot can outlive an ungracefully killed gateway (taskkill /F, OOM). Past
|
||||
# the freshness TTL with the recorded PID gone, say so instead of rendering stale live state.
|
||||
if gateway_state in ("running", "starting", "draining") and runtime_status_is_stale(state):
|
||||
if gateway_state in ("running", "degraded", "starting", "draining") and runtime_status_is_stale(state):
|
||||
if not runtime_status_pid_is_live(state):
|
||||
lines.append(
|
||||
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
|
||||
|
||||
@@ -1407,7 +1407,7 @@ def _probe_state_file(state_path: Path) -> None:
|
||||
age_str = f" (updated {age_seconds}s ago)"
|
||||
except Exception:
|
||||
pass
|
||||
_probe(5, gateway_state == "running", f"gateway_state.json state={gateway_state!r}{age_str}")
|
||||
_probe(5, gateway_state in ("running", "degraded"), f"gateway_state.json state={gateway_state!r}{age_str}")
|
||||
except Exception as exc:
|
||||
_probe(5, False, f"gateway_state.json present but unreadable: {exc}")
|
||||
|
||||
|
||||
@@ -306,7 +306,7 @@ async def _resolve_gateway_status(profile_dir: Optional[Path], health_url) -> Di
|
||||
# The health probe confirmed the gateway is alive, but the local runtime status
|
||||
# file may be stale (cross-container): override so the badge is correct.
|
||||
gateway_state = "running"
|
||||
elif gateway_state in {"running", "starting"} and runtime_status_is_stale(runtime):
|
||||
elif gateway_state in {"running", "degraded", "starting"} and runtime_status_is_stale(runtime):
|
||||
# Alive PID, but housekeeping stopped re-stamping the heartbeat: the loop or the
|
||||
# housekeeping thread wedged while the file still says 'running' (#113372). Same arm
|
||||
# as ``hermes gateway status`` so the sidebar strip and the CLI agree.
|
||||
|
||||
@@ -1224,6 +1224,15 @@ class TestGatewayBusyDerivation:
|
||||
gateway_running=True, gateway_state="running", active_agents=0
|
||||
) is False
|
||||
|
||||
def test_degraded_gateway_with_work_is_busy_and_drainable(self):
|
||||
# Serving with a parked platform (#91547): in-flight turns must not look idle to NAS.
|
||||
assert status.derive_gateway_busy(
|
||||
gateway_running=True, gateway_state="degraded", active_agents=2
|
||||
) is True
|
||||
assert status.derive_gateway_drainable(
|
||||
gateway_running=False, gateway_state="degraded"
|
||||
) is False
|
||||
|
||||
|
||||
class TestRespawnStormBreaker:
|
||||
def test_no_storm_under_threshold(self, tmp_path, monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user