fix(gateway): warn when a live gateway's heartbeat goes stale instead of printing running
The reporter's case in #113372 is "not a crash": the process stays alive, `gateway_state.json` keeps saying `running`, and housekeeping, cron and the kanban dispatcher are frozen. Nothing wrote `updated_at` periodically, so the file was a stored status, not a heartbeat, and both `hermes gateway status` and `/api/status` rendered a wedged gateway as healthy (the existing stale arm only fires when the recorded PID is gone). Make the housekeeping tick re-stamp `gateway_state.json` first thing every tick (60 s), so `updated_at` is a heartbeat that stops when the thread — or a chore blocked on the loop — wedges. Readers then warn on `running`/`starting` + stale stamp + live PID: `hermes gateway status` prints `⚠ Gateway heartbeat stale: housekeeping has not refreshed gateway_state.json for N s (event loop or housekeeping wedged; pid X alive)`, `/api/status` carries `gateway_heartbeat_stale_s` (null when healthy) and the sidebar strip shows "Heartbeat stale". Liveness (`gateway_running`, busy/drainable) still keys off the PID, never the stamp. Draining is excluded: shutdown drains the housekeeping thread before the process exits, so its stamp legitimately ages. Invariant tests: the CLI line names the age and the live PID and is silent on a fresh stamp; /api/status sets/clears `gateway_heartbeat_stale_s` the same way.
This commit is contained in:
@@ -65,6 +65,11 @@ export function gatewayLine(
|
||||
},
|
||||
stopped: { label: g.stopped, tone: "text-muted-foreground" },
|
||||
};
|
||||
// Alive but housekeeping stopped stamping the heartbeat: 'Running' would be the lie the
|
||||
// reporter saw (loop/housekeeping wedged while gateway_state.json still said running).
|
||||
if (status.gateway_heartbeat_stale_s != null) {
|
||||
return { label: g.heartbeatStale, tone: "text-destructive" };
|
||||
}
|
||||
if (status.gateway_state && byState[status.gateway_state]) {
|
||||
return byState[status.gateway_state];
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ export const en: Translations = {
|
||||
gatewayStrip: {
|
||||
degraded: "Degraded",
|
||||
failed: "Start failed",
|
||||
heartbeatStale: "Heartbeat stale",
|
||||
off: "Off",
|
||||
running: "Running",
|
||||
starting: "Starting",
|
||||
|
||||
@@ -86,6 +86,7 @@ export interface Translations {
|
||||
gatewayStrip: {
|
||||
degraded: string;
|
||||
failed: string;
|
||||
heartbeatStale: string;
|
||||
off: string;
|
||||
running: string;
|
||||
starting: string;
|
||||
|
||||
@@ -1927,6 +1927,10 @@ export interface StatusResponse {
|
||||
env_path: string;
|
||||
gateway_exit_reason: string | null;
|
||||
gateway_health_url: string | null;
|
||||
/** Seconds since the gateway's housekeeping last stamped gateway_state.json, set only when the
|
||||
* process is alive but the stamp is past the freshness TTL (loop/housekeeping wedged).
|
||||
* null when healthy; absent on older backends. */
|
||||
gateway_heartbeat_stale_s?: number | null;
|
||||
gateway_pid: number | null;
|
||||
gateway_platforms: Record<string, PlatformStatus>;
|
||||
gateway_running: boolean;
|
||||
|
||||
Reference in New Issue
Block a user