fix(gateway): spare a still-booting gateway from the Desktop orphan reap (#122533)

(cherry picked from commit b616766e30c830cf07096f02cedb1ca5098d3433)
This commit is contained in:
Halldrix
2026-09-25 06:17:17 -05:00
committed by kshitij
parent ad4e4496c2
commit 33a7ce0fa6
2 changed files with 39 additions and 3 deletions

View File

@@ -1765,11 +1765,37 @@ def _reaper_candidate_is_supervisor_owned(pid: int) -> bool:
return False
def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool:
def _gateway_process_age_s(pid: int) -> float:
"""Seconds since ``pid`` started, or ``0.0`` when undeterminable (never negative).
A probe failure must not widen a reap, so an unknown age reads as "too young to
touch" under a positive grace and is irrelevant when the grace is 0.
"""
try:
import psutil # type: ignore
import time as _time
return max(0.0, _time.time() - float(psutil.Process(int(pid)).create_time()))
except Exception:
return 0.0
def _reap_unsupervised_gateway_orphans(
extra_exclude: set | None = None, *, min_age_s: float = 0.0,
) -> bool:
"""Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts
the restart fallback runs the gateway in-process under a ``gateway restart`` argv; a stale pidfile
then lets a live orphan keep the webhook port while a restart stacks a duplicate. No-op where a
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed."""
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed.
``min_age_s`` spares a candidate younger than the grace: a gateway claims
``gateway.pid``/``gateway.lock`` only after imports + runner setup, so a process
that a previous Desktop generation (or a concurrent ``gateway start``) just launched
is scan-visible but not yet record-visible, and the argv sweep cannot tell it from
a corpse. Reaping it writes a planned-stop marker it consumes seconds later — a clean
exit 0 with no supervisor to revive it (#122533). Only the Desktop boot sweep passes a
grace (it is the one caller that races a launch); stop/restart keep reaping at once.
"""
try:
supervised_host = supports_systemd_services()
except Exception:
@@ -1801,6 +1827,8 @@ def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool
]
except Exception:
return False
if min_age_s > 0:
orphans = [p for p in orphans if _gateway_process_age_s(p) >= min_age_s]
if not orphans:
return False

View File

@@ -241,10 +241,18 @@ async def _lifespan(app: "FastAPI"):
# one that would race the same credential (#77276). Runs
# unconditionally; protection of a healthy standalone gateway lives
# INSIDE the reaper (registration probed with cleanup_stale=False).
# The grace covers the one race this call site creates: a gateway launched
# moments ago by the previous Desktop generation (or a concurrent start)
# is scan-visible but has not claimed gateway.pid/gateway.lock yet, and the
# argv sweep would reap it — writing a planned-stop marker that the booting
# gateway consumes seconds later, exiting 0 with no supervisor to revive it
# (#122533). Same rule the serve-process reaper applies to an unrecorded
# sibling (hermes_cli.dashboard_procs._REAP_MIN_AGE_SECONDS).
try:
from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
_reap_unsupervised_gateway_orphans()
_reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
except Exception:
_log.exception("Desktop startup: orphan gateway reap failed")