fix(gateway): spare a still-booting gateway from the Desktop orphan reap (#122533)
(cherry picked from commit b616766e30c830cf07096f02cedb1ca5098d3433)
This commit is contained in:
@@ -1765,11 +1765,37 @@ def _reaper_candidate_is_supervisor_owned(pid: int) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool:
|
||||
def _gateway_process_age_s(pid: int) -> float:
|
||||
"""Seconds since ``pid`` started, or ``0.0`` when undeterminable (never negative).
|
||||
|
||||
A probe failure must not widen a reap, so an unknown age reads as "too young to
|
||||
touch" under a positive grace and is irrelevant when the grace is 0.
|
||||
"""
|
||||
try:
|
||||
import psutil # type: ignore
|
||||
import time as _time
|
||||
|
||||
return max(0.0, _time.time() - float(psutil.Process(int(pid)).create_time()))
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _reap_unsupervised_gateway_orphans(
|
||||
extra_exclude: set | None = None, *, min_age_s: float = 0.0,
|
||||
) -> bool:
|
||||
"""Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts
|
||||
the restart fallback runs the gateway in-process under a ``gateway restart`` argv; a stale pidfile
|
||||
then lets a live orphan keep the webhook port while a restart stacks a duplicate. No-op where a
|
||||
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed."""
|
||||
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed.
|
||||
|
||||
``min_age_s`` spares a candidate younger than the grace: a gateway claims
|
||||
``gateway.pid``/``gateway.lock`` only after imports + runner setup, so a process
|
||||
that a previous Desktop generation (or a concurrent ``gateway start``) just launched
|
||||
is scan-visible but not yet record-visible, and the argv sweep cannot tell it from
|
||||
a corpse. Reaping it writes a planned-stop marker it consumes seconds later — a clean
|
||||
exit 0 with no supervisor to revive it (#122533). Only the Desktop boot sweep passes a
|
||||
grace (it is the one caller that races a launch); stop/restart keep reaping at once.
|
||||
"""
|
||||
try:
|
||||
supervised_host = supports_systemd_services()
|
||||
except Exception:
|
||||
@@ -1801,6 +1827,8 @@ def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool
|
||||
]
|
||||
except Exception:
|
||||
return False
|
||||
if min_age_s > 0:
|
||||
orphans = [p for p in orphans if _gateway_process_age_s(p) >= min_age_s]
|
||||
if not orphans:
|
||||
return False
|
||||
|
||||
|
||||
@@ -241,10 +241,18 @@ async def _lifespan(app: "FastAPI"):
|
||||
# one that would race the same credential (#77276). Runs
|
||||
# unconditionally; protection of a healthy standalone gateway lives
|
||||
# INSIDE the reaper (registration probed with cleanup_stale=False).
|
||||
# The grace covers the one race this call site creates: a gateway launched
|
||||
# moments ago by the previous Desktop generation (or a concurrent start)
|
||||
# is scan-visible but has not claimed gateway.pid/gateway.lock yet, and the
|
||||
# argv sweep would reap it — writing a planned-stop marker that the booting
|
||||
# gateway consumes seconds later, exiting 0 with no supervisor to revive it
|
||||
# (#122533). Same rule the serve-process reaper applies to an unrecorded
|
||||
# sibling (hermes_cli.dashboard_procs._REAP_MIN_AGE_SECONDS).
|
||||
try:
|
||||
from hermes_cli.dashboard_procs import _REAP_MIN_AGE_SECONDS
|
||||
from hermes_cli.gateway import _reap_unsupervised_gateway_orphans
|
||||
|
||||
_reap_unsupervised_gateway_orphans()
|
||||
_reap_unsupervised_gateway_orphans(min_age_s=_REAP_MIN_AGE_SECONDS)
|
||||
except Exception:
|
||||
_log.exception("Desktop startup: orphan gateway reap failed")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user