507 lines
18 KiB
Python
507 lines
18 KiB
Python
"""Process identity: spawn tags, the machine-wide spawn ledger, and the Windows job-object self-attach.
|
|
|
|
Three layers that make every long-lived Hermes process positively identifiable, so reapers (``hermes
|
|
update``, Desktop startup sweeps) never have to guess lineage from PPID archaeology or cmdline
|
|
pattern-matching:
|
|
|
|
3. **Job object self-attach** (Windows): a backend places itself in a job with ``KILL_ON_JOB_CLOSE``
|
|
so its whole child tree dies atomically with it — no launcher→worker two-hop chains left holding
|
|
``.pyd`` locks after the visible root is killed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import platform
|
|
import threading
|
|
import time
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SPAWN_ENV_VAR = "HERMES_SPAWN"
|
|
_TAG_VERSION = "v1"
|
|
LEDGER_FILENAME = "spawn-ledger.json"
|
|
|
|
#: Purposes a reaper may treat as "safe to kill when the owner is gone".
|
|
#: Interactive processes (chat, REPLs) are deliberately NOT in this set.
|
|
REAPABLE_PURPOSES = frozenset({"serve", "dashboard", "gateway", "mcp-helper"})
|
|
|
|
_IS_WINDOWS = platform.system() == "Windows"
|
|
|
|
# Module-global job handle: must live exactly as long as this process so the
|
|
# kernel closes it (and kills the job) when we die. Never close it manually.
|
|
_JOB_HANDLE = None
|
|
_LEDGER_LOCK = threading.Lock()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Install identity
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def install_id(project_root: Optional[Path] = None) -> str:
|
|
"""Stable 12-hex identifier for THIS install (derived from its path)."""
|
|
if project_root is None:
|
|
try:
|
|
from hermes_constants import PROJECT_ROOT as _root
|
|
|
|
project_root = Path(_root)
|
|
except Exception:
|
|
project_root = Path(__file__).resolve().parent.parent
|
|
try:
|
|
canonical = str(Path(project_root).resolve()).lower()
|
|
except OSError:
|
|
canonical = str(project_root).lower()
|
|
return hashlib.sha256(canonical.encode("utf-8", "replace")).hexdigest()[:12]
|
|
|
|
|
|
def _process_create_time(pid: int) -> Optional[float]:
|
|
"""``psutil`` create time for ``pid``; ``None`` when psutil can't say."""
|
|
try:
|
|
import psutil
|
|
|
|
return float(psutil.Process(pid).create_time())
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _own_create_time() -> Optional[float]:
|
|
return _process_create_time(os.getpid())
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Layer 1 — spawn tags
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass(frozen=True)
|
|
class SpawnTag:
|
|
install: str
|
|
purpose: str
|
|
spawner_pid: int
|
|
spawner_create: Optional[float]
|
|
|
|
|
|
def build_spawn_tag(purpose: str, *, project_root: Optional[Path] = None) -> str:
|
|
"""Value for the child's ``HERMES_SPAWN`` env var, stamped by the spawner."""
|
|
create = _own_create_time()
|
|
create_part = f"{create:.3f}" if create is not None else "-"
|
|
return ":".join((_TAG_VERSION, install_id(project_root), purpose, str(os.getpid()), create_part))
|
|
|
|
|
|
def spawn_env(purpose: str, *, project_root: Optional[Path] = None) -> dict[str, str]:
|
|
"""Env fragment a spawner merges into a child's environment."""
|
|
return {SPAWN_ENV_VAR: build_spawn_tag(purpose, project_root=project_root)}
|
|
|
|
|
|
def parse_spawn_tag(raw: object) -> Optional[SpawnTag]:
|
|
"""Parse a ``HERMES_SPAWN`` value; ``None`` for anything malformed."""
|
|
if not isinstance(raw, str):
|
|
return None
|
|
parts = raw.split(":")
|
|
if len(parts) != 5 or parts[0] != _TAG_VERSION:
|
|
return None
|
|
_, install, purpose, pid_s, create_s = parts
|
|
if not install or not purpose:
|
|
return None
|
|
try:
|
|
pid = int(pid_s)
|
|
create = None if create_s == "-" else float(create_s)
|
|
except ValueError:
|
|
return None
|
|
if pid <= 0:
|
|
return None
|
|
return SpawnTag(install=install, purpose=purpose, spawner_pid=pid, spawner_create=create)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Layer 2 — spawn ledger
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class LedgerEntry:
|
|
pid: int
|
|
create_time: Optional[float]
|
|
purpose: str
|
|
install: str
|
|
spawner_pid: Optional[int]
|
|
spawner_create: Optional[float]
|
|
registered_at: float
|
|
argv: str
|
|
# Structured launch identity (#63206): what a relauncher needs to bring
|
|
# this runtime back after an update, without parsing argv. Empty for
|
|
# purposes that don't supply it; readers must use .get() — older ledger
|
|
# files on disk predate these keys.
|
|
host: str = ""
|
|
port: Optional[int] = None
|
|
profile: str = ""
|
|
|
|
|
|
def _ledger_path() -> Path:
|
|
"""Machine-root ledger path (shared by every profile of this install)."""
|
|
try:
|
|
from hermes_constants import get_default_hermes_root
|
|
|
|
return Path(get_default_hermes_root()) / LEDGER_FILENAME
|
|
except Exception:
|
|
from hermes_cli.config import get_hermes_home
|
|
|
|
return Path(get_hermes_home()) / LEDGER_FILENAME
|
|
|
|
|
|
def _read_ledger(path: Path) -> Optional[list[dict]]:
|
|
"""Entries list, ``[]`` for empty/missing, ``None`` for CORRUPT.
|
|
|
|
Mirrors the #89298 contract: corrupt is a distinct state that must never be silently treated as
|
|
an empty roster.
|
|
"""
|
|
try:
|
|
text = path.read_text(encoding="utf-8")
|
|
except FileNotFoundError:
|
|
return []
|
|
except OSError:
|
|
return None
|
|
if not text.strip():
|
|
return []
|
|
try:
|
|
parsed = json.loads(text)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
if not isinstance(parsed, list):
|
|
return None
|
|
return [e for e in parsed if isinstance(e, dict)]
|
|
|
|
|
|
def _read_ledger_or_quarantine(path: Path) -> Optional[list[dict]]:
|
|
"""Ledger entries; ``None`` after parking a corrupt file. Caller holds ``_LEDGER_LOCK``."""
|
|
entries = _read_ledger(path)
|
|
if entries is None:
|
|
parked = path.with_suffix(path.suffix + ".corrupt")
|
|
try:
|
|
os.replace(path, parked)
|
|
logger.warning("spawn ledger was unreadable; moved to %s", parked)
|
|
except OSError:
|
|
pass
|
|
return entries
|
|
|
|
|
|
def _same_incarnation(proc, create_time: Optional[float]) -> bool:
|
|
"""Does the live ``proc`` match a recorded ``create_time`` (2 s tolerance; ``None`` matches)?"""
|
|
return create_time is None or abs(float(proc.create_time()) - float(create_time)) < 2.0
|
|
|
|
|
|
def _pid_alive_matches(pid: int, create_time: Optional[float]) -> Optional[bool]:
|
|
"""True/False when provable; ``None`` when psutil can't say."""
|
|
try:
|
|
import psutil
|
|
except Exception:
|
|
return None
|
|
try:
|
|
return _same_incarnation(psutil.Process(int(pid)), create_time)
|
|
except psutil.NoSuchProcess:
|
|
return False
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def register_self(
|
|
purpose: str,
|
|
*,
|
|
project_root: Optional[Path] = None,
|
|
detail: Optional[dict] = None,
|
|
) -> bool:
|
|
"""Record this process in the machine spawn ledger. Best-effort.
|
|
|
|
Called at the top of every long-lived entry point. Dead entries — ``(pid, create_time)`` no
|
|
longer live — are pruned on every write so the ledger tracks reality instead of growing
|
|
forever. ``detail`` may carry ``host``/``port``/``profile`` so the update pipeline can
|
|
relaunch a manually-started serve with its real bind address instead of guessing from argv.
|
|
"""
|
|
tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR))
|
|
spawner_pid: Optional[int] = tag.spawner_pid if tag else None
|
|
spawner_create: Optional[float] = tag.spawner_create if tag else None
|
|
if spawner_pid is None:
|
|
# Desktop compatibility: the Electron app already stamps children with
|
|
# HERMES_PARENT_PID (+ optional `winms:<ms>` start marker) for its
|
|
# parent-death watchdog. Reuse it as spawner identity so ledger
|
|
# lineage works with every Desktop version, no TS change needed.
|
|
try:
|
|
raw = int(os.environ.get("HERMES_PARENT_PID", ""))
|
|
if raw > 0:
|
|
spawner_pid = raw
|
|
except (TypeError, ValueError):
|
|
pass
|
|
marker = os.environ.get("HERMES_PARENT_START_MARKER", "")
|
|
if spawner_pid is not None and marker.startswith("winms:"):
|
|
try:
|
|
spawner_create = float(marker.split(":", 1)[1]) / 1000.0
|
|
except (ValueError, IndexError):
|
|
spawner_create = None
|
|
entry = _new_entry(os.getpid(), _own_create_time(), purpose, project_root, spawner_pid, spawner_create)
|
|
if detail:
|
|
try:
|
|
entry.host = str(detail.get("host") or "")
|
|
port = detail.get("port")
|
|
entry.port = int(port) if port is not None else None
|
|
entry.profile = str(detail.get("profile") or "")
|
|
except (TypeError, ValueError):
|
|
pass
|
|
try:
|
|
import sys as _sys
|
|
|
|
# 10 tokens (was 6): enough for `hermes serve --host X --port N
|
|
# --profile P` — the relaunch shapes #63206 needs — while still
|
|
# bounding pathological argv. Structured detail above is the
|
|
# canonical identity; argv is the human-readable fallback.
|
|
entry.argv = " ".join(_sys.argv[:10])
|
|
except Exception:
|
|
pass
|
|
|
|
return _append_entry(entry)
|
|
|
|
|
|
def _new_entry(
|
|
pid: int,
|
|
create_time: Optional[float],
|
|
purpose: str,
|
|
project_root: Optional[Path],
|
|
spawner_pid: Optional[int],
|
|
spawner_create: Optional[float],
|
|
) -> LedgerEntry:
|
|
return LedgerEntry(
|
|
pid=pid,
|
|
create_time=create_time,
|
|
purpose=purpose,
|
|
install=install_id(project_root),
|
|
spawner_pid=spawner_pid,
|
|
spawner_create=spawner_create,
|
|
registered_at=time.time(),
|
|
argv="",
|
|
)
|
|
|
|
|
|
def _append_entry(entry: LedgerEntry) -> bool:
|
|
"""Prune dead entries and append ``entry`` — the ONLY ledger write path.
|
|
|
|
Serialized under ``_LEDGER_LOCK`` with an atomic tmp+replace, exactly as ``register_self`` has
|
|
always written (kept single so #91660's lock- serialization guarantees hold: no writer ever
|
|
touches the file outside this function).
|
|
"""
|
|
path = _ledger_path()
|
|
with _LEDGER_LOCK:
|
|
entries = _read_ledger_or_quarantine(path) or []
|
|
pruned: list[dict] = []
|
|
for e in entries:
|
|
pid = e.get("pid")
|
|
if not isinstance(pid, int) or pid == entry.pid:
|
|
continue
|
|
alive = _pid_alive_matches(pid, e.get("create_time"))
|
|
if alive is False:
|
|
continue # provably dead → prune
|
|
pruned.append(e)
|
|
pruned.append(asdict(entry))
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
|
|
tmp.write_text(json.dumps(pruned, indent=2), encoding="utf-8")
|
|
os.replace(tmp, path)
|
|
return True
|
|
except OSError:
|
|
logger.debug("spawn ledger write failed", exc_info=True)
|
|
return False
|
|
|
|
|
|
def register_child(
|
|
pid: int,
|
|
purpose: str,
|
|
*,
|
|
project_root: Optional[Path] = None,
|
|
) -> bool:
|
|
"""Record a CHILD process this process just spawned. Best-effort.
|
|
|
|
Mirror of :func:`register_self` for children that cannot register themselves (stdio MCP helper
|
|
subprocesses, #61514: arbitrary ``npx``/binary servers never import Hermes code). The entry
|
|
records the child's ``(pid, create_time)`` with THIS process as the spawner, so reapers get the
|
|
same positive-identity contract:
|
|
|
|
- a live helper whose spawner is still alive is never reaped (``spawner_is_dead`` → ``False``);
|
|
- a helper whose spawner ``(pid, create_time)`` is provably gone is a reapable orphan.
|
|
"""
|
|
try:
|
|
pid = int(pid)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
if pid <= 0:
|
|
return False
|
|
child_create = _process_create_time(pid)
|
|
if child_create is None:
|
|
return False
|
|
entry = _new_entry(pid, child_create, purpose, project_root, os.getpid(), _own_create_time())
|
|
try:
|
|
import psutil
|
|
|
|
entry.argv = " ".join(psutil.Process(pid).cmdline()[:10])
|
|
except Exception:
|
|
pass
|
|
return _append_entry(entry)
|
|
|
|
|
|
def ledger_entries(*, project_root: Optional[Path] = None) -> list[dict]:
|
|
"""Live-verified ledger entries for THIS install.
|
|
|
|
A corrupt ledger is quarantined and read as empty — identical philosophy to the backend-
|
|
ownership fix (#89298): never let corruption erase or fake a roster; never let it block the
|
|
caller either.
|
|
"""
|
|
want_install = install_id(project_root)
|
|
with _LEDGER_LOCK:
|
|
entries = _read_ledger_or_quarantine(_ledger_path())
|
|
if entries is None:
|
|
return []
|
|
return [
|
|
e
|
|
for e in entries
|
|
if e.get("install") == want_install
|
|
and isinstance(e.get("pid"), int)
|
|
and _pid_alive_matches(e["pid"], e.get("create_time")) is not False
|
|
]
|
|
|
|
|
|
def spawner_is_dead(entry: dict) -> Optional[bool]:
|
|
"""Is the recorded spawner of this entry provably gone?"""
|
|
spawner_pid = entry.get("spawner_pid")
|
|
if not isinstance(spawner_pid, int) or spawner_pid <= 0:
|
|
return None
|
|
alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"))
|
|
return None if alive is None else not alive
|
|
|
|
|
|
def reap_orphaned_mcp_helpers(
|
|
*,
|
|
project_root: Optional[Path] = None,
|
|
kill_fn=None,
|
|
) -> list[int]:
|
|
"""Kill ledger-registered stdio MCP helpers whose spawner is provably dead.
|
|
|
|
Startup-sweep rung mirroring ``_reap_orphaned_desktop_local_serves`` (dashboard_procs.py), but
|
|
ledger-driven instead of cmdline-heuristic: a helper is reaped ONLY when
|
|
|
|
- it has a live ``(pid, create_time)`` ledger entry for THIS install with purpose ``mcp-helper``
|
|
(``ledger_entries`` already excludes dead/ foreign entries), and - its recorded spawner is
|
|
**provably dead** (``spawner_is_dead`` is ``True`` — never ``None``/unprovable, never a live
|
|
spawner).
|
|
"""
|
|
reaped: list[int] = []
|
|
try:
|
|
entries = ledger_entries(project_root=project_root)
|
|
except Exception:
|
|
return reaped
|
|
own_pid = os.getpid()
|
|
for entry in entries:
|
|
try:
|
|
if entry.get("purpose") != "mcp-helper":
|
|
continue
|
|
pid = entry.get("pid")
|
|
if not isinstance(pid, int) or pid <= 0 or pid == own_pid:
|
|
continue
|
|
if spawner_is_dead(entry) is not True:
|
|
continue # live or unprovable spawner → never touch
|
|
if kill_fn is not None:
|
|
kill_fn(pid)
|
|
else:
|
|
import psutil
|
|
|
|
proc = psutil.Process(pid)
|
|
# Re-verify identity at the moment of kill (PID-reuse guard).
|
|
if not _same_incarnation(proc, entry.get("create_time")):
|
|
continue
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=2.0)
|
|
except psutil.TimeoutExpired:
|
|
proc.kill()
|
|
reaped.append(pid)
|
|
except Exception:
|
|
logger.debug("mcp-helper orphan reap failed for %s", entry, exc_info=True)
|
|
if reaped:
|
|
logger.info("reaped %d orphaned stdio MCP helper(s): %s", len(reaped), reaped)
|
|
return reaped
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Layer 3 — Windows job-object self-attach
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def attach_self_to_kill_on_close_job() -> bool:
|
|
"""Place this process in a job that dies (whole tree) when we die.
|
|
|
|
Windows-only, best-effort, idempotent. ``BREAKAWAY_OK`` is included so children spawned with
|
|
``CREATE_BREAKAWAY_FROM_JOB`` (gateway relaunch during update, detached watchers) keep escaping
|
|
exactly as they do today.
|
|
"""
|
|
global _JOB_HANDLE
|
|
if not _IS_WINDOWS or _JOB_HANDLE is not None:
|
|
return _JOB_HANDLE is not None
|
|
try:
|
|
import ctypes
|
|
from ctypes import wintypes
|
|
|
|
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
|
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
|
|
JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x0800
|
|
JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x1000
|
|
JobObjectExtendedLimitInformation = 9
|
|
|
|
class IO_COUNTERS(ctypes.Structure):
|
|
_fields_ = [(n, ctypes.c_ulonglong) for n in (
|
|
"ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
|
|
"ReadTransferCount", "WriteTransferCount", "OtherTransferCount")]
|
|
|
|
class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
|
|
_fields_ = [
|
|
("PerProcessUserTimeLimit", wintypes.LARGE_INTEGER),
|
|
("PerJobUserTimeLimit", wintypes.LARGE_INTEGER),
|
|
("LimitFlags", wintypes.DWORD),
|
|
("MinimumWorkingSetSize", ctypes.c_size_t),
|
|
("MaximumWorkingSetSize", ctypes.c_size_t),
|
|
("ActiveProcessLimit", wintypes.DWORD),
|
|
("Affinity", ctypes.POINTER(wintypes.ULONG)),
|
|
("PriorityClass", wintypes.DWORD),
|
|
("SchedulingClass", wintypes.DWORD),
|
|
]
|
|
|
|
class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
|
|
_fields_ = [
|
|
("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
|
|
("IoInfo", IO_COUNTERS),
|
|
*((n, ctypes.c_size_t) for n in (
|
|
"ProcessMemoryLimit", "JobMemoryLimit", "PeakProcessMemoryUsed", "PeakJobMemoryUsed")),
|
|
]
|
|
|
|
job = kernel32.CreateJobObjectW(None, None)
|
|
if not job:
|
|
return False
|
|
info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
|
|
info.BasicLimitInformation.LimitFlags = (
|
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
| JOB_OBJECT_LIMIT_BREAKAWAY_OK
|
|
| JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK
|
|
)
|
|
ok = kernel32.SetInformationJobObject(
|
|
job, JobObjectExtendedLimitInformation, ctypes.byref(info), ctypes.sizeof(info)
|
|
)
|
|
if not ok or not kernel32.AssignProcessToJobObject(job, kernel32.GetCurrentProcess()):
|
|
kernel32.CloseHandle(job)
|
|
return False
|
|
_JOB_HANDLE = job # keep alive for the life of the process — never close
|
|
logger.debug("attached to kill-on-close job object")
|
|
return True
|
|
except Exception:
|
|
logger.debug("job object self-attach failed", exc_info=True)
|
|
return False
|