Files
hermes-agent/hermes_cli/gateway.py
kshitijk4poor 09472a30a7 fix(gateway): trim reap-grace docs and dead test state (#122533)
Review cleanups on the orphan-reap startup grace:
- Drop the docstring claim that the Desktop boot sweep is "the one caller
  that races a launch": web_server._spawn_gateway_restart also reaps
  (grace-less) before its coalesce check, so the claim was wrong.
- Cut the 7-line lifespan comment to one line; the full reasoning lives in
  the _reap_unsupervised_gateway_orphans docstring, so the two can't drift.
- Drop the never-asserted seen["extra_exclude"] and the constant-only
  `_REAP_MIN_AGE_SECONDS > 0` assert; a grace-less revert already fails the
  `seen["min_age_s"] == _REAP_MIN_AGE_SECONDS` check.

Co-authored-by: Halldrix <halldrix@users.noreply.github.com>
2026-09-27 18:28:26 +05:30

5683 lines
259 KiB
Python

"""Gateway subcommand for hermes CLI.
Handles: hermes gateway [run|start|stop|restart|status|install|uninstall|setup]
"""
import asyncio
import contextlib
from hermes_cli.cli_output import line_input # noqa: F401 — resolved lazily by siblings through the facade
import json
import logging
import os
import shutil
import signal
import socket
import shlex
import subprocess
import sys
import textwrap
import time
from dataclasses import dataclass
from pathlib import Path
from hermes_cli import setup_platforms # noqa: F401 — resolved lazily by siblings through the facade
# UV's bundled Python ships a minimal PATH; ensure launchctl/systemctl are discoverable.
if os.name == "posix":
_sys_dirs = {"/bin", "/usr/bin", "/usr/sbin", "/sbin"}
_path_dirs = set(os.environ.get("PATH", "").split(os.pathsep))
_missing = _sys_dirs - _path_dirs
if _missing:
os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + os.pathsep.join(sorted(_missing))
PROJECT_ROOT = Path(__file__).parent.parent.resolve()
from gateway.config import coerce_systemd_watchdog_seconds, load_gateway_config # noqa: F401 — resolved lazily by siblings through the facade
from gateway.status import terminate_pid
from gateway.restart import ( # noqa: F401 — resolved lazily by siblings through the facade
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
EXTERNAL_GATEWAY_SUPERVISOR_ENV,
GATEWAY_FATAL_CONFIG_EXIT_CODE,
GATEWAY_SERVICE_RESTART_EXIT_CODE,
is_gateway_supervisor_process,
parse_cron_drain_timeout,
parse_restart_after_turn_timeout,
parse_restart_drain_timeout,
resolve_restart_exit_wait_budget,
resolve_systemd_timeout_stop_sec,
)
from hermes_cli.config import ( # noqa: F401 — resolved lazily by siblings through the facade
get_env_value,
get_hermes_home,
is_managed,
managed_error,
read_raw_config,
save_env_value,
write_platform_config_field,
)
# display_hermes_home is imported lazily: hermes_constants may be a cached pre-update version.
from hermes_cli.setup import ( # noqa: F401 — resolved lazily by siblings through the facade
print_header,
print_info,
print_success,
print_warning,
print_error,
prompt,
prompt_choice,
prompt_yes_no,
)
from hermes_cli.colors import Colors, color
logger = logging.getLogger(__name__)
# Shared ``subprocess.run`` kwargs for text-mode probes (stdout/stderr captured, decode-tolerant).
_CAPTURE_TEXT = dict(capture_output=True, text=True, encoding="utf-8", errors="replace")
# =============================================================================
# Process Management (for manual gateway runs)
# =============================================================================
@dataclass(frozen=True)
class GatewayRuntimeSnapshot:
manager: str
service_installed: bool = False
service_running: bool = False
gateway_pids: tuple[int, ...] = ()
service_scope: str | None = None
@property
def running(self) -> bool:
return self.service_running or bool(self.gateway_pids)
@property
def has_process_service_mismatch(self) -> bool:
return self.service_installed and self.running and not self.service_running
@dataclass(frozen=True)
class ProfileGatewayProcess:
profile: str
path: Path
pid: int
create_time: float = 0.0
@dataclass(frozen=True)
class WindowsGatewayService:
"""A real Windows service supervising a profile gateway process tree."""
name: str
profile: str
service_pid: int
gateway_pid: int
descendant_pids: frozenset[int]
descendant_identities: tuple[tuple[int, float], ...]
service_create_time: float = 0.0
gateway_create_time: float = 0.0
def _get_service_pids(all_profiles: bool = False) -> set:
"""PIDs managed by systemd/launchd gateway services (excluded from stale-process sweeps).
Relies on the service manager committing the new PID before the restart command returns.
``all_profiles`` widens the current profile's unit/label to the whole ``hermes-gateway*`` /
``ai.hermes.gateway*`` fleet so update/reaper never kill a sibling's service gateway as "manual".
``all_profiles`` widens the launchd branch to every installed ``ai.hermes.gateway*`` LaunchAgent — the
update path needs the whole fleet excluded from its sweep (#41403, #73626): sibling-profile launchd
gateways found by the (BSD-fixed) ps scan must not be misclassified as manual processes and killed.
Default-scope callers (``gateway status``, cron checks) keep seeing only the current profile's service;
the orphan reaper passes all_profiles=True for the same friendly-fire reason. The systemd branch mirrors
this: default scope filters to the current profile's exact unit name; ``all_profiles=True`` widens to
the ``hermes-gateway*`` fleet glob.
"""
pids: set = set()
# --- systemd (Linux): user and system scopes ---
if supports_systemd_services():
pattern = "hermes-gateway*" if all_profiles else get_service_name()
for scope_args in [["systemctl", "--user"], ["systemctl"]]:
try:
# Belt-and-suspenders for the EXCLUDE use case (#74075): a bare ``launchctl list`` prefix
# scan also catches ai.hermes.gateway* agents the label derivation can't map (renamed
# profiles, other installs sharing this user). Over-inclusion is safe here — these PIDs are
# only ever protected from the kill sweep, never targeted. Restart paths use the
# label-derived set only.
result = subprocess.run(
scope_args
+ ["list-units", pattern, "--plain", "--no-legend", "--no-pager"],
timeout=5,
**_CAPTURE_TEXT,
)
for line in result.stdout.strip().splitlines():
parts = line.split()
if not parts or not parts[0].endswith(".service"):
continue
svc = parts[0]
try:
show = subprocess.run(
scope_args + ["show", svc, "--property=MainPID", "--value"],
timeout=5,
**_CAPTURE_TEXT,
)
pid = int(show.stdout.strip())
if pid > 0:
pids.add(pid)
except (ValueError, subprocess.TimeoutExpired):
pass
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
# --- launchd (macOS) ---
if is_macos():
labels = {get_launchd_label()}
if all_profiles:
# Whole fleet, mirroring the systemd ``hermes-gateway*`` glob above.
# Every gateway LaunchAgent, not just the invoking profile's — mirrors the systemd branch's
# ``hermes-gateway*`` pattern above. The update path restarts the whole fleet, and its
# stale-process sweep must not mistake a sibling service's fresh PID for a manual gateway it
# should kill (#41403).
labels.update(launchd_gateway_labels_for_install())
for label in sorted(labels):
try:
_domain, pid = _locate_launchd_gateway_service(label)
except subprocess.TimeoutExpired:
continue
if pid is not None and pid > 0:
pids.add(pid)
if all_profiles:
# Prefix scan also catches ai.hermes.gateway* agents the label derivation can't map
# (renamed profiles, other installs). Over-inclusion is safe: PIDs are only protected.
try:
result = subprocess.run(["launchctl", "list"], timeout=5, **_CAPTURE_TEXT)
if result.returncode == 0:
for line in result.stdout.strip().splitlines():
parts = line.split()
if len(parts) >= 3 and parts[-1].startswith("ai.hermes.gateway"):
try:
pid = int(parts[0])
if pid > 0:
pids.add(pid)
except ValueError:
pass
except (FileNotFoundError, subprocess.TimeoutExpired):
pass
return pids
def _get_parent_pid(pid: int) -> int | None:
"""Parent PID for ``pid``, or None. psutil first (works on Windows, where ``ps`` doesn't)."""
if pid <= 1:
return None
try:
import psutil # type: ignore
return psutil.Process(pid).ppid() or None
except ImportError:
pass
except Exception:
return None
# ps fallback, POSIX only: Git Bash's ps.exe would flash a console from the windowless backend.
if is_windows() or not shutil.which("ps"):
return None
try:
result = subprocess.run(["ps", "-o", "ppid=", "-p", str(pid)], timeout=5, **_CAPTURE_TEXT)
except (FileNotFoundError, subprocess.TimeoutExpired):
return None
raw = result.stdout.strip()
if result.returncode != 0 or not raw:
return None
try:
parent_pid = int(raw.splitlines()[-1].strip())
except ValueError:
return None
return parent_pid if parent_pid > 0 else None
def _is_pid_ancestor_of_current_process(target_pid: int) -> bool:
"""Return True when ``target_pid`` is this process or one of its ancestors."""
if target_pid <= 0:
return False
pid = os.getpid()
seen: set[int] = set()
while pid and pid not in seen:
if pid == target_pid:
return True
seen.add(pid)
pid = _get_parent_pid(pid) or 0
return False
def _request_gateway_self_restart(pid: int) -> bool:
"""Ask a running gateway ancestor to restart itself asynchronously."""
if not hasattr(signal, "SIGUSR1") or not _is_pid_ancestor_of_current_process(pid):
return False
try:
os.kill(pid, signal.SIGUSR1) # windows-footgun: ok — POSIX signal, guarded by hasattr(signal, 'SIGUSR1') above
except (ProcessLookupError, PermissionError, OSError):
return False
return True
def _graceful_restart_via_sigusr1(pid: int, drain_timeout: float, *, on_progress=None) -> bool:
"""SIGUSR1 (drain-aware restart) a gateway PID and wait for exit; False if unsent or it outlived the timeout.
gateway/run.py maps SIGUSR1 to ``request_restart(via_service=True)``: refuse new turns, drain,
``stop()``, exit; the supervisor relaunches. ``drain_timeout`` must cover after-turn wait + the full stop
envelope — pass ``resolve_restart_exit_wait_budget(...)``. ``on_progress`` (zero-arg) runs on every poll so
a long wait can report what the gateway is still holding for (``update_cmd_drain_report``).
"""
if not hasattr(signal, "SIGUSR1") or pid <= 0:
return False
try:
os.kill(pid, signal.SIGUSR1) # windows-footgun: ok — POSIX signal, guarded by hasattr(signal, 'SIGUSR1') above
except ProcessLookupError:
return True
except (PermissionError, OSError):
return False
return _wait_for_pid_exit(pid, max(drain_timeout, 1.0), on_progress=on_progress)
def _wait_for_pid_exit(pid: int, timeout: float, *, on_progress=None) -> bool:
"""Wait up to ``timeout``s for ``pid`` to exit; True once gone. (``launchctl bootstrap`` fails EIO
while the previous instance still drains, so teardown callers must wait for the real exit.)"""
if pid <= 0:
return True
# ``os.kill(pid, 0)`` hard-kills on Windows (TerminateProcess); use _pid_exists instead.
from gateway.status import _pid_exists
deadline = time.monotonic() + max(timeout, 0.0)
while True:
if not _pid_exists(pid):
return True
if time.monotonic() >= deadline:
return False
if on_progress is not None:
on_progress()
time.sleep(0.5)
# --- Wedged-gateway detection + bounded escalation ---------------------------
# A gateway whose asyncio loop is stalled cannot handle SIGTERM/SIGUSR1, so the drain wait burns
# its full budget and `hermes update` can deadlock. Two witnesses classify the loop BEFORE any
# drain wait: the heartbeat file ``state/gateway.heartbeat`` (rewritten every 30s on a thread, so
# staleness alone is not proof) and the loop-tick socket ``state/gateway.loop-tick.<pid>.sock``
# answered by the loop itself; the payload records whether the socket is armed (``loop_tick_socket``).
# ``alive``: socket answered, or fresh file not contradicted -> normal graceful drain. ``wedged``:
# heartbeat is this PID's, stale past several beats, AND the armed socket stays silent across
# ``tick_strikes`` consecutive misses -> callers may ``_escalate_wedged_gateway``; one silent probe
# is never authority. ``unknown``: no/unreadable heartbeat, PID mismatch, or witness conflict ->
# treated as alive; never escalate on ambiguity. Legacy payloads (no ``loop_tick_socket`` flag)
# wrote on-loop, so staleness alone remains proof.
# --- Wedged-gateway detection + bounded escalation (#81642) ----------------- A gateway whose asyncio loop
# is stalled (e.g. an in-loop compression pass, #72707) cannot process SIGTERM/SIGUSR1 shutdown: the drain
# wait then burns the full drain budget (180s by default), warns "still running after 180.0s — restart may
# fail", and `hermes update` can deadlock behind it. The loop publishes a liveness signal precisely for this
# case: an asyncio task rewrites ``state/gateway.heartbeat`` every 30s (#66892), so a frozen loop stops
# refreshing the file while a busy-but-alive loop keeps refreshing it. Since #90502 the heartbeat write runs
# on a thread (a stalling filesystem must not be able to block the loop the watchdog watches), which costs
# the file its status as *proof*: a stalled write or a saturated executor can age the file while the loop
# runs, and an off-loop write can land after the loop froze, keeping the file fresh for a dead loop. The
# loop therefore also arms a second witness — ``state/gateway.loop-tick.<pid>.sock``, a UNIX socket answered
# by the loop itself — and records whether it is armed in the heartbeat payload (``loop_tick_socket``).
# ``probe_gateway_loop_liveness`` reads both signals (a local stat + JSON read + a bounded socket ping,
# repeated up to ``tick_strikes`` times when a wedge is suspected — worst case ~3.4s, still far inside the
# 10s query tier of the subprocess timeout doc) and classifies the gateway BEFORE any drain wait begins: -
# ``alive`` — the loop answered the tick socket, or the file is fresh and the loop is not contradicted by
# the socket. Callers must take the normal graceful-drain path, which honours the in-flight cron drain floor
# (#86684). - ``wedged`` — the heartbeat belongs to this PID, is stale well past several missed beats, AND
# the tick socket is armed but stays silent across a sustained window of consecutive misses (default 3):
# both witnesses agree, sustained, that the loop is provably dead. One silent probe is never destructive
# authority — a transient synchronous stall can outlast a single recv timeout, so a lone miss falls to
# ``unknown``. Draining is pointless for a provably dead loop (nothing can run the drain), so callers may
# escalate immediately via ``_escalate_wedged_gateway``. - ``unknown`` — no heartbeat / unreadable / PID
# mismatch / witness conflict (fresh file with a silent loop, armed socket unreachable). Treated like
# ``alive``: never escalate on ambiguity. The distinction matters: only a *provably dead* loop may bypass
# the cron drain floor. A merely busy gateway still answers the probe (socket ping) and keeps its full drain
# budget — even when the filesystem is stalling the heartbeat write (the incident that motivated #90502).
# Legacy gateways (no ``loop_tick_socket`` flag in the payload) wrote the file on-loop, so their staleness
# remains proof and the old single-witness contract is unchanged.
GATEWAY_LOOP_ALIVE = "alive"
GATEWAY_LOOP_WEDGED = "wedged"
GATEWAY_LOOP_UNKNOWN = "unknown"
# 3 missed 30s beats (gateway.shutdown_watchdog.DEFAULT_HEARTBEAT_INTERVAL_S): decisive, not one slow write.
DEFAULT_LOOP_LIVENESS_STALE_AFTER_S = 90.0
# Sentinel for "the producer never wrote the witness flag" (legacy payload).
_LOOP_TICK_ABSENT = object()
def _probe_loop_tick_socket(pid: int, home: Path | None, timeout: float = 1.0) -> bool | None:
"""Ping the loop-tick witness socket: True answered, False node present but silent, None no node (not evidence)."""
try:
from gateway.shutdown_watchdog import get_loop_tick_socket_path
path = get_loop_tick_socket_path(home, pid)
if not path.is_socket():
return None
except Exception:
return None
return _ping_loop_tick_witness(socket.AF_UNIX, str(path), timeout)
def _ping_loop_tick_witness(family: int, address, timeout: float) -> bool:
"""Connect to a loop-tick witness and expect one byte ``"1"``; False on refusal/timeout/any error."""
sock = None
try:
sock = socket.socket(family, socket.SOCK_STREAM)
sock.settimeout(max(float(timeout), 0.0))
sock.connect(address)
return sock.recv(1) == b"1"
except Exception:
return False
finally:
if sock is not None:
with contextlib.suppress(Exception):
sock.close()
def _probe_loop_tick_tcp(port: int, timeout: float = 1.0) -> bool | None:
"""TCP-loopback variant of the tick probe for Windows (no AF_UNIX in asyncio); same semantics, None
on invalid port."""
try:
port_num = int(port)
if port_num <= 0 or port_num > 65535:
return None
except (TypeError, ValueError):
return None
return _ping_loop_tick_witness(socket.AF_INET, ("127.0.0.1", port_num), timeout)
def _probe_loop_tick_socket_sustained(
pid: int, home: Path | None, *, timeout: float = 1.0, strikes: int = 3, gap_s: float = 0.2,
tcp_port: int | None = None,
) -> bool | None:
"""Probe the tick socket up to ``strikes`` times, ``gap_s`` apart: True once answered, False if a node
stayed silent the whole window, None if the node vanished (not evidence). One silent probe is not
destructive evidence — a transient synchronous stall can outlast one recv timeout.
A single silent probe is NOT destructive evidence: the loop may be in a short transient synchronous
stall (a reconnect storm, a heavy synchronous callback, scheduler delay) that outlasts one recv timeout.
Killing a gateway on that would be a false wedge — the exact class of false positive #90502 exists to
prevent. Destructive authority therefore requires the loop to fail to answer across a bounded window of
``strikes`` consecutive misses, ``gap_s`` apart; any answer inside the window proves the loop is
dispatching and returns ``True``.
"""
total = max(int(strikes), 0)
for attempt in range(total):
if tcp_port is not None:
result = _probe_loop_tick_tcp(tcp_port, timeout=timeout)
else:
result = _probe_loop_tick_socket(pid, home, timeout=timeout)
if result is True:
return True
if result is None:
# No node: ambiguity, never a wedge — absence is not a miss.
return None
if attempt < total - 1 and gap_s > 0:
time.sleep(gap_s)
return False
def probe_gateway_loop_liveness(
pid: int, *, stale_after: float = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S, home: Path | None = None,
tick_timeout: float = 1.0, tick_strikes: int = 3, tick_gap_s: float = 0.2,
) -> str:
"""Classify a gateway PID's event loop as alive / wedged / unknown (see block comment above).
Stale heartbeat is ``wedged`` only when the payload declares the tick socket armed AND it stays
silent across ``tick_strikes`` misses; any answer is ``alive``; ambiguity is ``unknown``.
- the loop-tick socket (``state/gateway.loop-tick.<pid>.sock``): answered by the gateway loop itself, so
a reply is direct proof that the loop is dispatching. It is never refreshed by the heartbeat executor
thread and never stalled by a filesystem that is slow to fsync. - the heartbeat file
(``state/gateway.heartbeat``): rewritten every 30s on a thread since #90502, so freshness alone is no
longer proof of loop schedulability — a stalled write (measured at 112.6s max on the incident box) or a
saturated executor can age the file while the loop runs, and a write can land after the loop froze.
"""
try:
stale_budget = max(float(stale_after), 0.0)
except (TypeError, ValueError):
stale_budget = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S
try:
from gateway.shutdown_watchdog import get_loop_heartbeat_path
path = get_loop_heartbeat_path(home)
mtime = path.stat().st_mtime
payload = json.loads(path.read_text(encoding="utf-8-sig"))
heartbeat_pid = int(payload.get("pid", 0))
except Exception:
return GATEWAY_LOOP_UNKNOWN
if heartbeat_pid <= 0 or int(pid) <= 0 or heartbeat_pid != int(pid):
# Heartbeat is not this process's (old version, starting up, stale file): not evidence.
return GATEWAY_LOOP_UNKNOWN
# TCP loopback witness (Windows) takes priority when published; else the AF_UNIX socket.
tcp_port = payload.get("loop_tick_tcp_port")
try:
tcp_port_int = int(tcp_port) if tcp_port is not None else None
except (TypeError, ValueError):
tcp_port_int = None
if tcp_port_int is not None and tcp_port_int > 0:
witness = _probe_loop_tick_tcp(tcp_port_int, timeout=tick_timeout)
tick_armed = True
else:
witness = _probe_loop_tick_socket(pid, home, timeout=tick_timeout)
tick_armed = payload.get("loop_tick_socket", _LOOP_TICK_ABSENT)
if witness is True:
# Loop answered: a stale file is a stalled write, not a wedge.
return GATEWAY_LOOP_ALIVE
# The loop answered a ping — it is dispatching right now. See #90502.
age = time.time() - mtime
if age <= stale_budget:
if witness is False:
# Fresh file but silent loop: an off-loop write can land after the loop froze.
return GATEWAY_LOOP_UNKNOWN
return GATEWAY_LOOP_ALIVE
# Stale past the budget; the verdict depends on what the producer promised about its witness.
if tick_armed is _LOOP_TICK_ABSENT:
# Legacy on-loop writer: staleness proves the loop stopped scheduling.
return GATEWAY_LOOP_WEDGED
if tick_armed is not True:
# Witness could not be armed (bind failed); off-loop write means staleness is not proof.
return GATEWAY_LOOP_UNKNOWN
if witness is False:
# First miss. The probe above is miss #1, so ``tick_strikes - 1`` more attempts follow.
# One silent probe is NOT destructive authority: a short transient synchronous stall can outlast a
# single recv timeout, and killing a live gateway on it would be the exact false wedge #90502 exists
# to prevent.
sustained = _probe_loop_tick_socket_sustained(
pid, home, timeout=tick_timeout, strikes=tick_strikes - 1, gap_s=tick_gap_s, tcp_port=tcp_port_int
)
if sustained is False:
return GATEWAY_LOOP_WEDGED
if sustained is True:
return GATEWAY_LOOP_ALIVE # Transient stall, not a wedge.
return GATEWAY_LOOP_UNKNOWN # Witness vanished mid-window: ambiguity — never kill on it.
return GATEWAY_LOOP_UNKNOWN # Armed but unreachable socket: ambiguity — never kill on it.
def _escalate_wedged_gateway(pid: int, *, term_grace: float = 5.0, kill_wait: float = 5.0) -> bool:
"""Bounded stop (SIGTERM, ``term_grace``, SIGKILL, ``kill_wait``) for a provably dead loop; True once gone.
Callers MUST have classified ``GATEWAY_LOOP_WEDGED`` first: escalating a merely busy gateway
bypasses the cron drain floor and SIGKILLs live work.
See #86684.
"""
from gateway.status import get_process_start_time
expected_start_time = get_process_start_time(pid)
try:
terminate_pid(pid, force=False)
except (ProcessLookupError, PermissionError, OSError):
return _wait_for_pid_exit(pid, 1.0)
if _wait_for_pid_exit(pid, max(float(term_grace), 0.0)):
return True
try:
terminate_pid(pid, force=True, expected_start_time=expected_start_time)
print(f"⚠ Gateway PID {pid} unresponsive to SIGTERM; sent SIGKILL")
except (ProcessLookupError, PermissionError, OSError):
pass
return _wait_for_pid_exit(pid, max(float(kill_wait), 0.0))
def _get_ancestor_pids() -> set[int]:
"""PIDs of this process and its ancestors, so scans never count the invoking ``hermes`` CLI as a gateway.
Walks from the current PID up to PID 1 (init) so that process-table scans never match the calling CLI
process or any of its parents. This prevents ``hermes gateway status`` from falsely counting the
``hermes`` CLI that invoked it as a running gateway instance (see #13242).
"""
ancestors: set[int] = set()
pid = os.getpid()
for _ in range(64):
ancestors.add(pid)
parent = _get_parent_pid(pid)
if parent is None or parent <= 0 or parent in ancestors:
break
pid = parent
return ancestors
def _append_unique_pid(pids: list[int], pid: int | None, exclude_pids: set[int]) -> None:
if pid and pid > 0 and pid != os.getpid() and pid not in exclude_pids and pid not in pids:
pids.append(pid)
def _iter_proc_cmdlines(exclude_pids: set[int]):
"""Yield ``(pid, cmdline)`` from ``/proc`` (Docker without procps); raises if /proc is unusable."""
my_pid = os.getpid()
for entry in os.listdir("/proc"):
if not entry.isdigit():
continue
pid = int(entry)
if pid == my_pid or pid in exclude_pids:
continue
try:
with open(f"/proc/{pid}/cmdline", "rb") as _f:
cmdline = _f.read().decode("utf-8", errors="replace")
except (OSError, PermissionError):
continue
yield pid, cmdline.replace("\x00", " ")
def _scan_gateway_pids(
exclude_pids: set[int], all_profiles: bool = False, include_restart_managers: bool = False
) -> list[int]:
"""Best-effort process-table scan for gateway PIDs (backs up a stale/missing PID file; ``--all`` sweeps)."""
# Exclude the entire ancestor chain so the CLI process that invoked this scan (e.g. ``hermes gateway
# status``) is never mistaken for a running gateway. See #13242.
exclude_pids = exclude_pids | _get_ancestor_pids()
pids: list[int] = []
# Strict matcher shared with gateway.status: requires a real ``gateway run`` argv, so
# ``gateway status``/``dashboard`` siblings and ``python -m tui_gateway`` don't match.
from gateway.status import (
looks_like_gateway_command_line,
looks_like_gateway_runtime_command_line,
profile_flag_value,
hermes_home_assignments,
command_line_names_hermes_home,
)
current_home = str(get_hermes_home().resolve())
# Forward slashes on both sides of the HERMES_HOME= match (mirrors gateway.status), and no
# trailing separator: the assignments parser strips one, so the systemd ``Environment=``
# spelling (``HERMES_HOME=/root/.hermes/``) compares equal to the resolved home.
current_home_lc = current_home.lower().replace("\\", "/").rstrip("/")
current_profile_arg = _profile_arg(current_home)
current_profile_name = current_profile_arg.split()[-1] if current_profile_arg else ""
current_profile_name_lc = current_profile_name.lower()
def _matches_current_profile(command: str) -> bool:
command_lc = command.lower().replace("\\", "/")
if current_profile_name:
# Token equality, not substring: `-p ops` must not claim (or SIGTERM) an `-p ops-2` gateway.
if profile_flag_value(command_lc) == current_profile_name_lc:
return True
return command_line_names_hermes_home(command_lc, current_home_lc)
# Default profile: accept unless argv advertises another profile in any spelling the CLI
# pre-parser accepts (``--profile=ops`` slipped past a substring test, so a default-profile
# fallback stop could SIGTERM the named gateway). HERMES_HOME may come via env (invisible to
# wmic/CIM), so only a non-matching explicit HERMES_HOME= disqualifies.
if profile_flag_value(command_lc) is not None:
return False
return (not hermes_home_assignments(command_lc)
or command_line_names_hermes_home(command_lc, current_home_lc))
def _consider(pid: int, command: str) -> None:
matches_runtime = looks_like_gateway_command_line(command) or (
include_restart_managers and looks_like_gateway_runtime_command_line(command)
)
if matches_runtime and (all_profiles or _matches_current_profile(command)):
_append_unique_pid(pids, pid, exclude_pids)
try:
if is_windows():
listing = _windows_process_listing()
if listing is None:
return []
for pid, command in _iter_windows_list_processes(listing):
_consider(pid, command)
else:
# /proc first (Docker without procps), then `ps -Aww`.
_found_via_proc = False
if os.path.isdir("/proc"):
try:
for pid, command in _iter_proc_cmdlines(exclude_pids):
_consider(pid, command)
_found_via_proc = True
except Exception:
pass
if not _found_via_proc:
# ``-Aww`` not ``-A eww``: BSD/macOS ps rejects ``e``; ``-ww`` = unlimited width.
result = subprocess.run(["ps", "-Aww", "-o", "pid=,command="], timeout=10, **_CAPTURE_TEXT)
if result.returncode != 0:
return []
for line in result.stdout.split("\n"):
parsed = _parse_ps_line(line)
if parsed is not None:
_consider(*parsed)
except (OSError, subprocess.TimeoutExpired):
return []
# Windows: a venv ``pythonw.exe`` is a launcher stub that spawns the base Python with the same
# command line, so each gateway yields two matched PIDs. Drop a matched PID that parents another.
if is_windows() and len(pids) > 1:
pids = _filter_venv_launcher_stubs(pids)
return pids
def _parse_ps_line(line: str) -> tuple[int, str] | None:
"""``(pid, command)`` from one ``ps -o pid=,command=`` line; also accepts ``ps aux`` rows."""
stripped = line.strip()
if not stripped or "grep" in stripped:
return None
parts = stripped.split(None, 1)
if len(parts) == 2:
with contextlib.suppress(ValueError):
return int(parts[0]), parts[1]
aux_parts = stripped.split()
if len(aux_parts) > 10 and aux_parts[1].isdigit():
return int(aux_parts[1]), " ".join(aux_parts[10:])
return None
def _iter_windows_list_processes(listing: str):
"""Yield ``(pid, command_line)`` from wmic/CIM ``/FORMAT:LIST`` output."""
current_cmd = ""
for line in listing.split("\n"):
line = line.strip()
if line.startswith("CommandLine="):
current_cmd = line[len("CommandLine=") :]
elif line.startswith("ProcessId="):
with contextlib.suppress(ValueError):
yield int(line[len("ProcessId=") :]), current_cmd
current_cmd = ""
def _windows_process_listing() -> str | None:
"""``CommandLine=``/``ProcessId=`` LIST output for every Windows process (wmic, else Get-CimInstance), or None.
``bounded_probe_run``, NOT ``subprocess.run(timeout=...)``: run()'s post-timeout cleanup joins pipe
readers unbounded and a conhost.exe holding duplicated handles wedges the caller forever; it also
hides the console window this windowless pythonw backend would flash."""
# Prefer wmic when present (fast, stable output format). On modern Windows 11 / Win 10 late builds, wmic
# has been removed as part of the WMIC deprecation — fall back to PowerShell's Get-CimInstance. A spawn
# failure or timeout (result is None) trips the fallback. ``hermes update`` hung exactly there on
# slow-WMI machines where the full Win32_Process scan exceeds its budget (#87134). bounded_probe_run
# also hides the console window: this scan runs inside the windowless pythonw.exe gateway/desktop
# backend, so a bare wmic/powershell spawn would flash a conhost window on every watchdog probe.
from hermes_cli._subprocess_compat import bounded_probe_run
wmic_path = shutil.which("wmic")
result = None
if wmic_path is not None:
result = bounded_probe_run(
[wmic_path, "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], timeout=10, errors="ignore"
)
if result is None or result.returncode != 0 or not (result.stdout or ""):
powershell = shutil.which("powershell") or shutil.which("pwsh")
if powershell is None:
return None
ps_cmd = (
"Get-CimInstance Win32_Process | "
"ForEach-Object { "
" 'CommandLine=' + ($_.CommandLine -replace \"`r`n\",' ' -replace \"`n\",' '); "
" 'ProcessId=' + $_.ProcessId; "
" '' "
"}"
)
result = bounded_probe_run([powershell, "-NoProfile", "-Command", ps_cmd], timeout=15, errors="ignore")
if result is None:
return None
return None if result.returncode != 0 or result.stdout is None else result.stdout
def _filter_venv_launcher_stubs(pids: list[int]) -> list[int]:
"""Drop venv-launcher ``pythonw.exe`` stubs that parent another matched PID (see ``_scan_gateway_pids``)."""
try:
import psutil # type: ignore
except ImportError:
return pids
pid_set = set(pids)
drop: set[int] = set()
for pid in pids:
try:
ppid = psutil.Process(pid).ppid()
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
if ppid is not None and ppid in pid_set:
drop.add(ppid)
return [p for p in pids if p not in drop]
def find_gateway_pids(exclude_pids: set | None = None, all_profiles: bool = False) -> list:
"""Find running gateway PIDs for the current profile, or every profile with ``all_profiles`` (``hermes update``)."""
_exclude = set(exclude_pids or set())
pids: list[int] = []
if not all_profiles:
try:
from gateway.status import get_running_pid
_append_unique_pid(pids, get_running_pid(), _exclude)
except Exception:
pass
for pid in _get_service_pids(all_profiles=all_profiles):
_append_unique_pid(pids, pid, _exclude)
try:
include_restart_managers = not supports_systemd_services()
except Exception:
include_restart_managers = False
for pid in _scan_gateway_pids(_exclude, all_profiles=all_profiles, include_restart_managers=include_restart_managers):
_append_unique_pid(pids, pid, _exclude)
return pids
def find_profile_gateway_processes(exclude_pids: set | None = None, *, strict: bool = False) -> list[ProfileGatewayProcess]:
"""Return running gateway PIDs mapped to Hermes profiles via PID files."""
_exclude = set(exclude_pids or set())
processes: list[ProfileGatewayProcess] = []
try:
from gateway.status import get_running_pid, get_running_pid_identity_strict
from hermes_cli.profiles import list_profiles
except Exception:
if strict:
raise
return processes
seen: set[int] = set()
try:
profiles = list_profiles()
except Exception:
if strict:
raise
return processes
for profile in profiles:
try:
if strict:
identity = get_running_pid_identity_strict(profile.path / "gateway.pid")
pid = identity[0] if identity else None
create_time = identity[1] if identity else 0.0
else:
pid = get_running_pid(profile.path / "gateway.pid", cleanup_stale=False)
create_time = 0.0
except Exception as exc:
if strict:
raise RuntimeError(f"Could not inspect gateway PID for profile {profile.name}") from exc
continue
if pid is None or pid <= 0 or pid in _exclude or pid in seen:
continue
seen.add(pid)
processes.append(ProfileGatewayProcess(profile=profile.name, path=profile.path, pid=pid, create_time=create_time))
return processes
def _scm_service_field(service, field: str):
"""psutil ``WindowsService`` exposes getters as methods; ``as_dict()`` covers objects without them."""
getter = getattr(service, field, None)
return getter() if callable(getter) else service.as_dict().get(field)
def find_windows_gateway_services(
*, psutil_module=None, profile_processes: list[ProfileGatewayProcess] | None = None
) -> list[WindowsGatewayService]:
"""Profile gateways supervised by real, Hermes-owned Windows services. Service-logon processes may
hide their command lines, so identity = Hermes's own PID file + a parent chain ending at a running
SCM service PID whose name or binary path is Hermes's (``gateway_windows.hermes_owns_windows_service``).
The whole service subtree is returned so the Desktop preflight exempts exactly what the updater stops
through the SCM; a gateway under any other service (a Scheduled Task's svchost) is a plain process."""
if sys.platform != "win32":
return []
try:
if psutil_module is None:
import psutil as psutil_module # type: ignore[no-redef] # noqa: PLC0415
if profile_processes is None:
profile_processes = find_profile_gateway_processes(strict=True)
from hermes_cli.gateway_windows import hermes_owns_windows_service, hermes_service_roots
hermes_roots = hermes_service_roots()
service_names_by_pid: dict[int, set[str]] = {}
indeterminate_services_by_pid: dict[int, list[tuple[str, object]]] = {}
for service in psutil_module.win_service_iter():
try:
service_name = str(_scm_service_field(service, "name") or "")
if not service_name:
raise RuntimeError("SCM service has an empty name")
# Ownership before state: an OS service above the gateway (Task Scheduler's svchost for a
# task-launched gateway, BITS mid-transition) is never its supervisor, so neither its
# PID nor its status may steer the pause. Only Hermes-owned services reach the guards below.
# The name alone settles Hermes-named services; binpath (QueryServiceConfig) is asked only
# for the rest, and a service that refuses even that to this user is one this user could
# not `sc stop` either — never Hermes's, never a reason to abort the enumeration.
owned = hermes_owns_windows_service(service_name, "", hermes_roots)
if not owned:
try:
service_binpath = str(_scm_service_field(service, "binpath") or "")
except (psutil_module.AccessDenied, OSError):
continue
owned = hermes_owns_windows_service(service_name, service_binpath, hermes_roots)
if not owned:
continue
service_status = _scm_service_field(service, "status")
service_pid = int(_scm_service_field(service, "pid") or 0)
except FileNotFoundError:
# Deleted between enumeration and inspection.
continue
except Exception as exc:
raise RuntimeError("SCM service inspection failed") from exc
if service_status == "stopped":
continue
if service_status != "running":
if service_pid > 0:
indeterminate_services_by_pid.setdefault(service_pid, []).append((service_name, service_status))
continue
if service_pid <= 0:
raise RuntimeError(f"Running SCM service {service_name} has no valid process ID")
service_names_by_pid.setdefault(service_pid, set()).add(service_name)
except Exception as exc:
raise RuntimeError("SCM service enumeration failed") from exc
found: dict[str, WindowsGatewayService] = {}
for profile_process in profile_processes:
try:
gateway_process = psutil_module.Process(int(profile_process.pid))
gateway_create_time = float(gateway_process.create_time())
if profile_process.create_time <= 0 or abs(gateway_create_time - profile_process.create_time) > 0.001:
raise RuntimeError("Gateway process identity changed during SCM discovery")
ancestor_pids = [int(parent.pid) for parent in gateway_process.parents()]
for pid in ancestor_pids:
indeterminate_services = indeterminate_services_by_pid.get(pid, [])
if indeterminate_services:
service_name, service_status = indeterminate_services[0]
raise RuntimeError(f"SCM service {service_name} has indeterminate status: {service_status}")
shared_service_pids = [pid for pid in ancestor_pids if len(service_names_by_pid.get(pid, set())) > 1]
if shared_service_pids:
raise RuntimeError(
"Gateway ownership is ambiguous under shared SCM host PID(s): "
+ ", ".join(str(pid) for pid in shared_service_pids)
)
service_pid = next((pid for pid in ancestor_pids if len(service_names_by_pid.get(pid, set())) == 1), None)
if service_pid is None:
continue
service_name = next(iter(service_names_by_pid[service_pid]))
service_process = psutil_module.Process(service_pid)
service_create_time = float(service_process.create_time())
descendant_processes = service_process.children(recursive=True)
descendants = frozenset(int(child.pid) for child in descendant_processes)
if int(profile_process.pid) not in descendants:
continue
descendant_identities = tuple(
sorted((int(child.pid), float(child.create_time())) for child in descendant_processes)
)
found[service_name] = WindowsGatewayService(
name=service_name,
profile=str(profile_process.profile),
service_pid=service_pid,
gateway_pid=int(profile_process.pid),
descendant_pids=descendants,
descendant_identities=descendant_identities,
service_create_time=service_create_time,
gateway_create_time=gateway_create_time,
)
except RuntimeError:
raise
except Exception as exc:
raise RuntimeError(f"Could not determine SCM ownership for gateway profile {profile_process.profile}") from exc
return [found[name] for name in sorted(found)]
def _gateway_run_args_for_profile(profile: str) -> list[str]:
from hermes_cli._launchers import runtime_command
args = []
if profile != "default":
args.extend(["--profile", profile])
args.extend(["gateway", "run", "--replace"])
return runtime_command(PROJECT_ROOT, args)
def _capture_gateway_argv(pid: int) -> list[str] | None:
"""Live argv of a running gateway (snapshotted before update kills so unmapped gateways can respawn);
None if psutil is unavailable, the process is gone/denied, or the argv isn't a gateway command."""
if pid <= 1:
return None
try:
import psutil # type: ignore
except ImportError:
return None
try:
argv = list(psutil.Process(pid).cmdline() or [])
except Exception: # NoSuchProcess / AccessDenied / ZombieProcess included
return None
if not argv:
return None
# Never respawn an unrelated process the scan happened to report.
try:
from gateway.status import looks_like_gateway_command_line
if not looks_like_gateway_command_line(" ".join(argv)):
return None
except Exception:
pass
return argv
def _prepare_profile_gateway_update_restart(profile: str, pid: int) -> str | None:
"""Choose who relaunches a profile gateway after ``hermes update``: ``--external-supervisor`` gateways
exit back to their manager (a detached watcher would race its replacement); otherwise arm the
profile-derived detached watcher, falling back to replaying the captured command line.
When the profile-derived relaunch cannot be armed -- typically because ``_gateway_run_args_for_profile``
cannot rebuild a run argv for this profile -- fall back to replaying the process's own captured command
line, which is what ``launch_detached_gateway_restart_by_cmdline`` exists for and what the Windows
post-update path already does for its unmapped gateways. Without this the caller has no way to relaunch
the process and (before #88654) silently left it running pre-update modules against post-update code on
disk. ``argv`` is already captured above, so the fallback costs nothing extra.
"""
argv = _capture_gateway_argv(pid)
if argv and "--external-supervisor" in argv:
return "external-supervisor"
if launch_detached_profile_gateway_restart(profile, pid):
return "detached"
if argv and launch_detached_gateway_restart_by_cmdline(pid, list(argv)):
return "detached-cmdline"
return None
def launch_detached_gateway_restart_by_cmdline(old_pid: int, run_argv: list[str]) -> bool:
"""Relaunch a gateway with no profile→PID-file mapping by replaying its captured argv after exit."""
return old_pid > 0 and bool(run_argv) and _spawn_gateway_restart_watcher(old_pid, list(run_argv))
def launch_detached_profile_gateway_restart(profile: str, old_pid: int) -> bool:
"""Relaunch a manually-run profile gateway after its current PID exits."""
return old_pid > 0 and _spawn_gateway_restart_watcher(
old_pid,
_gateway_run_args_for_profile(profile),
host=profile == "default",
)
GATEWAY_RESTART_WATCHER_TIMEOUT_S = 120
"""How long the detached restart watcher waits for the old PID to exit before giving up.
``hermes update``'s post-relaunch liveness check budgets against this: the watcher spawns the new
gateway only AFTER the old PID is gone, so a verification window shorter than this can expire
before the relaunch it is verifying has even started (#107002).
"""
def _restart_argv_is_host_gateway(argv: list[str]) -> bool:
"""True when *argv* relaunches the host multiplexer, not a named profile's own gateway.
``--profile <name>`` (other than default) is that profile. A selector-less argv is
decided by ALREADY-SETTLED identity, never ambient coordinates alone (#93943):
1. this process's own settled multiplex verdict (``is_multiplex_active`` — set by
boot after ``resolve_multiplex_mode``; the gateway replaying its own restart);
2. the live host gateway's published rendezvous record (proof the RUNNING owner
settled multiplex — the update/fleet process replaying a foreign gateway's
captured argv has no settled flag of its own);
3. only then the compatibility default-root comparison.
"""
if not argv or "gateway" not in argv:
return False
for flag in ("--profile", "-p"):
if flag in argv:
idx = argv.index(flag)
name = argv[idx + 1] if idx + 1 < len(argv) else ""
return name == "default"
if any(part == "--profile=default" for part in argv):
return True
if any(part.startswith("--profile=") and not part.endswith("=default") for part in argv):
return False
try:
from agent.secret_scope import is_multiplex_active
if is_multiplex_active():
return True
except Exception:
pass
# The publishing gateway's SETTLED served set, not this process's ambient home:
# a host launched from a named profile must be replayed as the host even though
# the replaying process (the updater) sits on the named profile's home.
try:
from gateway import host_rendezvous as hr
record = hr.read_record(hr.ROLE_GATEWAY)
if record is not None and hr.liveness_is_proven(record) and len(record.profiles) > 1:
return True
except Exception:
pass
try:
from hermes_constants import get_default_hermes_root, get_hermes_home
return get_hermes_home().resolve() == get_default_hermes_root().resolve()
except Exception:
return False
def _host_gateway_watcher_env() -> dict[str, str]:
"""Scrubbed default-profile env for a detached host-gateway respawn watcher."""
from tools.environments.local import host_gateway_child_env
env = host_gateway_child_env()
env.pop("_HERMES_GATEWAY", None)
return env
def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str], *, host: bool | None = None) -> bool:
"""Spawn the detached watcher that respawns ``run_argv`` once ``old_pid`` exits. Watcher and respawn
both need platform-appropriate detach: POSIX setsid; on Windows ``start_new_session`` does NOT detach
(the watcher would die with the CLI console), so ``windows_detach_popen_kwargs()`` supplies flags."""
if old_pid <= 0 or not run_argv:
return False
from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway, windows_detach_popen_kwargs
# Windows: ``run_argv`` leads with the venv's console ``python.exe`` — the interpreter we want:
# the watcher respawns it under CREATE_NO_WINDOW detach flags so the gateway owns one hidden
# console all descendants inherit and nothing flashes (#54220/#56747). The spec helper
# normalizes the interpreter and captures a stable cwd + env overlay (HERMES_HOME,
# VIRTUAL_ENV, PYTHONPATH) so the respawn doesn't depend on the watcher's cwd. No-op on POSIX.
respawn_cwd = ""
# See gateway_windows.windowless_gateway_restart_spec. See #54220, #56747.
respawn_env_overlay: dict[str, str] = {}
if sys.platform == "win32":
try:
from hermes_cli.gateway_windows import windowless_gateway_restart_spec
run_argv, respawn_cwd, respawn_env_overlay = windowless_gateway_restart_spec(list(run_argv))
except Exception:
# Fall back to the original argv: a visible window beats a failed respawn.
respawn_cwd = ""
respawn_env_overlay = {}
# cwd/env overlay are embedded as JSON literals in the watcher source (no extra argv plumbing).
watcher = textwrap.dedent(
"""
import os
import subprocess
import sys
import time
# Stdlib-only imports: the watcher runs on the updater's interpreter, which after the
# package-manager handoff is the bare store Python without the dependency environment.
# ``-c`` only puts the cwd on sys.path, so name the checkout explicitly.
sys.path.insert(0, {project_root_literal})
from hermes_cli._subprocess_compat import (
_WINDOWS_GATEWAY_BREAKAWAY_ENV, pid_exists_stdlib, windows_detach_flags,
windows_detach_flags_without_breakaway,
)
pid = int(sys.argv[1])
cmd = sys.argv[2:]
_respawn_cwd = {respawn_cwd_literal}
_respawn_env_overlay = {respawn_env_literal}
deadline = time.monotonic() + {watcher_timeout_literal}
while time.monotonic() < deadline:
# ``os.kill(pid, 0)`` is not a no-op on Windows — use the cross-platform existence check.
if not pid_exists_stdlib(pid):
break
time.sleep(0.2)
# Route the respawned gateway's stray stdout/stderr to the same sidecar log _spawn_detached
# uses: with DEVNULL a gateway killed moments after respawn (parent Job Object teardown when
# breakaway is denied) left ZERO trace. Best-effort: DEVNULL when the log dir is unavailable.
_stdio_target = subprocess.DEVNULL
_stdio_fh = None
try:
from hermes_constants import get_hermes_home
from pathlib import Path
_log_dir = Path(get_hermes_home()) / "logs"
_log_dir.mkdir(parents=True, exist_ok=True)
_stdio_fh = open(_log_dir / "gateway-stdio.log", "ab", buffering=0)
_stdio_target = _stdio_fh
except Exception:
pass
# Platform-appropriate detach for the respawned gateway: POSIX start_new_session (setsid);
# Windows needs explicit creationflags. CREATE_BREAKAWAY_FROM_JOB is critical: the watcher may
# itself sit inside a job object (Electron/Tauri parent) and without breakaway the respawned
# gateway dies when that job tears down. See _subprocess_compat.windows_detach_flags().
_popen_kwargs = {{"stdout": _stdio_target, "stderr": _stdio_target}}
# Anchor at the stable working dir and overlay the env (VIRTUAL_ENV / PYTHONPATH /
# HERMES_HOME) the windowless base interpreter needs to import hermes_cli. Empty on POSIX.
if _respawn_cwd:
_popen_kwargs["cwd"] = _respawn_cwd
_base_env = {{**os.environ, **_respawn_env_overlay}}
try:
if sys.platform == "win32":
try:
_popen_kwargs["creationflags"] = windows_detach_flags()
# Stamp the breakaway state exactly like gateway_windows._spawn_detached so the
# respawned gateway's exit-diag / lifecycle records show whether it escaped the
# parent Job Object (a job-teardown kill is otherwise indistinguishable).
_popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "1"}}
subprocess.Popen(cmd, **_popen_kwargs)
except OSError:
# CREATE_BREAKAWAY_FROM_JOB is rejected with ERROR_ACCESS_DENIED when the parent's
# job object refuses breakaway; retry without it (mirrors _spawn_detached).
_popen_kwargs["creationflags"] = windows_detach_flags_without_breakaway()
_popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "0"}}
subprocess.Popen(cmd, **_popen_kwargs)
else:
if _respawn_env_overlay:
_popen_kwargs["env"] = _base_env
_popen_kwargs["start_new_session"] = True
subprocess.Popen(cmd, **_popen_kwargs)
finally:
if _stdio_fh is not None:
try:
_stdio_fh.close()
except OSError:
pass
"""
).strip().format(respawn_cwd_literal=json.dumps(respawn_cwd), respawn_env_literal=json.dumps(respawn_env_overlay),
watcher_timeout_literal=json.dumps(GATEWAY_RESTART_WATCHER_TIMEOUT_S),
project_root_literal=json.dumps(str(PROJECT_ROOT)))
watcher_argv = [sys.executable, "-c", watcher, str(old_pid), *run_argv]
devnull = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
# Host respawn must not inherit a named launcher's dotenv. The watcher copies os.environ
# into the gateway child, so the scrub has to be the watcher's own environ.
watcher_env = _host_gateway_watcher_env() if (
_restart_argv_is_host_gateway(run_argv) if host is None else host
) else None
popen_env = {"env": watcher_env} if watcher_env is not None else {}
# Same detach for the watcher itself, so closing the terminal doesn't kill it.
try:
subprocess.Popen(watcher_argv, **devnull, **popen_env, **windows_detach_popen_kwargs())
except OSError:
# Parent job object rejected CREATE_BREAKAWAY_FROM_JOB; retry without it (Windows only —
# ``start_new_session=True`` cannot raise OSError on POSIX).
fallback_kwargs: dict = (
{"creationflags": windows_detach_flags_without_breakaway()} if sys.platform == "win32"
else {"start_new_session": True}
)
try:
subprocess.Popen(watcher_argv, **devnull, **popen_env, **fallback_kwargs)
except OSError:
return False
return True
def _systemd_unit_is_active(system: bool) -> bool:
"""``systemctl is-active`` == "active" for the installed unit in ``system`` scope, else False."""
if not get_systemd_unit_path(system=system).exists():
return False
try:
result = _run_systemctl(["is-active", get_service_name()], system=system, timeout=10, **_CAPTURE_TEXT)
except (RuntimeError, subprocess.TimeoutExpired):
return False
return result.stdout.strip() == "active"
def _probe_systemd_service_running(system: bool = False) -> tuple[bool, bool]:
selected_system = _select_systemd_scope(system)
return selected_system, _systemd_unit_is_active(selected_system)
def _parse_kv_pairs(items) -> dict[str, str]:
"""``{key: value}`` from ``KEY=VALUE`` strings (later keys win; values stripped)."""
return {k: v.strip() for k, v in (item.split("=", 1) for item in items if "=" in item)}
def _systemctl_show(properties: tuple[str, ...], *, system: bool) -> dict[str, str]:
"""``systemctl show --property a,b`` for the gateway unit as ``{key: value}``; {} on failure."""
try:
result = _run_systemctl(
["show", get_service_name(), "--no-pager", "--property", ",".join(properties)],
system=_select_systemd_scope(system), timeout=10, **_CAPTURE_TEXT,
)
except (RuntimeError, subprocess.TimeoutExpired, OSError):
return {}
return _parse_kv_pairs(result.stdout.splitlines()) if result.returncode == 0 else {}
def _unit_environment_value(unit_path: Path, name: str) -> str | None:
"""Value of one ``Environment="NAME=…"`` directive in the unit file at *unit_path*, with
systemd's ``\\"``/``\\\\``/``%%`` quoting undone; None when the file or the key is absent."""
try:
text = unit_path.read_text(encoding="utf-8-sig")
except OSError:
return None
for line in text.splitlines():
body = line.strip()
if not body.startswith("Environment="):
continue
body = body[len("Environment=") :].strip()
if body.startswith('"') and body.endswith('"'):
body = body[1:-1].replace('\\"', '"').replace("\\\\", "\\").replace("%%", "%")
if body.startswith(f"{name}="):
return body.split("=", 1)[1].strip() or None
return None
def _hermes_home_pinned_by_unit(unit_path: Path) -> str | None:
"""``HERMES_HOME`` pinned by the unit file at *unit_path*, or None when absent/unreadable."""
return _unit_environment_value(unit_path, "HERMES_HOME")
def _hermes_home_from_systemd_unit_file(system: bool = False) -> str | None:
"""``HERMES_HOME`` from the on-disk unit file - what refresh/compare already read, and reliable under ``sudo``."""
return _hermes_home_pinned_by_unit(get_systemd_unit_path(system=system))
def _sync_hermes_home_from_systemd_unit(system: bool) -> None:
"""Adopt a system-scope unit's ``HERMES_HOME``: under ``sudo`` it is stripped and HOME=/root, so
get_hermes_home() would pick the wrong profile for runtime-status/PID reads."""
if not system:
return
# On-disk unit first; ``systemctl show`` for units that only exist in the manager.
unit_home = (_hermes_home_from_systemd_unit_file(system=True) or "").strip()
if not unit_home:
env_line = _systemctl_show(("Environment",), system=True).get("Environment", "")
unit_home = _parse_kv_pairs(env_line.split()).get("HERMES_HOME", "").strip()
if unit_home and os.environ.get("HERMES_HOME", "").strip() != unit_home:
os.environ["HERMES_HOME"] = unit_home
def _read_systemd_unit_properties(
system: bool = False,
properties: tuple[str, ...] = ("ActiveState", "SubState", "Result", "ExecMainStatus", "MainPID"),
) -> dict[str, str]:
"""Return selected ``systemctl show`` properties for the gateway unit."""
return _systemctl_show(properties, system=system)
def _positive_pid(value) -> int | None:
"""``int(value)`` when it parses and is > 0, else None."""
try:
pid = int(value or 0)
except (TypeError, ValueError):
return None
return pid if pid > 0 else None
def _systemd_main_pid_from_props(props: dict[str, str]) -> int | None:
return _positive_pid(props.get("MainPID", "0") or "0")
def _runtime_state_pid(state: dict | None) -> int:
"""``pid`` recorded in a runtime-status dict; 0 when absent, unparsable, or non-positive."""
return _positive_pid((state or {}).get("pid", 0)) or 0
def _systemd_main_pid(system: bool = False) -> int | None:
return _systemd_main_pid_from_props(_read_systemd_unit_properties(system=system))
def _read_gateway_runtime_status() -> dict | None:
try:
from gateway.status import read_runtime_status
state = read_runtime_status()
except Exception:
return None
return state if isinstance(state, dict) else None
def _systemd_cli_bits(system: bool) -> tuple[str, str, str]:
"""``(sudo_prefix, scope_flag, user_flag)`` for printed hints: ``("sudo ", " --system", "")`` in
system scope, ``("", "", "--user ")`` in user scope."""
return ("sudo ", " --system", "") if system else ("", "", "--user ")
def _wait_for_systemd_service_restart(
*,
system: bool = False,
previous_pid: int | None = None,
timeout: float | None = None,
replacement_observed: list[bool] | None = None,
) -> bool:
"""Wait for the gateway service to become active after a restart handoff."""
svc = get_service_name()
scope_label = _service_scope_label(system).capitalize()
if timeout is None:
timeout = _systemd_restart_wait_timeout(system=system)
deadline = time.monotonic() + timeout
printed_runtime_wait = False
while time.monotonic() < deadline:
props = _read_systemd_unit_properties(system=system)
active_state = props.get("ActiveState", "")
sub_state = props.get("SubState", "")
try:
from gateway.status import get_running_pid
new_pid = get_running_pid()
except Exception:
new_pid = None
new_pid = new_pid or _systemd_main_pid_from_props(props)
runtime_state = _read_gateway_runtime_status()
runtime_pid = _runtime_state_pid(runtime_state)
if (
previous_pid is not None
and replacement_observed is not None
and not replacement_observed
and any(p > 0 and p != previous_pid for p in (new_pid or 0, runtime_pid))
):
replacement_observed.append(True)
if active_state == "active" and new_pid and (previous_pid is None or new_pid != previous_pid):
if runtime_pid != new_pid:
runtime_state = _read_gateway_runtime_status()
if runtime_state and _runtime_state_pid(runtime_state) != new_pid:
runtime_state = None
gateway_state = (runtime_state or {}).get("gateway_state")
if gateway_state in ("running", "degraded"):
print(f"✓ {scope_label} service restarted (PID {new_pid})")
if gateway_state == "degraded":
# Serving, but a configured platform is parked or retrying: a real restart, not a
# failure — say so instead of waiting out the timeout and reporting one.
print(f"⚠ {scope_label} gateway is DEGRADED — see `hermes gateway status`")
return True
if gateway_state == "startup_failed":
reason = (runtime_state or {}).get("exit_reason") or "startup failed"
print(
f"⚠ {scope_label} service process restarted (PID {new_pid}), but gateway startup failed: {reason}"
)
return False
if not printed_runtime_wait:
print(f"⏳ {scope_label} service process started (PID {new_pid}); waiting for gateway runtime...")
printed_runtime_wait = True
if active_state == "activating" and sub_state == "auto-restart":
time.sleep(1)
continue
if _systemd_unit_is_start_limited(props):
_print_systemd_start_limit_wait(system=system)
return False
time.sleep(2)
sudo, _, user_flag = _systemd_cli_bits(system)
print(
f"⚠ {scope_label} service did not become active within {int(timeout)}s.\n"
f" Check status: {sudo}hermes gateway status\n"
f" Check logs: journalctl {user_flag}-u {svc} -l --since '2 min ago'"
)
return False
def _systemd_restart_wait_timeout(system: bool = False) -> float:
"""Cover systemd's relaunch delays before applying the runtime wait floor."""
from gateway.shutdown_forensics import parse_systemd_duration_to_us
props = _read_systemd_unit_properties(system=system, properties=("RestartUSec", "TimeoutStartUSec"))
supervisor_budget = 0.0
for name in ("RestartUSec", "TimeoutStartUSec"):
raw = props.get(name, "")
duration_us = int(raw) if raw.isdigit() else parse_systemd_duration_to_us(raw)
if duration_us is not None:
supervisor_budget += duration_us / 1_000_000
return 60.0 + supervisor_budget
def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool:
return "start-limit-hit" in (props.get("Result", "").lower(), props.get("SubState", "").lower())
def _systemd_error_indicates_start_limit(exc: subprocess.CalledProcessError) -> bool:
parts: list[str] = []
for attr in ("stderr", "stdout", "output"):
value = getattr(exc, attr, None)
if value:
parts.append(value.decode(errors="replace") if isinstance(value, bytes) else str(value))
text = "\n".join(parts).lower()
return "start-limit-hit" in text or "start request repeated too quickly" in text or "start-limit" in text
def _systemd_service_is_start_limited(system: bool = False) -> bool:
return _systemd_unit_is_start_limited(_read_systemd_unit_properties(system=system))
def _print_systemd_start_limit_wait(system: bool = False) -> None:
svc = get_service_name()
scope_label = _service_scope_label(system).capitalize()
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
print(f"⏳ {scope_label} service is temporarily rate-limited by systemd.")
print(" systemd is refusing another immediate start after repeated exits.")
print(f" Wait for the start-limit window to expire, then run: {sudo}hermes gateway restart{scope_flag}")
print(f" Or clear the failed state manually: systemctl {user_flag}reset-failed {svc}")
print(f" Check logs: journalctl {user_flag}-u {svc} -l --since '5 min ago'")
def _recover_pending_systemd_restart(system: bool = False, previous_pid: int | None = None) -> bool:
"""Recover a planned service restart that is stuck in systemd state."""
props = _read_systemd_unit_properties(system=system)
if not props:
return False
try:
from gateway.status import read_runtime_status
except Exception:
return False
if not (read_runtime_status() or {}).get("restart_requested"):
return False
active_state = props.get("ActiveState", "")
if active_state == "activating" and props.get("SubState", "") == "auto-restart":
print("⏳ Service restart already pending — waiting for systemd relaunch...")
return _wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
if active_state == "failed" and (
props.get("ExecMainStatus", "") == str(GATEWAY_SERVICE_RESTART_EXIT_CODE)
or props.get("Result", "") == "exit-code"
):
svc = get_service_name()
print(f"↻ Clearing failed state for pending {_service_scope_label(system)} service restart...")
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
_run_systemctl(["start", svc], system=system, check=False, timeout=90)
return _wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
return False
def _parse_launchd_pid_from_list_output(output: str) -> int | None:
"""PID from ``launchctl list <label>`` (``"PID" = <n>;``); None if absent (registered, not running)
or non-positive (crashed)."""
for line in output.splitlines():
stripped = line.strip()
if stripped.startswith(('"PID"', "PID")) and "=" in stripped:
return _positive_pid(stripped.split("=", 1)[1].strip().rstrip(";").strip('"'))
return None
def _parse_launchd_pid_from_print_output(output: str) -> int | None:
"""Live PID from ``launchctl print`` (first ``pid = <N>`` line wins); None if absent or non-positive."""
for line in output.splitlines():
stripped = line.strip()
if stripped.startswith("pid = "):
return _positive_pid(stripped[len("pid = "):].strip())
return None
def _launchd_print_service_pid(domain: str, label: str) -> tuple[bool, int | None]:
"""``(loaded, pid)`` for ``domain/label`` via ``launchctl print`` (domain-explicit; ``launchctl list``
infers it from caller context). ``TimeoutExpired`` propagates: a wedged launchctl is not "unloaded".
Domain-explicit on purpose: legacy ``launchctl list`` infers its domain from the caller's execution
context, which is exactly the ambiguity that sank the first fleet-restart attempt (#41403 review).
``TimeoutExpired`` propagates — fleet-restart callers own per-label failure accounting (a wedged
launchctl call must be reported, not read as "unloaded").
"""
try:
result = subprocess.run(["launchctl", "print", f"{domain}/{label}"], timeout=5, **_CAPTURE_TEXT)
except FileNotFoundError:
return (False, None)
if result.returncode != 0:
return (False, None)
return (True, _parse_launchd_pid_from_print_output(result.stdout))
def _launchd_service_registered(label: str, *, timeout: int = 5) -> bool:
"""True when launchd knows ``label`` (``launchctl list`` exit 0). Domain-agnostic, so still true on
macOS 26+ hosts whose per-user domains reject management. FileNotFoundError/TimeoutExpired propagate."""
result = subprocess.run(["launchctl", "list", label], timeout=timeout, **_CAPTURE_TEXT)
return result.returncode == 0
def _locate_launchd_gateway_service(label: str) -> tuple[str | None, int | None]:
"""``(domain, pid)`` for ``label``, probing ``gui/<uid>`` then ``user/<uid>``. Never uses the current
profile's cached ``_launchd_domain()`` — a fleet can mix domains. ``TimeoutExpired`` propagates."""
uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
for domain in (f"gui/{uid}", f"user/{uid}"):
loaded, pid = _launchd_print_service_pid(domain, label)
if loaded:
return (domain, pid)
return (None, None)
def _probe_launchd_service_running() -> bool:
"""True when the plist exists AND launchd is running a process for the current label."""
return get_launchd_plist_path().exists() and _launchctl_label_supervising_process(get_launchd_label())
def _s6_gateway_snapshot(gateway_pids: tuple[int, ...]) -> GatewayRuntimeSnapshot | None:
"""Snapshot for an s6-supervised container gateway, or None when s6 isn't the service manager."""
from hermes_cli.service_manager import detect_service_manager, get_service_manager
if detect_service_manager() != "s6":
return None
service_name = f"gateway-{_current_profile_name()}"
mgr = get_service_manager()
service_installed = service_running = False
try:
service_dir = getattr(mgr, "scandir", None)
if service_dir is not None:
service_installed = (service_dir / service_name).is_dir()
except Exception:
service_installed = False
if service_installed:
try:
service_running = bool(mgr.is_running(service_name))
except Exception:
service_running = False
return GatewayRuntimeSnapshot(
manager="s6 (container supervisor)",
service_installed=service_installed,
service_running=service_running,
gateway_pids=gateway_pids,
service_scope="s6",
)
def get_gateway_runtime_snapshot(system: bool = False) -> GatewayRuntimeSnapshot:
"""Return a unified view of gateway liveness for the current profile."""
gateway_pids = tuple(find_gateway_pids())
if is_termux():
return GatewayRuntimeSnapshot(manager="Termux / manual process", gateway_pids=gateway_pids)
from hermes_constants import is_container
if is_linux() and is_container():
# Report s6 supervision under our /init; other container runtimes keep "docker (foreground)".
try:
snapshot = _s6_gateway_snapshot(gateway_pids)
if snapshot is not None:
return snapshot
except Exception:
pass # Fall through to the legacy label on any detection error.
return GatewayRuntimeSnapshot(manager="docker (foreground)", gateway_pids=gateway_pids)
if supports_systemd_services():
selected_system, service_running = _probe_systemd_service_running(system=system)
scope_label = _service_scope_label(selected_system)
return GatewayRuntimeSnapshot(
manager=f"systemd ({scope_label})",
service_installed=get_systemd_unit_path(system=selected_system).exists(),
service_running=service_running,
gateway_pids=gateway_pids,
service_scope=scope_label,
)
if is_macos():
return GatewayRuntimeSnapshot(
manager="launchd",
service_installed=get_launchd_plist_path().exists(),
service_running=_probe_launchd_service_running(),
gateway_pids=gateway_pids,
service_scope="launchd",
)
return GatewayRuntimeSnapshot(manager="manual process", gateway_pids=gateway_pids)
def _format_gateway_pids(pids: tuple[int, ...] | list[int], *, limit: int | None = 3) -> str:
rendered = [str(pid) for pid in (pids if limit is None else pids[:limit]) if pid > 0]
if limit is not None and len(pids) > limit:
rendered.append("...")
return ", ".join(rendered)
def _print_gateway_process_mismatch(snapshot: GatewayRuntimeSnapshot) -> None:
if not snapshot.has_process_service_mismatch:
return
print()
pids_line = f" PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}"
# Managed detached fallback (launchd exit-5 path) vs. a genuinely manual run.
if _launchd_unsupported_marker_exists():
print("⚠ Gateway is running as a detached fallback process — launchd cannot supervise it")
print(pids_line)
print(" Auto-start at login and auto-restart on crash are NOT available.")
print(" Stop it with: hermes gateway stop")
else:
print("⚠ Gateway process is running for this profile, but the service is not active")
print(pids_line)
print(" This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
print(" can refuse to start another copy until this process stops.")
def _print_multiplex_standalone_reason() -> None:
"""The boot guard kept an unset-default gateway standalone: say so in status, with the remedy."""
from hermes_cli.gateway_multiplex_mode import recorded_standalone_warning_lines
for line in recorded_standalone_warning_lines():
print(line)
def _print_served_ingress_urls(profile: str | None = None) -> None:
"""Callback URLs of inbound-port platforms the live multiplexer serves for secondary profiles
(the value to paste into the Twilio / LINE / Teams / BlueBubbles console)."""
try:
from hermes_cli.gateway_multiplex_served import format_ingress_url_lines, served_profile_ingress_urls
urls = served_profile_ingress_urls(profile)
except Exception:
return
if not urls:
return
print()
print("Inbound callback URLs on the shared listener:")
for name, per_platform in sorted(urls.items()):
for line in format_ingress_url_lines(per_platform, indent=f" {name}/" if not profile else " "):
print(line)
def _print_unserved_shared_ingress(profile: str | None) -> None:
"""Shared-ingress platforms (WhatsApp/Relay) this served profile enabled that the multiplexer runs
only on the default profile — the ``whatsapp: not served under multiplex`` line."""
try:
from hermes_cli.gateway_multiplex_served import served_profile_unserved_platforms
unserved = served_profile_unserved_platforms(profile or "")
except Exception:
return
if not unserved:
return
print()
for platform, reason in sorted(unserved.items()):
print(f" ⚠ {platform}: {reason}")
print(" Enable it on the default profile (shared ingress serves every profile), or disable it here.")
def _print_other_profiles_gateway_status() -> None:
"""Print other profiles' running gateways at the bottom of ``hermes gateway status``."""
try:
from hermes_cli.profiles import get_active_profile_name
current = get_active_profile_name()
other_processes = [p for p in find_profile_gateway_processes() if p.profile != current]
if not other_processes:
return
print()
print("Other profiles:")
for proc in other_processes:
print(f" ✓ {proc.profile:<16s} — PID {proc.pid}")
except Exception:
pass
def _print_duplicate_credential_warnings() -> None:
"""The migrate preflight's duplicate-credential findings, so ``gateway status`` explains a parked
or racing bot (and why the fleet will not fold) with the same words as ``migrate --dry-run``."""
with contextlib.suppress(Exception):
from hermes_cli.gateway_migrate import duplicate_credential_findings
lines = duplicate_credential_findings()
if lines:
print()
for line in lines:
print(f"⚠ {line}")
def _gateway_list() -> None:
"""List every profile and whether its gateway is running."""
try:
from hermes_cli.profiles import list_profiles, get_active_profile_name
except Exception:
print("Unable to list profiles.")
return
profiles = list_profiles()
if not profiles:
print("No profiles found.")
return
current = get_active_profile_name()
print("Gateways:")
for prof in profiles:
marker = "✓" if prof.gateway_running else "✗"
label = prof.name + (" (current)" if prof.name == current else "")
parts = [f" {marker} {label:<24s}"]
if prof.gateway_running:
pid = None
try:
from gateway.status import get_running_pid
pid = get_running_pid(prof.path / "gateway.pid", cleanup_stale=False)
except Exception:
pass
if pid:
parts.append(f"PID {pid}")
elif named_profile_served_by_running_multiplexer(prof.name):
parts.append("served by the default multiplexer")
else:
parts.append("not running")
print(" — ".join(parts))
def kill_gateway_processes(force: bool = False, exclude_pids: set | None = None, all_profiles: bool = False) -> int:
"""Kill running gateway processes (force-kill if ``force``); ``exclude_pids`` skips e.g. just-
restarted service PIDs. Returns count killed."""
killed = 0
for pid in find_gateway_pids(exclude_pids=exclude_pids, all_profiles=all_profiles):
try:
expected_start_time = None
if force:
# Re-verify the LIVE cmdline at kill time: a PID recycled since the scan must never be tree-killed.
# Re-verify at kill time, not just scan time: the cmdline match inside find_gateway_pids()
# is stale by the time we get here, and a recycled PID could otherwise be tree-killed
# (#89614 class). _capture_gateway_argv re-reads the LIVE cmdline and returns None for
# anything that no longer looks like a gateway — refuse those.
if _capture_gateway_argv(pid) is None:
continue
from gateway.status import get_process_start_time
expected_start_time = get_process_start_time(pid)
terminate_pid(pid, force=force, expected_start_time=expected_start_time)
killed += 1
except ProcessLookupError:
pass
except PermissionError:
print(f"⚠ Permission denied to kill PID {pid}")
except OSError as exc:
print(f"Failed to kill PID {pid}: {exc}")
return killed
_REAPER_SUPERVISOR_WALK_LIMIT = 12
def _reaper_candidate_is_supervisor_owned(pid: int) -> bool:
"""True when ``pid``'s parent chain reaches ``services.exe`` (Task Scheduler-owned gateway). Windows-only
reaper backstop: ``_get_service_pids()`` is empty there, so a Scheduled-Task gateway with a stale
pidfile would look like an orphan. Fail-open once the Task's bootstrap parent exits. Not applied
on POSIX, where everything descends from PID 1 and would look supervised.
See #83683, #86098.
This check is deliberately NOT applied on POSIX: there, every process has PID 1 (launchd / init /
systemd) in its ancestry — and a genuine orphan is *reparented directly to PID 1* — so supervisor-name
ancestry carries zero signal and would spare every orphan the reaper exists to kill (#51325, 75936).
POSIX supervised gateways are already covered pidfile- independently by the ``_get_service_pids()``
exclusion.
"""
if not is_windows():
return False
try:
import psutil # type: ignore
parent = psutil.Process(pid).parent()
for _ in range(_REAPER_SUPERVISOR_WALK_LIMIT):
if parent is None:
break
with contextlib.suppress(Exception):
if (parent.name() or "").lower() == "services.exe":
return True
parent = parent.parent()
except Exception:
pass
return False
def _reap_unsupervised_gateway_orphans(
extra_exclude: set | None = None, *, min_age_s: float = 0.0,
) -> bool:
"""Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts
the restart fallback runs the gateway in-process under a ``gateway restart`` argv; a stale pidfile
then lets a live orphan keep the webhook port while a restart stacks a duplicate. No-op where a
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed.
``min_age_s`` spares a candidate younger than the grace: a gateway claims
``gateway.pid``/``gateway.lock`` only after imports + runner setup, so a process
that a previous Desktop generation (or a concurrent ``gateway start``) just launched
is scan-visible but not yet record-visible, and the argv sweep cannot tell it from
a corpse. Reaping it writes a planned-stop marker it consumes seconds later — a clean
exit 0 with no supervisor to revive it (#122533). The Desktop boot sweep passes a
grace; stop/restart keep reaping at once.
"""
try:
supervised_host = supports_systemd_services()
except Exception:
supervised_host = True
if supervised_host:
return False
# Task Scheduler is a supervisor too; its state beats a parent-chain walk (broken once the bootstrap exits).
# A Scheduled Task gateway whose conhost/VBS bootstrap has already exited is invisible to
# `_reaper_candidate_is_supervisor_owned` (the parent chain breaks before services.exe, fail-open), yet
# it is alive and supervised. After that launcher exits the task is typically Ready, not Running —
# treating only Running as supervised still kills the detached gateway on every desktop serve start
# (#86098, #87001).
if is_windows():
try:
from hermes_cli.gateway_windows import get_task_name # profile-aware task name
_task_name = get_task_name()
except Exception:
_task_name = "Hermes_Gateway"
if _windows_scheduled_task_supervises(_task_name):
return False
from gateway.status import _pid_exists, get_process_start_time, write_planned_stop_marker
own = _reaper_exclusion_pids(extra_exclude)
try:
# On Windows also drop Task Scheduler-owned candidates (the pidfile-less gap).
orphans = [
p for p in find_gateway_pids(exclude_pids=own) if p and p > 0 and not _reaper_candidate_is_supervisor_owned(p)
]
except Exception:
return False
if min_age_s > 0:
from hermes_cli.dashboard_procs import _process_age_seconds
def _old_enough(pid: int) -> bool:
try: # an undeterminable age must never widen the reap
return _process_age_seconds(pid) >= min_age_s
except Exception:
return False
orphans = [p for p in orphans if _old_enough(p)]
if not orphans:
return False
# Pin each orphan's start time now: the delayed SIGKILL must never hit a recycled PID.
# Pin each orphan's identity NOW: the cmdline scan above matched at scan-time only, and the SIGKILL
# escalation below fires seconds later. A PID recycled inside that window must never be force-killed
# (#89614 class). Fingerprint capture is best-effort — SIGTERM below proceeds regardless (it targets the
# process verified by the scan an instant ago), but the delayed SIGKILL requires a still-matching
# fingerprint.
orphan_identity: dict[int, int] = {}
for pid in orphans:
start = get_process_start_time(pid)
if start is not None:
orphan_identity[pid] = start
reaped = False
for pid in orphans:
with contextlib.suppress(Exception):
write_planned_stop_marker(pid)
# ``os.kill(..., SIGTERM)`` maps to TerminateProcess on Windows, so it
# would kill the gateway before its marker watcher can drain and close
# state cleanly. Let the bounded survivor wait below escalate instead.
if is_windows():
reaped = True
continue
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
continue
except PermissionError:
print(f"⚠ Permission denied to kill orphaned gateway PID {pid}")
continue
reaped = True
# Wait, then force-kill survivors so the replacement can bind the port cleanly.
# Fail-closed: SIGKILL only a PID that still names the process fingerprinted at scan time.
_force_kill_survivors([
pid for pid in _await_gateway_exit(orphans, pid_exists=_pid_exists)
if pid in orphan_identity and get_process_start_time(pid) == orphan_identity[pid]
])
return reaped
def _reaper_exclusion_pids(extra_exclude: set | None) -> set[int]:
"""PIDs the orphan reaper must never kill: self, caller extras, service-managed, recorded."""
own = {os.getpid()} | (extra_exclude or set())
# Service-managed gateways are never orphans (on macOS supports_systemd_services() is False, so a
# launchd gateway would otherwise be SIGTERM'd); all_profiles because the scan sees siblings too.
with contextlib.suppress(Exception):
# This covers macOS launchd (supports_systemd_services() is False there, so without this the launchd
# gateway looks like an unsupervised orphan and gets SIGTERM'd, causing launchd to restart it — or
# leaving it down under KeepAlive.SuccessfulExit=false) and any systemd unit reachable from a host
# that got past the gate above (#83683, #85344).
# all_profiles=True: the reaper's process scan sees every profile's gateway (and on macOS the
# now-working ps fallback surfaces sibling launchd gateways, #73626), so the service exclusion must
# cover the whole ai.hermes.gateway* fleet — not just the current profile's label — or a sibling
# profile's launchd gateway is misclassified as an unsupervised orphan and reaped. Same class as the
# update-sweep fix in #74075.
own |= _get_service_pids(all_profiles=True)
# Exempt the recorded gateway PID and its parent chain (on Windows the Scheduled-Task bootstrap's
# ``gateway run`` argv matches the scan; killing it takes the gateway down). Use the RAW pidfile +
# lock records, not only the validated probe: get_running_pid returns None on any validation
# hiccup — exactly when a healthy standalone gateway would be hard-killed (Windows SIGTERM is
# TerminateProcess, no drain). For a KILL exclusion list a stale PID at worst spares one process;
# a false negative kills a live gateway. The probe still supplies the runtime-status fallback PID.
try:
from gateway.status import _pid_from_record, _read_gateway_lock_record, _read_pid_record, get_running_pid
recorded_pids = {_pid_from_record(rec) for rec in (_read_pid_record(), _read_gateway_lock_record())}
recorded_pids.add(get_running_pid(cleanup_stale=False))
for recorded in recorded_pids:
if not recorded or recorded <= 0:
continue
own.add(recorded)
try:
import psutil # type: ignore
parent = psutil.Process(recorded).parent()
while parent is not None:
own.add(parent.pid)
parent = parent.parent()
except Exception:
pass
except Exception:
pass
return own
# A retiring gateway runs a PASSIVE WAL checkpoint in ``SessionDB.close()``; a SIGKILL mid-checkpoint
# corrupts ``state.db``. It keeps serving while we wait, so a long grace only delays the port bind.
_ORPHAN_EXIT_GRACE_SECONDS = 30.0
_ORPHAN_EXIT_POLL_SECONDS = 0.2
def _await_gateway_exit(
pids, *, pid_exists, sleep=None, grace_s: float = _ORPHAN_EXIT_GRACE_SECONDS, poll_s: float = _ORPHAN_EXIT_POLL_SECONDS
):
"""Poll up to *grace_s* for *pids* to exit; return survivors. ``pid_exists``/``sleep`` injectable for tests."""
if sleep is None:
sleep = time.sleep
survivors = list(pids)
for _ in range(max(1, int(grace_s / poll_s))):
survivors = [p for p in survivors if pid_exists(p)]
if not survivors:
break
sleep(poll_s)
else:
# Re-check after the LAST sleep, or a recycled PID could get the SIGKILL.
survivors = [p for p in survivors if pid_exists(p)]
return survivors
def _force_kill_survivors(survivors, *, kill=None) -> None:
"""SIGKILL processes that outlasted the grace period, loudly — a force-kill can tear the store, so
it must leave a trace."""
kill = kill or os.kill
for pid in survivors:
logger.warning(
"Gateway PID %s did not exit within %.0fs of the stop request (SIGTERM, or the planned-stop "
"marker on Windows) — sending "
"SIGKILL. A kill during a WAL checkpoint can corrupt state.db; "
"the next start will run an integrity check.",
pid, _ORPHAN_EXIT_GRACE_SECONDS,
)
with contextlib.suppress((ProcessLookupError, PermissionError, OSError)):
kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
def _mark_planned_stop(pid: int | None = None) -> None:
"""Best-effort planned-stop marker for ``pid`` (default: the recorded gateway PID)."""
try:
from gateway.status import get_running_pid, write_planned_stop_marker
if pid is None:
pid = get_running_pid(cleanup_stale=False)
if pid is not None:
write_planned_stop_marker(pid)
except Exception:
pass
def stop_profile_gateway() -> bool:
"""Stop only this profile's gateway via its PID file; True if a process was stopped. Without a
supervisor the pidfile can be stale while a live orphan holds the webhook port, so fall back to
the orphan-aware scan rather than stacking a duplicate.
Even when the pid file is valid and points to the current gateway, older orphans may linger from prior
restarts that overwrote the pid file before the old process exited. After killing the recorded PID, also
sweep for any remaining orphans so each restart produces at most one live gateway (#75936).
"""
try:
from gateway.status import get_running_pid, remove_pid_file
except ImportError:
return False
pid = get_running_pid()
if pid is None:
return _reap_unsupervised_gateway_orphans()
if is_windows():
# Windows maps SIGTERM to TerminateProcess. The marker watcher is the
# gateway's graceful-stop IPC, so wait for it before force-killing a
# wedged process.
from gateway.status import get_process_start_time
from hermes_cli.gateway_windows import (
_drain_gateway_pid,
_force_terminate_known_gateway_pids,
_windows_stop_drain_timeout,
)
# Capture identity BEFORE the drain (as _escalate_wedged_gateway does): if the PID is
# recycled during the wait, terminate_pid's start-time mismatch refuses the taskkill.
expected_start_time = get_process_start_time(pid)
if not _drain_gateway_pid(pid, _windows_stop_drain_timeout()):
_force_terminate_known_gateway_pids({pid: expected_start_time})
else:
_mark_planned_stop(pid)
try:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
pass # Already gone
except PermissionError:
print(f"⚠ Permission denied to kill PID {pid}")
return False
# ``_pid_exists``, NOT ``os.kill(pid, 0)`` (TerminateProcess on Windows).
from gateway.status import _pid_exists
for _ in range(20):
if not _pid_exists(pid):
break
time.sleep(0.5)
if get_running_pid() is None:
remove_pid_file()
# Reap orphans from prior restarts whose pidfile entry was overwritten; skip the PID just killed.
try:
# Exclude the PID we just killed so the sweep doesn't double-kill a process that's still tearing
# down — _reap_unsupervised_gateway_orphans already excludes our own PID. See #75936.
_reap_unsupervised_gateway_orphans(extra_exclude={pid} if pid else None)
except Exception as exc:
logger.debug("orphan reap after stop_profile_gateway failed: %s", exc)
return True
def is_linux() -> bool:
return sys.platform.startswith("linux")
from hermes_constants import is_container, is_termux, is_wsl
def _wsl_systemd_operational() -> bool:
"""WSL2 with ``systemd=true`` in wsl.conf has working systemd; WSL1/without it does not."""
return _systemd_operational(system=True)
def _systemd_operational(system: bool = False) -> bool:
"""Return True when the requested systemd scope is usable."""
try:
result = _run_systemctl(["is-system-running"], system=system, timeout=5, **_CAPTURE_TEXT)
except (RuntimeError, subprocess.TimeoutExpired, OSError):
return False
# "running", "degraded", "starting" all mean systemd is PID 1
return result.stdout.strip().lower() in {"running", "degraded", "starting", "initializing"}
def supports_systemd_services() -> bool:
if not is_linux() or is_termux() or shutil.which("systemctl") is None:
return False
if is_wsl():
return _wsl_systemd_operational()
if is_container():
# A container whose init is systemd (nspawn, some k8s pods) behaves like a host.
return _systemd_operational(system=False) or _systemd_operational(system=True)
return True
def is_macos() -> bool:
return sys.platform == "darwin"
def is_windows() -> bool:
return sys.platform == "win32"
def _gw_windows():
"""Lazily import :mod:`hermes_cli.gateway_windows` (Windows-only service backend)."""
from hermes_cli import gateway_windows
return gateway_windows
# Task Scheduler states meaning "still supervised" (Ready = steady state after the launcher exits).
# Task Scheduler states that mean "this profile still has an official supervisor". Queued is a rare
# in-between. Disabled / MISSING are not supervisors. See #87001.
_WINDOWS_TASK_SUPERVISOR_STATES = frozenset({"Running", "Ready", "Queued"})
def _windows_scheduled_task_state(task_name: str) -> str | None:
"""Locale-independent Task Scheduler state, or None on failure.
Query the COM API directly: Get-ScheduledTask auto-loads the CIM module,
which can stall desktop backend startup for the entire ten-second timeout.
Keep the existing supervisor semantics (Ready and Queued count as owned).
"""
if not is_windows():
return None
quoted_name = task_name.replace("'", "''")
ps_cmd = (
"$ErrorActionPreference = 'Stop'; "
"$s = [Activator]::CreateInstance([type]::GetTypeFromProgID('Schedule.Service')); "
"$s.Connect(); "
"try { "
f"$t = $s.GetFolder('\\').GetTask('{quoted_name}'); "
# TASK_STATE values are stable, unlike localized schtasks.exe output.
"@('Unknown', 'Disabled', 'Queued', 'Ready', 'Running')[[int]$t.State] "
"} catch { "
"$e = $_.Exception; while ($e.InnerException) { $e = $e.InnerException }; "
"if ($e.HResult -in @(-2147024894, -2147024893)) { 'MISSING' } else { throw } "
"}"
)
try:
powershell = shutil.which("powershell") or shutil.which("pwsh")
if powershell is None:
return None
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", ps_cmd],
capture_output=True, text=True, encoding="utf-8", errors="ignore", timeout=10,
)
if result.returncode != 0:
return None
return (result.stdout or "").strip() or None
except (OSError, subprocess.TimeoutExpired):
return None
def _windows_scheduled_task_supervises(task_name: str) -> bool:
"""True when Task Scheduler still owns this profile's gateway (Ready counts: the task is Ready, not
Running, after bootstrap exits). Any failure returns False so callers fall back to pidfile / parent-chain.
Used to treat Task Scheduler as a gateway supervisor on Windows: the orphan-reap sweep must not kill a
gateway that a scheduled task launched and left detached. After the bootstrap exits the task is Ready,
not Running; a Running-only check still writes the planned-stop marker, the gateway exits cleanly with
code 0, and the scheduler never restarts it — silently killing A2A/messaging on every desktop-app launch
(#86098, #87001).
"""
return _windows_scheduled_task_state(task_name) in _WINDOWS_TASK_SUPERVISOR_STATES
def _gateway_detached_env() -> bool:
return _truthy_env(os.getenv("HERMES_GATEWAY_DETACHED"))
def _stdin_is_tty() -> bool | None:
"""``sys.stdin.isatty()``; None when stdin is closed/invalid."""
try:
return bool(sys.stdin and sys.stdin.isatty())
except (ValueError, OSError):
return None
def _windows_gateway_should_absorb_console_controls() -> bool:
"""True for detached Windows gateway runs that should ignore Ctrl+C (``HERMES_GATEWAY_DETACHED=1``
or no interactive stdin); foreground runs stay interruptible."""
if not is_windows():
return False
if _gateway_detached_env():
return True
return not _stdin_is_tty()
def _windows_console_window_attached() -> bool | None:
"""Return whether Windows assigned this process a console window."""
if not is_windows():
return None
try:
import ctypes
return bool(ctypes.windll.kernel32.GetConsoleWindow()) # type: ignore[attr-defined]
except (OSError, AttributeError):
return None
def _windows_gateway_breakaway_state() -> bool | None:
"""Consume private spawn metadata without guessing for older launchers."""
if not is_windows():
return None
from hermes_cli._subprocess_compat import _WINDOWS_GATEWAY_BREAKAWAY_ENV
return {"1": True, "0": False}.get(os.environ.pop(_WINDOWS_GATEWAY_BREAKAWAY_ENV, None))
# =============================================================================
# Service Configuration
# =============================================================================
_SERVICE_BASE = "hermes-gateway"
SERVICE_DESCRIPTION = "Hermes Agent Gateway - Messaging Platform Integration"
_SYSTEM_UNIT_DIR = Path("/etc/systemd/system")
def _profile_name_from_home(home: Path, default: Path) -> str | None:
"""Profile name when ``home`` is ``<default>/profiles/<name>`` with a service-safe name, else None."""
import re
try:
parts = home.relative_to((default / "profiles").resolve()).parts
except ValueError:
return None
if len(parts) == 1 and re.match(r"^[a-z0-9][a-z0-9_-]{0,63}$", parts[0]):
return parts[0]
return None
def _native_service_homes() -> set[Path]:
"""This process's native default home plus, when root under sudo, the invoking user's (see
``_profile_suffix`` for why sudo matters)."""
from hermes_constants import _get_platform_default_hermes_home, sudo_invoker_default_home
homes = {_get_platform_default_hermes_home().resolve()}
sudo_home = sudo_invoker_default_home()
if sudo_home is not None:
homes.add(sudo_home.resolve())
return homes
def _bare_unit_pinned_home() -> Path | None:
"""Resolved ``HERMES_HOME`` pinned by an installed ``hermes-gateway.service``, or None. The unit is the
one naming basis that holds still across the sudo mid-command switch (see ``_profile_suffix``) and it
covers every elevated identity — ``sudo -i`` and cron included, where SUDO_USER is absent.
Linux- and root-gated: a systemd unit is not an identity authority for launchd labels, Windows
scheduled tasks, or s6 slots, which share ``_profile_suffix()``, and only an elevated process ever
operates the system unit — an unprivileged user-scope command must keep naming its own units, or a
bare system unit pinning ``profiles/<name>`` would alias that profile onto the user's default unit.
``is_linux()`` is a plain ``sys.platform`` test; ``supports_systemd_services()`` would be wrong here,
since it can shell out to ``systemctl is-system-running`` on WSL/containers and this runs on every
name resolution.
"""
if not is_linux() or os.geteuid() != 0: # windows-footgun: ok — behind is_linux()
return None
pinned = _hermes_home_pinned_by_unit(_SYSTEM_UNIT_DIR / f"{_SERVICE_BASE}.service")
if not pinned:
return None
try:
return Path(pinned).expanduser().resolve()
except (RuntimeError, ValueError): # hand-edited unit: ``~nouser`` or an embedded NUL
return None
def _profile_suffix() -> str:
"""Service-name suffix for HERMES_HOME: "" for a home that owns the bare name, the profile name for
``<root>/profiles/<name>``, else a short hash of the path.
Bare-name owners: this process's platform-native default (``~/.hermes``), under sudo the invoking
user's native default, and the home pinned by an installed ``hermes-gateway.service``. Under sudo the
naming basis moves MID-COMMAND — sudo strips HERMES_HOME and sets HOME=/root, then
``_sync_hermes_home_from_systemd_unit()`` adopts the unit's own HERMES_HOME into ``os.environ`` — so a
basis derived from the process alone names one unit before the adoption and another after it. The
unit-pinned check must precede the profile branch: ``sudo hermes gateway install --system`` resolves
the BARE name from root's default, then pins the invoking user's remapped home, so the bare unit
legitimately carries a ``<root>/profiles/<name>`` home.
The bare name is deliberately NOT tied to ``get_default_hermes_root()``: that helper treats any
HERMES_HOME outside ``~/.hermes`` (Docker ``/opt/data``, a temp dir) as "the root itself", which let a
temp-home harness resolve to the default profile's ``hermes-gateway`` unit and uninstall the
production gateway. Service names are host-wide identities; a home with no installed bare unit and
no native default keeps its own suffix.
"""
import hashlib
from hermes_constants import get_default_hermes_root
home = get_hermes_home().resolve()
if home in _native_service_homes() or home == _bare_unit_pinned_home():
return ""
name = _profile_name_from_home(home, get_default_hermes_root().resolve())
return name or hashlib.sha256(str(home).encode()).hexdigest()[:8]
def _current_profile_name() -> str:
"""Profile id relative to the profile ROOT: ``default`` for the root itself (Docker's ``/opt/data``
included), ``<name>`` for ``<root>/profiles/<name>``, else the service hash. s6 slots and the
multiplexer ask which PROFILE this is; ``_profile_suffix()`` answers which HOST SERVICE this is."""
from hermes_constants import profile_name_for_home
return profile_name_for_home(get_hermes_home()) or _profile_suffix()
def _profile_arg(hermes_home: str | None = None, default_root: str | Path | None = None) -> str:
"""``--profile <name>`` for ``<root>/profiles/<name>``, else "". *hermes_home*/*default_root* let a
sudo/root process generate a unit for another user (the defaults would refer to root)."""
from hermes_constants import get_default_hermes_root
home = Path(hermes_home or str(get_hermes_home())).resolve()
default = Path(default_root).resolve() if default_root else get_default_hermes_root().resolve()
if home == default:
return ""
name = _profile_name_from_home(home, default)
return f"--profile {name}" if name else ""
def get_service_name() -> str:
"""Systemd service name: ``hermes-gateway`` for default HERMES_HOME, ``hermes-gateway-<profile>``
or ``-<hash>`` otherwise."""
suffix = _profile_suffix()
return f"{_SERVICE_BASE}-{suffix}" if suffix else _SERVICE_BASE
def user_systemd_unit_dir() -> Path:
"""``$XDG_CONFIG_HOME/systemd/user`` (``~/.config`` only as the spec's default).
Hardcoding ``~/.config`` made every unit probe silently false-negative on a host that moves
XDG_CONFIG_HOME — doctor then reported no gateway unit while systemd was running ours.
"""
config_home = os.environ.get("XDG_CONFIG_HOME", "").strip()
base = Path(config_home) if config_home else Path.home() / ".config"
return base / "systemd" / "user"
def get_systemd_unit_path(system: bool = False) -> Path:
name = get_service_name()
if system:
return _SYSTEM_UNIT_DIR / f"{name}.service"
return user_systemd_unit_dir() / f"{name}.service"
class UserSystemdUnavailableError(RuntimeError):
"""``systemctl --user`` cannot reach the user D-Bus session (fresh SSH sessions with linger off,
so ``/run/user/$UID/bus`` never exists). ``args[0]`` is a user-facing remediation message."""
class SystemScopeRequiresRootError(RuntimeError):
"""System-scope gateway operation attempted as non-root. Typed (not ``sys.exit(1)``) so the setup
wizard can print remediation; ``args`` = (message, action) and ``str(e)`` is the message only."""
def __str__(self) -> str:
return self.args[0] if self.args else ""
def _user_runtime_dir() -> Path:
"""``$XDG_RUNTIME_DIR`` or ``/run/user/<uid>`` (regardless of existence)."""
return Path(os.environ.get("XDG_RUNTIME_DIR") or f"/run/user/{os.getuid()}") # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
def _user_dbus_socket_path() -> Path:
"""Return the expected per-user D-Bus socket path (regardless of existence)."""
return _user_runtime_dir() / "bus"
def _user_systemd_private_socket_path() -> Path:
"""Return the per-user systemd private socket path (regardless of existence)."""
return _user_runtime_dir() / "systemd" / "private"
def _path_exists_safe(path: Path) -> bool:
"""``Path.exists()`` treating an inaccessible path as absent: a leaked ``XDG_RUNTIME_DIR`` from
another user (``/run/user/0`` is 0700) would otherwise crash the preflight with EACCES.
``Path.exists()`` only swallows a subset of ``OSError`` (ENOENT/ENOTDIR/ EBADF/ELOOP); ``EACCES`` still
propagates. When ``XDG_RUNTIME_DIR`` leaks from another user — the classic ``su``/``sudo -u`` from a
root shell case, where ``/run/user/0`` is ``0700 root:root`` — stat-ing a socket underneath it raises
``PermissionError`` that escapes the systemd preflight as a raw traceback (#86558). An unreadable path
is, for our purposes, not reachable.
"""
try:
return path.exists()
except OSError: # e.g. EACCES on another user's runtime dir
return False
def _runtime_dir_is_ours(runtime_dir: str) -> bool:
"""True when *runtime_dir* exists and is owned by our uid (a leaked foreign XDG_RUNTIME_DIR must not be trusted)."""
try:
return Path(runtime_dir).stat().st_uid == os.getuid() # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
except OSError:
return False
def _user_systemd_socket_ready() -> bool:
"""True when the user D-Bus socket OR the per-user systemd private socket exists (some distros
expose only the latter and ``systemctl --user`` still works). Inaccessible counts as not-ready."""
return _path_exists_safe(_user_dbus_socket_path()) or _path_exists_safe(_user_systemd_private_socket_path())
def _ensure_user_systemd_env() -> None:
"""Set XDG_RUNTIME_DIR / DBUS_SESSION_BUS_ADDRESS so ``systemctl --user`` works on headless (SSH)
hosts; an XDG_RUNTIME_DIR leaked from another user is replaced with our own ``/run/user/{uid}``.
An ``XDG_RUNTIME_DIR`` that leaked from another user (``su``/``sudo -u`` from root, where the env still
points at ``/run/user/0``) is dropped in favour of our own ``/run/user/{uid}`` so ``systemctl --user``
targets the right instance instead of an unreadable foreign socket (#86558).
"""
uid = os.getuid() # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
xdg = os.environ.get("XDG_RUNTIME_DIR")
if (not xdg or not _runtime_dir_is_ours(xdg)) and _runtime_dir_is_ours(f"/run/user/{uid}"):
os.environ["XDG_RUNTIME_DIR"] = f"/run/user/{uid}"
if "DBUS_SESSION_BUS_ADDRESS" not in os.environ:
bus_path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{uid}")) / "bus"
if _path_exists_safe(bus_path):
os.environ["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={bus_path}"
def _wait_for_user_dbus_socket(timeout: float = 3.0) -> bool:
"""Poll up to ``timeout`` s for a user systemd control socket (user@.service takes a moment after enable-linger)."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if _user_systemd_socket_ready():
_ensure_user_systemd_env()
return True
time.sleep(0.2)
return _user_systemd_socket_ready()
def _wait_for_target_user_bus(uid: int, timeout: float = 5.0) -> bool:
"""Poll for ``/run/user/<uid>/bus`` of ANOTHER account (the system unit's ``User=`` while root installs).
Only the D-Bus socket counts — ``systemd/private`` alone is enough for ``systemctl --user`` but not for
the ``systemd-run --user`` that restart-safe workers need. Never adopts anything into our env."""
bus = Path(f"/run/user/{uid}/bus")
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if _path_exists_safe(bus):
return True
time.sleep(0.2)
return _path_exists_safe(bus)
def _loginctl_enable_linger(username: str) -> subprocess.CompletedProcess:
"""``loginctl enable-linger <username>`` (check=False, 30s); exceptions propagate to the caller."""
return subprocess.run(["loginctl", "enable-linger", username], check=False, timeout=30, **_CAPTURE_TEXT)
def _completed_process_detail(result) -> str:
"""stderr, else stdout, else ``exit <rc>`` — stripped."""
return (result.stderr or result.stdout or f"exit {result.returncode}").strip()
def _preflight_user_systemd(*, auto_enable_linger: bool = True) -> None:
"""Ensure ``systemctl --user`` can reach user-scope systemd; raise UserSystemdUnavailableError otherwise.
No-op when a control socket exists; else wait briefly if linger is on, or (``auto_enable_linger``)
try ``loginctl enable-linger`` (non-root works when polkit permits)."""
_ensure_user_systemd_env()
if _user_systemd_socket_ready():
return
import getpass
username = getpass.getuser()
linger_enabled, linger_detail = get_systemd_linger_status()
sudo_hint = f" sudo loginctl enable-linger {username}"
if linger_enabled is True:
if _wait_for_user_dbus_socket(timeout=3.0):
return
# Linger is on but socket still missing — unusual; fall through to error.
_raise_user_systemd_unavailable(
username,
reason="User systemd control sockets are missing even though linger is enabled.",
fix_hint=(
f" systemctl start user@{os.getuid()}.service\n" # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
" (may require sudo; try again after the command succeeds)"
),
)
if auto_enable_linger and shutil.which("loginctl"):
try:
result = _loginctl_enable_linger(username)
except Exception as exc:
_raise_user_systemd_unavailable(
username, reason=f"loginctl enable-linger failed ({exc}).", fix_hint=sudo_hint
)
else:
if result.returncode == 0:
if _wait_for_user_dbus_socket(timeout=5.0):
print(f"✓ Enabled linger for {username} — user D-Bus now available")
return
# enable-linger succeeded but the socket never appeared.
_raise_user_systemd_unavailable(
username,
reason="Linger was enabled, but the user D-Bus socket did not appear.",
fix_hint=(
" Log out and log back in, then re-run the command.\n"
f" Or reboot and run: systemctl --user start {get_service_name()}"
),
)
_raise_user_systemd_unavailable(
username,
reason=f"loginctl enable-linger was denied: {_completed_process_detail(result)}",
fix_hint=sudo_hint,
)
_raise_user_systemd_unavailable(
username,
reason=f"User D-Bus session is not available ({linger_detail or 'linger disabled'}).",
fix_hint=sudo_hint,
)
def _raise_user_systemd_unavailable(username: str, *, reason: str, fix_hint: str) -> None:
"""Build a user-facing error message and raise UserSystemdUnavailableError."""
msg = (
f"{reason}\n"
" systemctl --user cannot reach the user D-Bus session in this shell.\n"
"\n"
" To fix:\n"
f"{fix_hint}\n"
"\n"
" Alternative: run the gateway in the foreground (stays up until\n"
" you exit / close the terminal):\n"
" hermes gateway run"
)
raise UserSystemdUnavailableError(msg)
def _systemctl_cmd(system: bool = False) -> list[str]:
if not system:
_ensure_user_systemd_env()
return ["systemctl"] if system else ["systemctl", "--user"]
def _run_systemctl(args: list[str], *, system: bool = False, **kwargs) -> subprocess.CompletedProcess:
"""Run systemctl; raise RuntimeError (not raw FileNotFoundError) if missing, for callers bypassing
``supports_systemd_services()``."""
try:
return subprocess.run(_systemctl_cmd(system) + args, **kwargs)
except FileNotFoundError:
from hermes_cli.gateway_command_errors import SystemctlUnavailableError
raise SystemctlUnavailableError() from None
def _service_scope_label(system: bool = False) -> str:
return "system" if system else "user"
def get_installed_systemd_scopes() -> list[str]:
scopes: list[str] = []
seen_paths: set[Path] = set()
for system, label in ((False, "user"), (True, "system")):
unit_path = get_systemd_unit_path(system=system)
if unit_path not in seen_paths and unit_path.exists():
scopes.append(label)
seen_paths.add(unit_path)
return scopes
def has_conflicting_systemd_units() -> bool:
return len(get_installed_systemd_scopes()) > 1
# Legacy pre-rename names: explicit allowlist (NOT a glob) so profile and third-party units never match.
_LEGACY_SERVICE_NAMES: tuple[str, ...] = ("hermes.service",)
# ExecStart markers identifying a unit as running our gateway; a legacy unit is flagged only if one matches.
_LEGACY_UNIT_EXECSTART_MARKERS: tuple[str, ...] = (
"hermes_cli.main gateway",
"hermes_cli/main.py gateway",
"gateway/run.py",
" hermes gateway ",
"/hermes gateway ",
)
def _legacy_unit_search_paths() -> list[tuple[bool, Path]]:
"""``[(is_system, base_dir), ...]`` to scan for legacy units; factored out so tests can monkeypatch."""
return [(False, user_systemd_unit_dir()), (True, _SYSTEM_UNIT_DIR)]
def _find_legacy_hermes_units() -> list[tuple[str, Path, bool]]:
"""``[(unit_name, unit_path, is_system)]`` for legacy gateway units (e.g. ``hermes.service``), which
fight the current unit for the bot token (SIGTERM flap loop). Explicit name allowlist + ExecStart
marker check so profile/third-party units never match; no mutation.
Detects unit files installed by older Hermes versions that used a different service name (e.g. When both
a legacy unit and the current ``hermes-gateway.service`` are active, they fight over the same bot token
— the PR #5646 signal-recovery change turns this into a 30-second SIGTERM flap loop.
"""
results: list[tuple[str, Path, bool]] = []
for is_system, base in _legacy_unit_search_paths():
for name in _LEGACY_SERVICE_NAMES:
unit_path = base / name
try:
if not unit_path.exists():
continue
text = unit_path.read_text(encoding="utf-8-sig", errors="ignore")
except (OSError, PermissionError):
continue
if any(marker in text for marker in _LEGACY_UNIT_EXECSTART_MARKERS):
results.append((name, unit_path, is_system))
return results
def has_legacy_hermes_units() -> bool:
"""Return True when any legacy Hermes gateway unit files exist."""
return bool(_find_legacy_hermes_units())
def print_legacy_unit_warning() -> None:
"""Warn about installed legacy gateway units; prints nothing when there are none."""
legacy = _find_legacy_hermes_units()
if not legacy:
return
print_warning("Legacy Hermes gateway unit(s) detected from an older install:")
for name, path, is_system in legacy:
print_info(f" {path} ({_service_scope_label(is_system)} scope)")
print_info(" These run alongside the current hermes-gateway service and")
print_info(" cause SIGTERM flap loops — both try to use the same bot token.")
print_info(" Remove them with:")
print_info(" hermes gateway migrate-legacy")
def remove_legacy_hermes_units(interactive: bool = True, dry_run: bool = False) -> tuple[int, list[Path]]:
"""Stop, disable, and remove legacy gateway units. ``interactive=False`` skips the prompt; ``dry_run``
only lists. Returns ``(removed_count, remaining_paths)`` (remaining: e.g. system-scope when not root)."""
legacy = _find_legacy_hermes_units()
if not legacy:
print("No legacy Hermes gateway units found.")
return 0, []
print()
print("Legacy Hermes gateway unit(s) found:")
for name, path, is_system in legacy:
print(f" {path} ({_service_scope_label(is_system)} scope)")
print()
if dry_run:
print("(dry-run — nothing removed)")
return 0, [p for _, p, _ in legacy]
if interactive and not prompt_yes_no("Remove these legacy units?", True):
print("Skipped. Run again with: hermes gateway migrate-legacy")
return 0, [p for _, p, _ in legacy]
removed = 0
remaining: list[Path] = []
def _remove_units(units: list[tuple[str, Path]], *, system: bool) -> None:
nonlocal removed
for name, path in units:
try:
_run_systemctl(["stop", name], system=system, check=False, timeout=90)
_run_systemctl(["disable", name], system=system, check=False, timeout=30)
path.unlink(missing_ok=True)
print(f" ✓ Removed {path}")
removed += 1
except (OSError, RuntimeError) as e:
print(f" ⚠ Could not remove {path}: {e}")
remaining.append(path)
with contextlib.suppress(RuntimeError):
_run_systemctl(["daemon-reload"], system=system, check=False, timeout=30)
user_units = [(n, p) for n, p, is_sys in legacy if not is_sys]
system_units = [(n, p) for n, p, is_sys in legacy if is_sys]
if user_units:
_remove_units(user_units, system=False)
# System-scope removal (needs root)
if system_units:
if os.geteuid() != 0: # windows-footgun: ok — Linux systemd removal path, guarded by `if system == "Linux"` / systemd-only branch
print()
print_warning("System-scope legacy units require root to remove.")
print_info(" Re-run with: sudo hermes gateway migrate-legacy")
remaining.extend(path for _, path in system_units)
else:
_remove_units(system_units, system=True)
print()
if remaining:
print_warning(f"{len(remaining)} legacy unit(s) still present — see messages above.")
else:
print_success(f"Removed {removed} legacy unit(s).")
return removed, remaining
def print_systemd_scope_conflict_warning() -> None:
scopes = get_installed_systemd_scopes()
if len(scopes) < 2:
return
print_warning(f"Both user and system gateway services are installed ({' + '.join(scopes)}).")
print_info(" This is confusing and can make start/stop/status behavior ambiguous.")
print_info(" Default gateway commands target the user service unless you pass --system.")
print_info(" Keep one of these:")
print_info(" hermes gateway uninstall")
print_info(" sudo hermes gateway uninstall --system")
def refuses_container_user_scope_install(system: bool) -> bool:
"""True (after printing the guidance) when a fresh USER-scope unit was requested inside a container.
A systemd container passes ``supports_systemd_services()`` on purpose so ``--system`` keeps working,
but a user unit there is not container-scoped: the unit file and its ``default.target.wants`` symlink
land in ``~/.config/systemd/user`` — commonly the host's own home bind-mounted in — so the host's
``systemd --user`` enables it too and a second gateway polls the same bot token outside the container.
Callers decide between ``sys.exit(1)`` (CLI) and skipping the install (wizard)."""
if system or not is_container():
return False
print_error("Refusing to install a user-scope systemd gateway service inside a container.")
_print_info_lines(
"The unit file and its enable symlink would be written to the home directory, which is",
"commonly the host's own home bind-mounted in — the host's user manager then enables and",
"starts the same unit, so a second gateway polls the same bot token outside the container",
"(Telegram: 'Conflict: terminated by other getUpdates request').",
"",
" hermes gateway run # run as the container's main process",
" docker run --restart unless-stopped ... # container restart policy",
"",
"If systemd manages this container (systemd as PID 1), install an isolated system service instead:",
" sudo hermes gateway install --system --run-as-user <user>",
)
return True
def _require_root_for_system_service(action: str) -> None:
if os.geteuid() != 0: # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
raise SystemScopeRequiresRootError(f"System gateway {action} requires root. Re-run with sudo.", action)
def _system_service_identity(run_as_user: str | None = None) -> tuple[str, str, str, int]:
import getpass
import grp
import pwd
username = (
run_as_user or os.getenv("SUDO_USER") or os.getenv("USER") or os.getenv("LOGNAME") or getpass.getuser()
).strip()
if not username:
raise ValueError("Could not determine which user the gateway service should run as")
if username == "root" and not run_as_user:
raise ValueError(
"Refusing to install the gateway system service as root; pass --run-as-user root to override (e.g. in LXC containers)"
)
if username == "root":
print_warning("Installing gateway service to run as root.")
print_info(" This is fine for LXC/container environments but not recommended on bare-metal hosts.")
try:
user_info = pwd.getpwnam(username)
except KeyError as e:
raise ValueError(f"Unknown user: {username}") from e
return username, grp.getgrgid(user_info.pw_gid).gr_name, user_info.pw_dir, user_info.pw_uid
def _read_systemd_user_from_unit(unit_path: Path) -> str | None:
if not unit_path.exists():
return None
for line in unit_path.read_text(encoding="utf-8-sig").splitlines():
if line.startswith("User="):
return line.split("=", 1)[1].strip() or None
return None
def _default_system_service_user() -> str | None:
for candidate in (os.getenv("SUDO_USER"), os.getenv("USER"), os.getenv("LOGNAME")):
candidate = (candidate or "").strip()
if candidate and candidate != "root":
return candidate
return None
def prompt_linux_gateway_install_scope() -> str | None:
# Only root can create a boot-time system service; never hand a non-root user a "re-run under sudo" recipe.
is_root = os.geteuid() == 0 # windows-footgun: ok — Linux systemd install wizard, never invoked on Windows
options = ["User service (no sudo; best for laptops/dev boxes; may need linger after logout)"]
values: list[str | None] = ["user"]
if is_root:
options.append("System service (starts on boot; runs as your chosen user)")
values.append("system")
options.append("Skip service install for now")
values.append(None)
choice = prompt_choice(" Choose how the gateway should run in the background:", options, default=0)
if not is_root and choice == 0:
print_info(" Tip: for a boot-time system service, re-run setup as root (e.g. from a root shell or `sudo -i`).")
return values[choice]
def install_linux_gateway_from_setup(force: bool = False, enable_on_startup: bool = True) -> tuple[str | None, bool]:
scope = prompt_linux_gateway_install_scope()
if scope is None:
return None, False
if scope == "system":
run_as_user = _default_system_service_user()
if os.geteuid() != 0: # windows-footgun: ok — Linux systemd install wizard, never invoked on Windows
# Unreachable from the wizard (system scope only offered to root); defensive guard for direct callers.
print_warning(
" System service install requires root. Re-run setup from a "
"root shell, or install a user service instead: hermes gateway install"
)
return scope, False
while not run_as_user:
run_as_user = (prompt(" Run the system gateway service as which user?", default="") or "").strip()
if not run_as_user:
print_error(" Enter a username.")
systemd_install(force=force, system=True, run_as_user=run_as_user, enable_on_startup=enable_on_startup)
return scope, True
if refuses_container_user_scope_install(system=False):
return scope, False
systemd_install(force=force, system=False, enable_on_startup=enable_on_startup)
return scope, True
def ensure_gateway_service(context: str = "setup") -> bool:
"""Install and start a user-scope gateway service without prompting (``hermes setup``/``import``).
A zero-platform gateway is a supported degraded mode (cron runs), so this never gates on messaging
config. Never raises; True when a service is installed and running."""
from hermes_constants import is_container
if is_container():
# Containers use restart policies, not service managers.
print_info("Start the gateway to bring your bots online:")
print_info(" hermes gateway run # Run as container main process")
print_info("")
print_info("For automatic restarts, use a Docker restart policy:")
print_info(" docker run --restart unless-stopped ...")
return False
supports_systemd = supports_systemd_services()
if not (supports_systemd or is_macos() or is_windows()):
print_info(" No supported service manager found on this host.")
print_info(" Run the gateway in the foreground with: hermes gateway")
return False
try:
if _is_service_running():
return True
if _served_profile_needs_no_service():
return True
if not _is_service_installed():
if supports_systemd and has_conflicting_systemd_units():
# Both units would fight over bot tokens; don't pile a fresh install onto a conflicted state.
print_systemd_scope_conflict_warning()
return False
print_info(" Installing the gateway background service ...")
if supports_systemd:
systemd_install(force=False, non_interactive=True)
elif is_macos():
launchd_install(force=False)
else:
_gw_windows().install(force=False) # Registers the Scheduled Task AND starts it.
print_success(" Gateway service installed and started.")
return True
if supports_systemd:
systemd_start()
elif is_macos():
launchd_start()
else:
_gw_windows().start()
print_success(" Gateway service running (cron jobs + messaging platforms).")
return True
except UserSystemdUnavailableError as e:
print_warning(" Could not reach user systemd to start the gateway service:")
_print_indented(str(e), print_info)
except SystemScopeRequiresRootError as e:
print_warning(f" Gateway service needs root for this scope: {e}")
_print_system_scope_remediation("start")
except SystemExit:
# Some install/start paths sys.exit() on hard failures (temp-HOME guard); never abort setup/import.
print_warning(" Gateway service install did not complete.")
print_info(" You can retry manually: hermes gateway install")
except Exception as e:
print_warning(f" Gateway service install failed: {e}")
print_info(" You can retry manually: hermes gateway install")
return False
def get_systemd_linger_status(username: str | None = None) -> tuple[bool | None, str]:
"""Linger status for *username* or the current user when omitted.
System-scope gateway installation runs as root but the service runs as a
configured target user, so querying the caller would validate the wrong
user manager.
"""
if is_termux():
return None, "not supported in Termux"
if not is_linux():
return None, "not supported on this platform"
if not shutil.which("loginctl"):
return None, "loginctl not found"
if username is None:
username = os.getenv("USER") or os.getenv("LOGNAME")
if not username:
try:
import pwd
username = pwd.getpwuid(os.getuid()).pw_name # windows-footgun: ok — POSIX loginctl helper, never invoked on Windows
except Exception:
return None, "could not determine current user"
try:
result = subprocess.run(
["loginctl", "show-user", username, "--property=Linger", "--value"],
check=False, timeout=10, **_CAPTURE_TEXT,
)
except Exception as e:
return None, str(e)
if result.returncode != 0:
return None, _completed_process_detail(result) or "loginctl query failed"
value = (result.stdout or "").strip().lower()
if value in {"yes", "true", "1"}:
return True, ""
if value in {"no", "false", "0"}:
return False, ""
return None, f"unexpected loginctl output: {value or '<empty>'}"
def get_launchd_plist_path() -> Path:
"""``~/Library/LaunchAgents/ai.hermes.gateway[-<profile>].plist`` under the real account home."""
import pwd
suffix = _profile_suffix()
name = f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
# Real account home: profile mode may point HOME at a profile dir.
home = Path(pwd.getpwuid(os.getuid()).pw_dir) # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
return home / "Library" / "LaunchAgents" / f"{name}.plist"
def launchd_gateway_labels_for_install() -> list[str]:
"""Launchd labels for every profile of THIS install (root first, then profiles by name). Derived from
the profile layout, NOT by globbing ``~/Library/LaunchAgents``, so a sandboxed HERMES_HOME never
restarts another install's fleet. Names that can't map to a suffix are skipped."""
import re as _re
from hermes_cli.profiles import list_profiles
root_label: list[str] = []
profile_labels: list[str] = []
for profile in list_profiles():
if profile.is_default:
root_label.append("ai.hermes.gateway")
elif _re.match(r"^[a-z0-9][a-z0-9_-]{0,63}$", profile.name):
profile_labels.append(f"ai.hermes.gateway-{profile.name}")
return root_label + sorted(profile_labels)
def legacy_launchd_labels_for_install(exclude=()) -> list[str]:
"""Launchd labels of THIS install that the profile-layout derivation can't map (#115254).
A unit whose label predates the profile-name suffix scheme (``ai.hermes.gateway-<8hex>`` from the
historical hash suffix) is invisible to ``launchd_gateway_labels_for_install()`` and therefore to
the update restart pass. This reads the account's LaunchAgents and credits a plist only when its
pinned ``HERMES_HOME`` is this install's root or one of its ``profiles/<name>`` homes — ownership
judged from the plist's content, never from label shape or directory membership — so the
derivation's boundary holds: a sandboxed HERMES_HOME (tests, side-by-side installs) never
enumerates, let alone restarts, another install's fleet (#41403).
"""
import plistlib
import pwd
from hermes_constants import get_default_hermes_root
try:
home = Path(pwd.getpwuid(os.getuid()).pw_dir) # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
root = get_default_hermes_root().resolve()
except Exception:
return []
agents_dir = home / "Library" / "LaunchAgents"
if not agents_dir.is_dir():
return []
excluded = set(exclude)
labels: set[str] = set()
for plist_path in sorted(agents_dir.glob("ai.hermes.gateway*.plist")):
try:
data = plistlib.loads(plist_path.read_bytes())
label = data["Label"]
pinned = Path(str(data["EnvironmentVariables"]["HERMES_HOME"])).expanduser().resolve()
rel = pinned.relative_to(root).parts
except Exception:
continue # unreadable plist, no pinned home, or a home outside this root: not ours — fail closed
if not isinstance(label, str) or label in excluded or not label.startswith("ai.hermes.gateway"):
continue
if not rel or (len(rel) == 2 and rel[0] == "profiles"):
labels.add(label)
return sorted(labels)
def get_python_path() -> str:
from hermes_cli._launchers import resolve_store_python
return str(resolve_store_python(PROJECT_ROOT) or sys.executable)
# =============================================================================
# Systemd (Linux)
# =============================================================================
def _build_user_local_paths(home: Path, path_entries: list[str]) -> list[str]:
"""Return user-local bin dirs that exist and aren't already in *path_entries*."""
candidates = [
str(home / ".local" / "bin"), # uv, uvx, pip-installed CLIs
str(home / ".cargo" / "bin"), # Rust/cargo tools
str(home / "go" / "bin"), # Go tools
str(home / ".npm-global" / "bin"), # npm global packages
]
return [p for p in candidates if p not in path_entries and Path(p).exists()]
def _build_wsl_interop_paths(path_entries: list[str]) -> list[str]:
"""WSL Windows-interop PATH entries for generated units: systemd services don't inherit the
Windows PATH (``/mnt/c/WINDOWS/System32``…), so ``powershell.exe``/``cmd.exe`` break unless persisted."""
if not is_wsl():
return []
candidates = [entry for entry in os.environ.get("PATH", "").split(os.pathsep) if entry.startswith("/mnt/")]
for executable in ("powershell.exe", "cmd.exe", "explorer.exe", "wsl.exe"):
resolved = shutil.which(executable)
if resolved:
candidates.append(str(Path(resolved).parent))
candidates += [
entry
for entry in (
"/mnt/c/WINDOWS/system32",
"/mnt/c/WINDOWS",
"/mnt/c/WINDOWS/System32/Wbem",
"/mnt/c/WINDOWS/System32/WindowsPowerShell/v1.0/",
"/mnt/c/WINDOWS/System32/OpenSSH/",
)
if Path(entry).exists()
]
result: list[str] = []
seen = set(path_entries)
for entry in candidates:
if entry and entry not in seen:
seen.add(entry)
result.append(entry)
return result
def _remap_path_for_user(path: str, target_home_dir: str) -> str:
"""Swap the ``Path.home()`` prefix of *path* for *target_home_dir*; other paths return unchanged.
Intentionally does NOT resolve symlinks."""
current_home = Path.home()
p = Path(path).expanduser()
try:
relative = p.relative_to(current_home)
return str(Path(target_home_dir) / relative)
except ValueError:
return str(p)
def _systemd_env_line(name: str, value: str) -> str:
"""One ``Environment="NAME=value"`` line: ``\\`` and ``"`` escaped for systemd's quoting, ``%``
doubled so specifier expansion leaves the value alone."""
escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("%", "%%")
return f'Environment="{name}={escaped}"\n'
def _installed_unit_ld_library_path(system: bool) -> str:
"""``LD_LIBRARY_PATH`` baked into the installed unit; ``""`` when absent."""
return _unit_environment_value(get_systemd_unit_path(system=system), "LD_LIBRARY_PATH") or ""
def _ld_library_path_line(system: bool, target_home_dir: str | None = None) -> str:
"""Carry the installer's LD_LIBRARY_PATH into the unit (glibc reads it only at process start, so
~/.hermes/.env is too late for CUDA libs — #14613); system units remap caller-home components.
The installed unit is the fallback source: the unit is regenerated and compared on every
start/restart/status, and a shell without the export (ssh, cron, ``sudo`` strips ``LD_*``)
must not be able to "repair" the line away."""
raw = os.environ.get("LD_LIBRARY_PATH", "") or _installed_unit_ld_library_path(system)
components = [p for p in raw.split(":") if p]
if target_home_dir is not None:
components = [_remap_path_for_user(p, target_home_dir) for p in components]
return _systemd_env_line("LD_LIBRARY_PATH", ":".join(components)) if components else ""
def _hermes_home_for_target_user(target_home_dir: str) -> str:
"""Remap the current HERMES_HOME (root's, under sudo) to the target user's equivalent:
``/root/.hermes[/profiles/x]`` → ``/home/alice/.hermes[/profiles/x]``; custom paths kept as-is."""
current_hermes_raw = os.environ.get("HERMES_HOME", "").strip()
current_hermes = Path(current_hermes_raw).expanduser() if current_hermes_raw else get_hermes_home()
# Keep paths lexical: resolving a non-existent path can bake a different HERMES_HOME into the unit.
current_default = Path.home() / ".hermes"
target_default = Path(target_home_dir) / ".hermes"
try:
# Default ~/.hermes or a profile/subdir of it → preserve the relative structure under the target.
return str(target_default / current_hermes.relative_to(current_default))
except ValueError:
return str(current_hermes) # Completely custom path (not under ~/.hermes) — keep as-is
def _build_service_path_dirs(project_root: Path | None = None) -> list[str]:
"""Build PATH directory list for service units, excluding non-existent dirs."""
if project_root is None:
project_root = PROJECT_ROOT
def _is_dir(path: Path) -> bool:
try:
return path.is_dir()
except OSError:
return False
candidates = []
# Python and dependency executable paths are selected at boot, not persisted.
hermes_home = get_hermes_home()
extras = (project_root / "node_modules" / ".bin", hermes_home / "node" / "bin", hermes_home / "node_modules" / ".bin")
for extra in extras:
if _is_dir(extra):
candidates.append(str(extra))
return candidates
def _stable_service_working_dir() -> str:
"""WorkingDirectory that won't disappear under systemd (HERMES_HOME, else PROJECT_ROOT). cwd is
irrelevant to ``-m`` resolution, and a pinned transient checkout rots: systemd fails at CHDIR
(status=200) before Python loads, so the unit self-heal never runs and Restart=always crash-loops."""
try:
home = get_hermes_home()
if home and Path(home).is_dir():
return str(Path(home).resolve())
except Exception:
pass
return str(PROJECT_ROOT)
def _systemd_watchdog_seconds(hermes_home: str | Path | None = None) -> int:
"""Resolve the managed-overlay-aware watchdog setting for a service home."""
override_token = reset_home_override = None
if hermes_home is not None:
from hermes_constants import (reset_hermes_home_override, set_hermes_home_override)
override_token = set_hermes_home_override(hermes_home)
reset_home_override = reset_hermes_home_override
try:
config = load_gateway_config()
return coerce_systemd_watchdog_seconds(getattr(config, "systemd_watchdog_seconds", 0))
except Exception:
logger.debug("Could not resolve effective systemd watchdog configuration", exc_info=True)
return 0
finally:
if override_token is not None and reset_home_override is not None:
reset_home_override(override_token)
def _pm_managed_node_dirs(home: Path) -> list[str]:
"""Node dirs pm's installed-state records under *home*'s store, resolved
via ``Facts.env_for`` (``{{store}}`` templates against the store beside
the recorded facts). Only dirs present on disk count; a record whose
store dirs are gone vouches for nothing.
"""
from pm.lock import Facts
store = Path(home) / "tools"
facts = Facts(store / "facts.json")
dirs: list[str] = []
for name in ("node", "npm", "npx"):
for value in facts.env_for(name, store).get("PATH") or []:
if value and Path(value).is_dir():
dirs.append(str(value))
return dirs
def _append_node_dir_for_service(path_entries: list[str], hermes_root: Path | None = None) -> None:
"""Append the Node dir a service unit should use.
PM's installed-state is the owner: facts.json under the target hermes
home's store records node/npm PATH entries, and those dirs — resolved via
Facts.env_for — are used verbatim. With managed Node recorded, consulting
the invoker's PATH would make a system unit depend on who ran sudo, so
lookup stops there. The legacy ``<hermes>/node`` tree and finally a PATH
lookup are fallbacks for installs pm never recorded.
"""
home = Path(hermes_root) if hermes_root is not None else Path(get_hermes_home())
try:
managed_dirs = _pm_managed_node_dirs(home)
except Exception:
managed_dirs = [] # pm absent or unreadable: fall through to the legacy probe
for entry in managed_dirs:
if entry not in path_entries:
path_entries.append(entry)
if managed_dirs:
return
from hermes_constants import (hermes_managed_node_tree_present, iter_hermes_node_dirs)
managed_node_present = hermes_managed_node_tree_present(hermes_root)
for directory in iter_hermes_node_dirs(hermes_root) if managed_node_present else ():
entry = str(directory)
try:
present = directory.is_dir()
except OSError:
present = False
if present and entry not in path_entries:
path_entries.append(entry)
# With managed Node present, consulting the invoker's PATH would make a system unit depend on who ran sudo.
if managed_node_present:
return
resolved_node = shutil.which("node")
if not resolved_node:
return
# Use the dir where node is FOUND, not the symlink target (~/.local/bin/node often links into one profile).
resolved_node_dir = str(Path(resolved_node).parent)
if resolved_node_dir not in path_entries:
path_entries.append(resolved_node_dir)
def _systemd_command(argv: list[str]) -> str:
"""Quote argv for systemd, including its non-shell specifier expansion."""
return " ".join('"' + part.replace("\\", "\\\\").replace('"', '\\"')
.replace("%", "%%").replace("$", "$$") + '"' for part in argv)
def _prepare_service_launcher(*, system: bool = False, run_as_user: str | None = None) -> None:
"""Publish the source command before a service definition references it."""
from hermes_cli._launchers import ENTRY_POINTS, ensure_install_launchers, resolve_store_python
from hermes_constants import set_hermes_home_override, reset_hermes_home_override
root, home = PROJECT_ROOT, get_hermes_home()
owner = None
if system:
username, _group, home_dir, uid = _system_service_identity(run_as_user)
root = Path(_remap_path_for_user(str(root), home_dir))
home = Path(_hermes_home_for_target_user(home_dir))
owner = (uid, username)
token = set_hermes_home_override(home)
try:
if resolve_store_python(root) is None:
return # Externally owned Nix/developer runtime.
local = root / ".hermes" / "bin"
paths = ensure_install_launchers(root, local)
if len(paths) != len(ENTRY_POINTS):
raise RuntimeError("Could not publish the gateway installation launcher")
if owner is not None:
import pwd
uid, username = owner
gid = pwd.getpwnam(username).pw_gid
for path in (local.parent, local, *map(Path, paths)):
os.chown(path, uid, gid)
finally:
reset_hermes_home_override(token)
def generate_systemd_unit(system: bool = False, run_as_user: str | None = None) -> str:
from hermes_cli._launchers import installation_command
python_path = get_python_path()
working_dir = _stable_service_working_dir()
project_root = PROJECT_ROOT
path_entries = _build_service_path_dirs()
if not system:
# System units add managed Node once the TARGET user's home is known (not the sudo caller's).
_append_node_dir_for_service(path_entries)
# TimeoutStopSec must cover the full stop budget (cron drain + cleanup) or systemd SIGKILLs mid-drain.
restart_timeout = resolve_systemd_timeout_stop_sec(_get_restart_drain_timeout(), _get_cron_drain_timeout())
if system:
username, group_name, home_dir, uid = _system_service_identity(run_as_user)
hermes_home = _hermes_home_for_target_user(home_dir)
# Profile arg relative to the TARGET user's ~/.hermes when hermes_home lives under it.
target_root = Path(home_dir) / ".hermes"
try:
Path(hermes_home).resolve().relative_to(target_root.resolve())
profile_arg = _profile_arg(hermes_home, default_root=target_root)
except ValueError:
profile_arg = _profile_arg(hermes_home)
# Remap paths under the calling user's home (/root/) to the target user's so the service can read them.
python_path = _remap_path_for_user(python_path, home_dir)
working_dir = str(hermes_home) if hermes_home else _remap_path_for_user(working_dir, home_dir)
project_root = Path(_remap_path_for_user(str(project_root), home_dir))
path_entries = [_remap_path_for_user(p, home_dir) for p in path_entries]
# Managed Node for the TARGET user's tree, prepended so it outranks remapped shell-PATH entries.
_target_node_entries: list[str] = []
_append_node_dir_for_service(_target_node_entries, Path(hermes_home) if hermes_home else None)
path_entries = [e for e in _target_node_entries if e not in path_entries] + path_entries
user_home = Path(home_dir)
identity_lines = f"User={username}\nGroup={group_name}\n"
# Restart-safe cron/Kanban workers cross `systemd-run --user`, which needs this user's manager;
# without the ordering the gateway and user@<uid>.service race at boot and the one-shot bus
# adoption in run_gateway() can miss (#104893).
ordering_lines = f"After=user@{uid}.service\nWants=user@{uid}.service\n"
env_lines = (
f'Environment="HOME={home_dir}"\n'
f'Environment="USER={username}"\n'
f'Environment="LOGNAME={username}"\n'
) + _ld_library_path_line(system=True, target_home_dir=home_dir)
wanted_by = "multi-user.target"
else:
hermes_home = str(get_hermes_home().resolve())
profile_arg = _profile_arg(hermes_home)
user_home = Path.home()
identity_lines = ordering_lines = ""
env_lines = _ld_library_path_line(system=False)
wanted_by = "default.target"
watchdog_seconds = _systemd_watchdog_seconds(hermes_home)
systemd_type, systemd_watchdog_directives = "simple", ""
if watchdog_seconds > 0:
systemd_type, systemd_watchdog_directives = "notify", f"NotifyAccess=main\nWatchdogSec={watchdog_seconds}s\n"
path_entries.extend(_build_user_local_paths(user_home, path_entries))
path_entries.extend(_build_wsl_interop_paths(path_entries))
path_entries.extend(["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"])
sane_path = ":".join(path_entries)
start = installation_command(project_root, [*shlex.split(profile_arg), "gateway", "run"],
python=python_path, home=hermes_home)
cleanup = installation_command(project_root, module="gateway.cgroup_cleanup",
python=python_path, home=hermes_home)
stop_mark = installation_command(project_root, module="gateway.systemd_stop_mark",
python=python_path, home=hermes_home)
return f"""[Unit]
Description={SERVICE_DESCRIPTION}
After=network-online.target
Wants=network-online.target
{ordering_lines}StartLimitIntervalSec=0
[Service]
Type={systemd_type}
{systemd_watchdog_directives}{identity_lines}ExecStart={_systemd_command(start)}
WorkingDirectory={working_dir}
{env_lines}Environment="PATH={sane_path}"
Environment="HERMES_HOME={hermes_home}"
Environment="HERMES_SUPERVISED_CHILD=1"
Restart=always
RestartSec=5
RestartForceExitStatus={GATEWAY_SERVICE_RESTART_EXIT_CODE}
SuccessExitStatus={GATEWAY_SERVICE_RESTART_EXIT_CODE}
RestartPreventExitStatus={GATEWAY_FATAL_CONFIG_EXIT_CODE}
KillMode=mixed
KillSignal=SIGTERM
ExecReload=/bin/kill -USR1 $MAINPID
ExecStop=-{_systemd_command(stop_mark)}
ExecStopPost=-{_systemd_command(cleanup)}
TimeoutStopSec={restart_timeout}
StandardOutput=journal
StandardError=journal
[Install]
WantedBy={wanted_by}
"""
def _normalize_service_definition(text: str) -> str:
return "\n".join(line.rstrip() for line in text.strip().splitlines())
# Directives older systemd silently strips; ignored in stale-checks so such units aren't flagged forever.
_SYSTEMD_OPTIONAL_DIRECTIVES = ("RestartMaxDelaySec", "RestartSteps")
def _strip_optional_systemd_directives(text: str) -> str:
"""Remove systemd directives that older hosts silently drop."""
filtered = []
for line in text.splitlines():
stripped = line.strip()
is_directive = stripped and not stripped.startswith("#")
if not (is_directive and stripped.split("=", 1)[0].strip() in _SYSTEMD_OPTIONAL_DIRECTIVES):
filtered.append(line)
return "\n".join(filtered)
def _normalize_launchd_plist_for_comparison(text: str) -> str:
"""Normalize plist text for staleness checks, ignoring the PATH payload: the generated PATH is
captured from the invoking shell and varies across shells."""
import re
return re.sub(
r"(<key>PATH</key>\s*<string>)(.*?)(</string>)", r"\1__HERMES_PATH__\3",
_normalize_service_definition(text), flags=re.S,
)
def systemd_unit_is_current(system: bool = False) -> bool:
# HERMES_HOME sync chokepoint for every compare/regenerate path: under `sudo … --system` it is often
# stripped to /root/.hermes, so refresh would rewrite a correct unit and status warn forever.
# Idempotent; the os.environ mutation persists for later runtime reads (restart's PID/drain).
_sync_hermes_home_from_systemd_unit(system=system)
unit_path = get_systemd_unit_path(system=system)
if not unit_path.exists():
return False
installed = unit_path.read_text(encoding="utf-8-sig")
expected_user = _read_systemd_user_from_unit(unit_path) if system else None
expected = generate_systemd_unit(system=system, run_as_user=expected_user)
# Ignore directives older systemd drops (RestartMaxDelaySec, RestartSteps) to avoid a perpetual "outdated" flag.
norm = lambda text: _normalize_service_definition(_strip_optional_systemd_directives(text)) # noqa: E731
return norm(installed) == norm(expected)
def _temp_home_in_service_definition(definition: str) -> str | None:
"""Temp-dir HERMES_HOME baked into a systemd unit / launchd plist, or None. A temp home means a
test/E2E harness generated it; installing it leaves the gateway "running" but deaf to every platform."""
import re
import tempfile
candidates = re.findall(r'HERMES_HOME=([^"\n]+)', definition)
candidates += re.findall(r"<key>HERMES_HOME</key>\s*<string>(.*?)</string>", definition, flags=re.S)
temp_roots = {
Path(tempfile.gettempdir()).resolve(),
Path("/tmp"), Path("/var/tmp"), Path("/private/tmp"), Path("/private/var/tmp"), # no-tmp: ok — detects a temp HERMES_HOME in service definitions
}
for raw in candidates:
try:
resolved = Path(raw.strip().strip('"')).resolve()
except (OSError, ValueError):
continue
if any(resolved == root or root in resolved.parents for root in temp_roots):
return raw.strip()
return None
def _refuse_temp_home_service_write(definition: str, kind: str) -> bool:
"""Refuse (with guidance) when a service definition carries a temp HERMES_HOME."""
temp_home = _temp_home_in_service_definition(definition)
if temp_home is None:
return False
print(f"✗ Refusing to write the gateway {kind}: HERMES_HOME resolves to a temporary directory ({temp_home}).")
print(
" This usually means a test/E2E environment exported HERMES_HOME. "
"Unset it (or run from a clean shell) and retry."
)
return True
def _retire_hermes_replace_dropin(system: bool = False) -> bool:
"""Remove only the legacy ``--replace`` drop-in written by Hermes."""
unit_path = get_systemd_unit_path(system=system)
dropin = unit_path.parent / f"{unit_path.name}.d" / "20-replace.conf"
try:
text = dropin.read_text(encoding="utf-8-sig")
except OSError:
return False
if not all(token in text for token in ("Added to end the gateway respawn storm", "--replace", "ExecStart=")):
return False
dropin.unlink()
return True
def refresh_systemd_unit_if_needed(system: bool = False) -> bool:
"""Rewrite the installed systemd unit when the generated definition has changed."""
unit_path = get_systemd_unit_path(system=system)
if not unit_path.exists():
return False
# systemd_unit_is_current is the HERMES_HOME-sync chokepoint; its env mutation persists for the regenerate below.
current = systemd_unit_is_current(system=system)
if _retire_hermes_replace_dropin(system=system):
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
print(f"↻ Removed the stale Hermes --replace drop-in from the gateway {_service_scope_label(system)} service")
if current:
return True
elif current:
return False
expected_user = _read_systemd_user_from_unit(unit_path) if system else None
new_unit = generate_systemd_unit(system=system, run_as_user=expected_user)
# Test safety belt: the user unit path is under Path.home(), which conftest does NOT sandbox, and a
# pytest-tmp HERMES_HOME baked into the developer's real unit breaks their gateway on next reboot.
if not system and any(m in new_unit for m in ("/pytest-of-", '/hermes_test"', "/hermes_test/")):
return False
# Structural variant: refuse ANY temp-dir HERMES_HOME (manual E2E homes lack the pytest markers).
if _refuse_temp_home_service_write(new_unit, "systemd unit"):
return False
_prepare_service_launcher(system=system, run_as_user=expected_user)
unit_path.write_text(new_unit, encoding="utf-8")
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
print(f"↻ Updated gateway {_service_scope_label(system)} service definition to match the current Hermes install")
return True
def _print_linger_enable_warning(username: str, detail: str | None = None, *, system: bool = False) -> None:
print()
if system:
print(f"⚠ Linger not enabled for {username} — cron and Kanban workers cannot start (no user D-Bus).")
else:
print("⚠ Linger not enabled — gateway may stop when you close this terminal.")
if detail:
print(f" Auto-enable failed: {detail}")
print()
print(" Enable it manually:" if system else " On headless servers (VPS, cloud instances) run:")
print(f" sudo loginctl enable-linger {username}")
print()
print(" Then restart the gateway:")
sudo, _, user_flag = _systemd_cli_bits(system)
print(f" {sudo}systemctl {user_flag}restart {get_service_name()}.service")
print()
def _ensure_linger_enabled(username: str | None = None, *, system: bool = False) -> bool:
"""Enable linger for *username* (default: the current user) when possible.
A user unit needs linger so the gateway survives logout. A system unit (``system=True``) needs
it for its ``User=`` so ``user@<uid>.service`` provides the D-Bus that ``systemd-run --user
--scope`` — every restart-safe cron/Kanban worker — connects to (#104893). Returns True only
when linger was enabled by this call.
"""
if is_termux() or not is_linux():
return False
if username is None:
import getpass
username = getpass.getuser()
enabled_msg = (
f"✓ Systemd linger is enabled for {username} (worker D-Bus available)" if system
else "✓ Systemd linger is enabled (service survives logout)"
)
if Path(f"/var/lib/systemd/linger/{username}").exists():
print(enabled_msg)
return False
linger_enabled, linger_detail = get_systemd_linger_status(username)
if linger_enabled is True:
print(enabled_msg)
return False
if not shutil.which("loginctl"):
_print_linger_enable_warning(username, linger_detail or "loginctl not found", system=system)
return False
if system:
print(f"Enabling linger for {username} so cron and Kanban workers can reach systemd-run --user...")
else:
print("Enabling linger so the gateway survives SSH logout...")
try:
result = _loginctl_enable_linger(username)
except Exception as e:
_print_linger_enable_warning(username, str(e), system=system)
return False
if result.returncode != 0:
_print_linger_enable_warning(username, _completed_process_detail(result) or linger_detail, system=system)
return False
print(f"✓ Enabled linger for {username}" if system else "✓ Linger enabled — gateway will persist after logout")
return True
def _ensure_system_service_linger(username: str) -> None:
"""Enable linger for the installed unit's ``User=`` (root included: restart-safe workers always cross
``systemd-run --user``, so a root gateway needs ``user@0.service`` just the same).
After a fresh enable, wait for the TARGET user's bus: logind starts ``user@<uid>.service``
asynchronously and ``--start-now`` boots the gateway immediately. A gateway that was already running
keeps its bus-less environment and ``systemctl start`` on an active unit is a no-op — say so rather
than let the repair silently not take."""
if not _ensure_linger_enabled(username, system=True):
return
import pwd
uid = pwd.getpwnam(username).pw_uid # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
if _wait_for_target_user_bus(uid):
print(f"✓ /run/user/{uid}/bus is up — cron and Kanban workers can use systemd-run --user")
else:
print(f"⚠ /run/user/{uid}/bus did not appear within 5s.")
print(f" Start the user manager: sudo systemctl start user@{uid}.service")
if _systemd_unit_is_active(system=True):
print(" The running gateway was started without a user D-Bus; restart it to pick one up:")
print(f" sudo systemctl restart {get_service_name()}.service")
def _select_systemd_scope(system: bool = False) -> bool:
return system or (get_systemd_unit_path(system=True).exists() and not get_systemd_unit_path(system=False).exists())
def _system_scope_wizard_would_need_root(system: bool = False) -> bool:
"""True when the wizard would trigger a system-scope operation as non-root — mirrors
``_select_systemd_scope`` so the dead-end is detected BEFORE prompting."""
if os.geteuid() == 0: # windows-footgun: ok — systemd scope wizard decision, never invoked on Windows
return False
return _select_systemd_scope(system=system)
def _print_system_scope_remediation(action: str) -> None:
"""Print remediation when the wizard skips a system-scope action because the user isn't root."""
print_warning(f"Gateway is installed as a system-wide service — {action} requires root.")
print_info(" Options:")
print_info(f" 1. {action.capitalize()} it this time:")
print_info(f" sudo systemctl {action} {get_service_name()}")
print_info(" 2. Switch to a per-user service (recommended for personal use):")
print_info(" sudo hermes gateway uninstall --system")
print_info(" hermes gateway install")
print_info(" hermes gateway start")
def _get_restart_drain_timeout() -> float:
"""Return the configured gateway restart drain timeout in seconds."""
raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip()
if not raw:
cfg = read_raw_config()
agent_cfg = cfg.get("agent", {}) if isinstance(cfg, dict) else {}
raw = str(agent_cfg.get("restart_drain_timeout", DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT))
return parse_restart_drain_timeout(raw)
def _agent_timeout_setting(env_var: str, key: str, parse) -> float:
"""``parse(env)`` when the env var is non-empty, else ``parse(agent.<key>)`` (None if unset)."""
env_raw = os.getenv(env_var)
if env_raw is not None and str(env_raw).strip() != "":
return parse(env_raw)
cfg = read_raw_config()
agent_cfg = cfg.get("agent", {}) if isinstance(cfg, dict) else {}
if isinstance(agent_cfg, dict) and key in agent_cfg:
return parse(agent_cfg.get(key))
return parse(None)
def _get_cron_drain_timeout() -> float:
"""Return the configured cron-only drain floor in seconds.
See #82161.
"""
return _agent_timeout_setting("HERMES_CRON_DRAIN_TIMEOUT", "cron_drain_timeout", parse_cron_drain_timeout)
def _get_restart_exit_wait_budget() -> float:
"""CLI wait for gateway exit after SIGUSR1 / self-restart (#77184)."""
return resolve_restart_exit_wait_budget(
_get_restart_drain_timeout(),
_agent_timeout_setting(
"HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout
),
_get_cron_drain_timeout(),
)
def systemd_install(
force: bool = False,
system: bool = False,
run_as_user: str | None = None,
enable_on_startup: bool = True,
non_interactive: bool = False,
):
if system:
_require_root_for_system_service("install")
# Offer to remove legacy units first: alongside the new unit they flap-fight for the bot token.
if has_legacy_hermes_units():
print()
print_legacy_unit_warning()
print()
if non_interactive or prompt_yes_no("Remove the legacy unit(s) before installing?", True):
remove_legacy_hermes_units(interactive=False)
print()
unit_path = get_systemd_unit_path(system=system)
scope_label = _service_scope_label(system)
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
# Existing system units already pin HERMES_HOME; adopt it before any regenerate.
if unit_path.exists():
_sync_hermes_home_from_systemd_unit(system=system)
if unit_path.exists() and not force:
if not systemd_unit_is_current(system=system):
print(f"↻ Repairing outdated {scope_label} systemd service at: {unit_path}")
refresh_systemd_unit_if_needed(system=system)
if enable_on_startup:
_run_systemctl(["enable", get_service_name()], system=system, check=True, timeout=30)
print(f"✓ {scope_label.capitalize()} service definition updated")
else:
print(f"Service already installed at: {unit_path}")
print("Use --force to reinstall")
# Same post-install guarantee as a fresh install: a repaired user unit must survive logout too.
configured_user = _read_systemd_user_from_unit(unit_path) if system else None
if configured_user:
_ensure_system_service_linger(configured_user)
elif not system:
_ensure_linger_enabled()
return
unit_path.parent.mkdir(parents=True, exist_ok=True)
new_unit = generate_systemd_unit(system=system, run_as_user=run_as_user)
if _refuse_temp_home_service_write(new_unit, "systemd unit"):
return
print(f"Installing {scope_label} systemd service to: {unit_path}")
_prepare_service_launcher(system=system, run_as_user=run_as_user)
unit_path.write_text(new_unit, encoding="utf-8")
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
if enable_on_startup:
_run_systemctl(["enable", get_service_name()], system=system, check=True, timeout=30)
print()
print(f"✓ {scope_label.capitalize()} service {'installed and enabled' if enable_on_startup else 'installed'}!")
print()
print("Next steps:")
print(f" {sudo}hermes gateway start{scope_flag} # Start the service")
print(f" {sudo}hermes gateway status{scope_flag} # Check status")
print(f" journalctl {user_flag}-u {get_service_name()} -f # View logs")
print()
if system:
configured_user = _read_systemd_user_from_unit(unit_path)
if configured_user:
print(f"Configured to run as: {configured_user}")
_ensure_system_service_linger(configured_user)
else:
_ensure_linger_enabled()
print_systemd_scope_conflict_warning()
print_legacy_unit_warning()
def _systemd_scope_preamble(
action: str, system: bool, *, require_installed: bool = True, preflight_user: bool = False
) -> bool:
"""Resolve the effective scope, then enforce root (system) / user D-Bus reachability (user, when
``preflight_user``) and — when ``require_installed`` — that the unit exists. Returns the scope."""
system = _select_systemd_scope(system)
if system:
_require_root_for_system_service(action)
elif preflight_user:
# Fail fast with guidance when the user D-Bus session is unreachable (raises UserSystemdUnavailableError).
_preflight_user_systemd()
if require_installed:
_require_service_installed(action, system=system)
return system
def _systemd_unit_belongs_to_current_home(system: bool = False) -> bool:
"""False (with a warning) when the installed unit pins a HERMES_HOME other than this process's: the
service name then resolved to ANOTHER install's gateway, and stop/disable/unlink would take it down."""
_sync_hermes_home_from_systemd_unit(system=system) # sudo strips HERMES_HOME; adopt the unit's first
unit_home = _hermes_home_from_systemd_unit_file(system=system)
if unit_home is None or Path(unit_home).expanduser().resolve() == get_hermes_home().resolve():
return True
print_warning(
f"Refusing to remove {get_systemd_unit_path(system=system)}: it runs HERMES_HOME={unit_home}, "
f"but this process has HERMES_HOME={get_hermes_home()}"
)
return False
def systemd_uninstall(system: bool = False):
system = _systemd_scope_preamble("uninstall", system, require_installed=False)
if not _systemd_unit_belongs_to_current_home(system):
return
_run_systemctl(["stop", get_service_name()], system=system, check=False, timeout=90)
_run_systemctl(["disable", get_service_name()], system=system, check=False, timeout=30)
unit_path = get_systemd_unit_path(system=system)
if unit_path.exists():
unit_path.unlink()
print(f"✓ Removed {unit_path}")
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
print(f"✓ {_service_scope_label(system).capitalize()} service uninstalled")
def _print_service_not_installed(system: bool) -> None:
sudo, scope_flag, _ = _systemd_cli_bits(system)
print("✗ Gateway service is not installed")
print(f" Run: {sudo}hermes gateway install{scope_flag}")
def _require_service_installed(action: str, system: bool = False) -> None:
if not get_systemd_unit_path(system=system).exists():
_print_service_not_installed(system)
sys.exit(1)
def systemd_start(system: bool = False):
system = _systemd_scope_preamble("start", system, preflight_user=True)
# HERMES_HOME sync happens in refresh's systemd_unit_is_current gate; the unit is guaranteed to exist here.
refresh_systemd_unit_if_needed(system=system)
_run_systemctl(["start", get_service_name()], system=system, check=True, timeout=30)
print(f"✓ {_service_scope_label(system).capitalize()} service started")
def systemd_stop(system: bool = False):
system = _systemd_scope_preamble("stop", system)
_sync_hermes_home_from_systemd_unit(system=system)
_mark_planned_stop()
try:
_run_systemctl(["stop", get_service_name()], system=system, check=True, timeout=90)
except subprocess.TimeoutExpired:
print(
f"Gateway {_service_scope_label(system)} service is still stopping after 90s; "
"check `hermes gateway status` or logs for final shutdown state."
)
return
print(f"✓ {_service_scope_label(system).capitalize()} service stopped")
def systemd_restart(system: bool = False):
system = _systemd_scope_preamble("restart", system, preflight_user=True)
# HERMES_HOME sync happens in refresh's systemd_unit_is_current gate; its os.environ mutation
# persists for the get_running_pid / drain-timeout reads below.
refresh_systemd_unit_if_needed(system=system)
from gateway.status import get_running_pid
pid = get_running_pid() or _systemd_main_pid(system=system)
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
# Event loop provably dead: SIGUSR1 can't drain it, so bounded SIGTERM → SIGKILL and let systemd relaunch.
print(
# Health probe says the event loop is provably dead (#81642): SIGUSR1 can never drain it, so the
# graceful wait below would burn the full budget. A busy-but-alive gateway (fresh heartbeat)
# never takes this path — its in-flight work, including the #86684 cron drain floor, keeps the
# full graceful budget.
# Health probe says the event loop is provably dead (#81642): the gateway cannot process a
# graceful shutdown, so waiting the full drain budget only stalls the restart (and `hermes
# update` behind it) for 180s. Bounded escalation instead: SIGTERM grace → SIGKILL → proceed,
# ~10s worst case. Never taken for a busy-but-alive gateway — a fresh heartbeat keeps the drain
# path (and the #86684 cron drain floor) fully intact.
f"⚠ Gateway PID {pid} event loop is unresponsive — "
"skipping graceful drain and forcing a bounded stop..."
)
_escalate_wedged_gateway(pid)
svc = get_service_name()
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
_run_systemctl(["restart", svc], system=system, check=False, timeout=90)
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
return
if pid is not None:
service_action = _systemd_graceful_restart_action(system, pid)
if service_action:
_systemd_reset_and_run(service_action, system=system, previous_pid=pid)
return
if _recover_pending_systemd_restart(system=system, previous_pid=pid):
return
_systemd_reset_and_run("restart", system=system, previous_pid=pid)
def _systemd_graceful_restart_action(system: bool, pid: int) -> str | None:
"""SIGUSR1-drain the live gateway ``pid``; return the follow-up ``systemctl`` verb (``"start"`` /
``"restart"``) the caller must still issue, or None when systemd already owns the relaunch."""
scope_label = _service_scope_label(system).capitalize()
# Graceful in-band restart, mirroring the systemd branch. Previously this sent a bare SIGTERM and waited
# ``_get_restart_drain_timeout()`` — which defaults to 0, so the wait could never succeed and every
# restart fell through to ``kickstart -k``. A bare SIGTERM also leaves ``restart_requested`` False, so
# the gateway exits 1 instead of 75 and reports itself to chat as "shutting down" rather than
# "restarting", losing the resume_pending handoff. SIGUSR1 is the drain-aware path: refuse new turns,
# wait for in-flight work (``agent.restart_after_turn_timeout``), then stop() within
# ``agent.restart_drain_timeout``. The wait budget must cover BOTH phases plus headroom (#77184) — the
# raw drain timeout covers only the second. Announce the wait BEFORE it runs: it can last the full
# budget while the old gateway finishes in-flight agent runs, and it streams into surfaces with no other
# feedback — the desktop updater's live output most of all, where a silent stop here reads as "update
# stuck" (#44515).
wait_budget = _get_restart_exit_wait_budget()
print(
f"⏳ {scope_label} service restarting gracefully (PID {pid}) — "
f"waiting up to {wait_budget:.0f}s for in-flight turns + drain..."
)
from hermes_cli.update_cmd_drain_report import drain_progress_reporter
if not _graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)):
print(f"⚠ Graceful restart did not complete within {int(wait_budget)}s; forcing a service restart...")
return "restart"
# Exit 75 hands restart ownership to systemd; observe that replacement rather than restarting again.
replacement_observed: list[bool] = []
if _wait_for_systemd_service_restart(system=system, previous_pid=pid, replacement_observed=replacement_observed):
return None
if replacement_observed or _systemd_service_is_start_limited(system=system):
return None
# A replacement may have started but not reached runtime readiness in time; never stop that generation.
props = _read_systemd_unit_properties(system=system)
if not props:
return None
replacement_pid = _systemd_main_pid_from_props(props)
if (
props.get("ActiveState") in {"active", "activating", "reloading"}
or props.get("SubState") == "auto-restart"
or (replacement_pid is not None and replacement_pid != pid)
):
return None
print("⚠ Systemd did not relaunch the gateway after its graceful exit; starting the inactive service...")
# ``start`` is intentionally idempotent: a replacement appearing after the snapshot must not be stopped.
return "start"
def _systemd_reset_and_run(action: str, *, system: bool, previous_pid) -> None:
"""``reset-failed`` then ``systemctl <action>``, then wait for the relaunch. Start-limit
rejection prints the wait hint instead of raising; a 90s timeout prints where to look."""
svc = get_service_name()
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
try:
_run_systemctl([action, svc], system=system, check=True, timeout=90)
except subprocess.CalledProcessError as exc:
if _systemd_error_indicates_start_limit(exc) or _systemd_service_is_start_limited(system=system):
_print_systemd_start_limit_wait(system=system)
return
raise
except subprocess.TimeoutExpired:
print(
f"Gateway {_service_scope_label(system)} service is still restarting after 90s; "
"check `hermes gateway status` or logs for final state."
)
return
_wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
def systemd_status(deep: bool = False, system: bool = False, full: bool = False):
system = _select_systemd_scope(system)
unit_path = get_systemd_unit_path(system=system)
svc = get_service_name()
scope_label = _service_scope_label(system).capitalize()
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
if not unit_path.exists():
_print_service_not_installed(system)
return
if has_conflicting_systemd_units():
print_systemd_scope_conflict_warning()
print()
if has_legacy_hermes_units():
print_legacy_unit_warning()
print()
if not systemd_unit_is_current(system=system):
print("⚠ Installed gateway service definition is outdated")
print(f" Run: {sudo}hermes gateway restart{scope_flag} # auto-refreshes the unit")
print()
status_cmd = ["status", svc, "--no-pager"] + (["-l"] if full else [])
_run_systemctl(status_cmd, system=system, capture_output=False, timeout=10)
result = _run_systemctl(["is-active", svc], system=system, timeout=10, **_CAPTURE_TEXT)
if result.stdout.strip() == "active":
print(f"✓ {scope_label} gateway service is running")
else:
print(f"✗ {scope_label} gateway service is stopped")
print(f" Run: {sudo}hermes gateway start{scope_flag}")
configured_user = _read_systemd_user_from_unit(unit_path) if system else None
if configured_user:
print(f"Configured to run as: {configured_user}")
_print_runtime_health()
unit_props = _read_systemd_unit_properties(system=system)
active_state = unit_props.get("ActiveState", "")
result_code = unit_props.get("Result", "")
if active_state == "activating" and unit_props.get("SubState", "") == "auto-restart":
print(" ⏳ Restart pending: systemd is waiting to relaunch the gateway")
elif _systemd_unit_is_start_limited(unit_props):
print(" ⏳ Restart pending: systemd is temporarily rate-limiting starts")
print(f" Run after the start-limit window expires: {sudo}hermes gateway restart{scope_flag}")
print(f" Or clear it manually: systemctl {user_flag}reset-failed {svc}")
elif active_state == "failed" and unit_props.get("ExecMainStatus", "") == str(GATEWAY_SERVICE_RESTART_EXIT_CODE):
print(" ⚠ Planned restart is stuck in systemd failed state (exit 75)")
print(f" Run: systemctl {user_flag}reset-failed {svc} && {sudo}hermes gateway start{scope_flag}")
elif active_state == "failed" and result_code:
print(f" ⚠ Systemd unit result: {result_code}")
if system:
print("✓ System service starts at boot without requiring systemd linger")
else:
linger_enabled, linger_detail = get_systemd_linger_status()
if linger_enabled is True:
print("✓ Systemd linger is enabled (service survives logout)")
elif linger_enabled is False:
print("⚠ Systemd linger is disabled (gateway may stop when you log out)")
print(" Run: sudo loginctl enable-linger $USER")
elif deep:
print(f"⚠ Could not verify systemd linger ({linger_detail})")
print(" If you want the gateway user service to survive logout, run:")
print(" sudo loginctl enable-linger $USER")
if deep:
print()
print("Recent logs:")
log_cmd = ["journalctl"] + ([] if system else ["--user"]) + ["-u", svc, "-n", "20", "--no-pager"]
if full:
log_cmd.append("-l")
subprocess.run(log_cmd, timeout=10)
# =============================================================================
# Launchd (macOS)
# =============================================================================
from hermes_cli.gateway_launchd import ( # noqa: E402,F401 — facade re-exports; tests patch here
get_launchd_label,
_probe_launchd_domain_for_label,
_launchd_domain,
_LAUNCHD_JOB_UNLOADED_EXIT_CODES,
_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES,
_launchd_error_indicates_unloaded,
_launchctl_domain_unsupported,
_LAUNCHCTL_BOOTSTRAP_EIO,
_launchctl_bootstrap,
_launchd_reload_log_path,
_append_launchd_reload_log,
_launchd_reload_budget,
_launchctl_supervised_pid,
_launchctl_label_supervising_process,
_retry_launchctl_bootstrap_until_registered,
_launchd_unsupported_marker_path,
_write_launchd_unsupported_marker,
_clear_launchd_unsupported_marker,
_launchd_unsupported_marker_exists,
_gateway_run_command,
_timestamped_stderr_gateway_command,
_spawn_detached_gateway,
_launchd_fallback_to_detached,
_launchd_degrade_or_raise,
generate_launchd_plist,
launchd_plist_is_current,
_spawn_deferred_launchd_reload,
refresh_launchd_plist_if_needed,
launchd_install,
launchd_uninstall,
launchd_start,
_launchctl_kickstart_current,
_launchd_bootstrap_and_kickstart,
_launchd_ok,
launchd_stop,
_launchd_kickstart,
_wait_for_launchd_service_pid,
launchd_restart,
LAUNCHD_SUPERVISION_VERIFY_TIMEOUT,
wait_for_launchd_gateway_supervision,
launchd_status,
)
# Cached launchd domain — probe once per process invocation.
_resolved_launchd_domain: str | None = None
def _wait_for_gateway_exit(timeout: float = 10.0, force_after: float | None = 5.0) -> bool:
"""Wait up to ``timeout`` s for the gateway (by gateway.pid, not launchd labels, so multiple
HERMES_HOMEs work) to exit; SIGKILL it after ``force_after`` s of graceful waiting."""
from gateway.status import get_process_start_time, get_running_pid
deadline = time.monotonic() + timeout
force_deadline = (time.monotonic() + force_after) if force_after is not None else None
force_sent = False
while time.monotonic() < deadline:
pid = get_running_pid()
if pid is None:
return True # Process exited cleanly.
if force_after is not None and not force_sent and time.monotonic() >= force_deadline:
# Grace period expired — force-kill the specific PID.
try:
terminate_pid(pid, force=True, expected_start_time=get_process_start_time(pid))
print(f"⚠ Gateway PID {pid} did not exit gracefully; sent SIGKILL")
except (ProcessLookupError, PermissionError, OSError):
return True # Already gone or we can't touch it.
force_sent = True
time.sleep(0.3)
# Timed out even after force-kill.
remaining_pid = get_running_pid()
if remaining_pid is not None:
print(f"⚠ Gateway PID {remaining_pid} still running after {timeout}s — restart may fail")
return False
return True
def _wait_for_tcp_port_free(host: str, port: int, *, timeout: float = 10.0) -> bool:
"""Wait until nothing accepts TCP connections on host:port.
PID exit is not enough on macOS: api_server disables SO_REUSEADDR, so a restart that wins
the race logs EADDRINUSE and keeps running with no API. Connection-refused means the
listener is gone; a timed-out connect is a live listener with a slow accept queue.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with socket.create_connection((host, port), timeout=0.2):
pass
except ConnectionRefusedError:
return True
except TimeoutError:
pass # a slow accept queue is still a live listener
except OSError:
return True # unresolvable/unreachable address: nothing to wait for; the bind retry covers it
time.sleep(0.1)
return False
def _wait_for_api_server_port_free(*, timeout: float = 10.0) -> bool:
"""Wait for the configured api_server listen address to stop accepting.
Only when api_server is enabled: with the platform off, a foreign listener on the default
port is nobody's race and must not delay the restart."""
from gateway.config import Platform
from gateway.platforms.api_server import listen_address
pconfig = load_gateway_config().platforms.get(Platform.API_SERVER)
if pconfig is None or not pconfig.enabled:
return True
host, port = listen_address(pconfig.extra or {})
freed = _wait_for_tcp_port_free(host, port, timeout=timeout)
if not freed:
print(
f"⚠ {host}:{port} still accepting connections — "
"new api_server may fail to bind"
)
return freed
# =============================================================================
# Gateway Runner
# =============================================================================
def _truthy_env(value: str | None) -> bool:
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
def _is_official_docker_checkout() -> bool:
return str(PROJECT_ROOT) == "/opt/hermes" and (PROJECT_ROOT / "docker" / "entrypoint.sh").is_file()
def _running_under_gateway_supervisor() -> bool:
"""True when this process IS the supervisor-launched gateway, so the conflict guard never wedges
the service into a respawn/refuse loop. Markers: systemd INVOCATION_ID, launchd XPC_SERVICE_NAME
(shells inherit "0"), s6 HERMES_S6_SUPERVISED_CHILD, or ``--external-supervisor``."""
return is_gateway_supervisor_process()
def host_multiplexer_serving(profile_name: str | None = None):
"""The ONE live host gateway when it serves ``profile_name`` (default: the current profile).
Companion to :func:`named_profile_served_by_running_multiplexer`, which answers the narrower
"is this a SATELLITE of the default's multiplexer" and is hard-False for ``default`` — the
profile that usually owns the shared process. Every lifecycle guard built on it was therefore
blind to the host process itself. This one is true for ``default`` too, and it reports WHICH
profiles the host process serves. Returns a ``gateway.host_attach.HostGateway`` or None.
"""
try:
from gateway.host_attach import host_gateway_serving
name = profile_name if profile_name is not None else _current_profile_name()
return host_gateway_serving(name or "default")
except Exception:
logger.debug("Host multiplexer probe failed", exc_info=True)
return None
def _served_by_another_host_gateway(profile_name: str | None = None):
"""The host gateway serving ``profile_name`` when it is NOT this home's own process.
The owner must never be guarded out of restarting itself: a refusal keyed on "something serves
you" would make `hermes gateway restart` impossible for the profile that launched the host
process. Guards want "ANOTHER process already serves you", which is this.
"""
gateway = host_multiplexer_serving(profile_name)
if gateway is None:
return None
try:
from gateway.status import _get_process_hermes_home, _same_hermes_home
if _same_hermes_home(gateway.home, _get_process_hermes_home()):
return None
except Exception:
logger.debug("Host multiplexer home comparison failed", exc_info=True)
return gateway
def named_profile_served_by_running_multiplexer(profile_name: str | None = None) -> bool:
"""True when a live default multiplexer already ticks this named profile (a satellite profile has no
gateway.pid; the multiplexer fires its jobs and serves its platforms). Defaults to the current profile.
See #97120.
"""
try:
suffix = profile_name if profile_name is not None else _current_profile_name()
except Exception:
return False
if not suffix or suffix == "default":
return False
# The host record answers first: it names the live host process whatever home launched it, so a
# multiplexer started by a named profile is visible here too.
if host_multiplexer_serving(suffix) is not None:
return True
try:
from hermes_constants import get_default_hermes_root
default_root = get_default_hermes_root()
except Exception:
return False
try:
from hermes_cli.gateway_multiplex_served import live_default_gateway_pid, recorded_served_profiles
if live_default_gateway_pid() is None:
return False
from hermes_cli.profiles import normalize_profile_name
# The live gateway's own record wins: the CLI process cannot see an env-only opt-in on the
# default profile (`hermes -p X` loads X's .env) and a config edit after start is not live yet.
# Only a record without the key (pre-multiplex writer) falls through to config derivation.
recorded = recorded_served_profiles(default_root)
if recorded is not None:
return normalize_profile_name(suffix) in {normalize_profile_name(p) for p in recorded}
# No record (older gateway): only an EXPLICIT opt-in counts. The unset default is settled by
# the gateway at boot (it may have stayed standalone); a CLI process must not guess it on.
from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag
return explicit_multiplex_flag(default_root) is True # a multiplexer serves every named profile
except Exception:
logger.debug("Multiplexer-serving probe failed", exc_info=True)
return False
def _served_profile_needs_no_service() -> bool:
"""Print the "already served" note and return True when a setup flow must not install a standalone
service: a live multiplexing default gateway already serves this named profile, so the unit/plist it
would register can only sit dead (the start guard refuses it) or double-bind its platforms.
Shared by ``hermes setup gateway`` / ``hermes setup`` / ``hermes import`` (``ensure_gateway_service``)
and the ``hermes gateway setup`` wizard. See #111958."""
if not named_profile_served_by_running_multiplexer():
# Not served (yet): a named profile still gets no service of its own — same rule and text
# as `gateway install`, so `hermes -p X setup` cannot grow a fleet member the verb refuses.
return _named_profile_refused_under_multiplexer()
from hermes_cli.profiles import profile_is_standalone
if profile_is_standalone(get_hermes_home()):
from gateway.host_attach import standalone_rescan_message
print_info(standalone_rescan_message(_current_profile_name()))
return True
print_success(
f"Profile '{_current_profile_name()}' is already served by the default multiplexer."
)
print_info(" (served now by the running multiplexed gateway — add its bot token and it connects)")
print_info(" No standalone gateway service was installed or started.")
return True
def _named_profile_refused_under_multiplexer(force: bool = False) -> bool:
"""Print the refusal and return True when a NAMED profile must not get a gateway of its own.
One gateway per host serves every profile, so a ``<root>/profiles/<name>`` home never installs or
starts a standalone gateway: either the host gateway already serves it (a second one would
double-bind its platforms: two pollers on one token, port fights) or no host gateway runs yet and
the DEFAULT profile is where it is installed. Refusing only the served case let a host with no
multiplexer running (or one that had not rescanned yet) grow a brand-new per-profile fleet member.
``--force`` is the one escape (a fleet split across UNIX users or a ``HERMES_HOME`` outside
``profiles/``); a service it already installed stays startable without it. Shared by ``run`` and the service verbs (``start``/``install``/``restart``): a
refusal only inside ``gateway run`` leaves the service manager to discover it — systemd parks the
unit on exit 78 while the CLI prints "started"; launchd (KeepAlive, no exit-status gating)
respawns it every ThrottleInterval forever."""
if force:
return False
try:
suffix = _current_profile_name()
from hermes_constants import profile_name_for_home
from hermes_cli.profiles import profile_is_standalone
# A profile that authored gateway.standalone: true opted out of the host multiplexer: it is
# allowed a gateway of its own without --force. Only a RUNNING host record that still lists
# it (the host has not rescanned since the key was set) is refused with the rescan remedy.
standalone = (profile_name_for_home(get_hermes_home()) not in (None, "default")
and profile_is_standalone(get_hermes_home()))
# A unit/plist/task already registered for this home was installed with --force: that fleet
# member (and the supervisor relaunching it, whose ExecStart carries no --force) is not NEW.
new_standalone = (profile_name_for_home(get_hermes_home()) not in (None, "default")
and not _is_service_installed())
except Exception:
return False
owner = _served_by_another_host_gateway()
served = owner is not None or named_profile_served_by_running_multiplexer()
if standalone:
if not served:
return False
from gateway.host_attach import standalone_rescan_message
print_error(standalone_rescan_message(suffix))
return True
if not served and not new_standalone:
return False
if served:
print_error(f"The host gateway already serves profile '{suffix}'.")
if owner is not None:
print(f" {owner.describe()}")
else:
print_error(f"Profile '{suffix}' does not get a gateway of its own.")
print(
" Exactly one gateway per host is the inbound process for every\n"
" profile. Starting a separate gateway for this profile would\n"
" double-bind its platforms (two pollers on one bot token, port\n"
" conflicts).\n"
)
if served:
print(" Manage the host gateway instead:")
print()
print(f" hermes -p {owner.profile_label if owner is not None else 'default'} gateway restart")
else:
print(" Install or start the host gateway from the default profile; it serves this one too:")
print()
print(" hermes gateway install")
print()
print(" Or fold an existing per-profile fleet onto one host gateway:")
print()
print(" hermes gateway migrate --multiplex")
print()
print(" A separate per-profile gateway (for a fleet split across UNIX users or a")
print(f" HERMES_HOME outside profiles/) needs --force: hermes -p {suffix} gateway install --force")
print()
from hermes_constants import display_hermes_home
from hermes_cli.gateway_multiplex_mode import STANDALONE_DEPRECATION_NOTICE
print(" Temporary compatibility path while multiplexing gaps are closed: set")
print(f" gateway.standalone: true in {display_hermes_home(get_hermes_home())}/config.yaml,")
print(" then wait for the host gateway to rescan (<=30s) or send its rescan-profiles control verb.")
print(f" ({STANDALONE_DEPRECATION_NOTICE})")
return True
def _guard_named_profile_under_multiplexer(force: bool = False) -> None:
"""Exit-78 form of ``_named_profile_refused_under_multiplexer`` for the CLI entry points."""
if not _named_profile_refused_under_multiplexer(force=force):
return
# EX_CONFIG, not 1: the refusal is decided purely by config, so it is permanent. The systemd unit
# (Restart=always, StartLimitIntervalSec=0) relies on RestartPreventExitStatus=78 as its only
# backstop — exit 1 turned a correct refusal into an unbounded restart loop; s6 maps 78 to
# "permanent failure" too.
# This refusal is decided entirely by configuration (multiplex_profiles plus the allowlist), so it is
# permanent: no number of retries can change the answer. Exiting 1 made it look transient to a service
# manager -- and the systemd unit this module generates pairs Restart=always/RestartSec=5 with
# StartLimitIntervalSec=0, deliberately trading systemd's generic start-rate limiter for the specific
# RestartPreventExitStatus=GATEWAY_FATAL_CONFIG_EXIT_CODE backstop declared beside it. Returning 1 left
# that backstop unarmed with the limiter already off, so a correct refusal became an unbounded restart
# loop. 78 also reaches the s6 finish script's 125 "permanent failure" translation (see #51228), the
# same path the other fatal-config exits take.
sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
def _host_decision_exit_code(decision) -> int:
"""Exit code for a host-attach verdict a supervisor may be watching.
``GATEWAY_FATAL_CONFIG_EXIT_CODE`` (78) is the PERMANENT refusal: systemd parks the unit on it
(``RestartPreventExitStatus``), the s6 finish script maps it to 125, launchd maps it to a
deliberate stop. That is right for a config-derived refusal and wrong for a runtime one — "some
other process serves me right now" ends the moment that process goes away, and parking the unit
on it strands the profile until a human notices. Transient verdicts therefore use
``GATEWAY_SERVICE_RESTART_EXIT_CODE`` (75, EX_TEMPFAIL), which every supervisor we generate
already retries: systemd has ``RestartForceExitStatus=75`` with ``RestartSec=5``, the s6 finish
script passes it through, and launchd relaunches a non-78 failure. Exit 0 would NOT do: s6
parks a clean exit too.
"""
if getattr(decision, "transient", False):
return GATEWAY_SERVICE_RESTART_EXIT_CODE
return GATEWAY_FATAL_CONFIG_EXIT_CODE
def _attach_to_host_gateway_or_guard(force: bool = False, replace: bool = False) -> None:
"""``gateway run`` against the ONE host gateway: attach, rescan-then-attach, replace, or refuse.
A profile the host process already serves has nothing to run: print who serves it and exit 0
without spawning anything. Under a service supervisor the SAME situation exits 75 instead, so
the unit is RETRIED rather than parked (see :func:`_host_decision_exit_code`).
``--replace`` and ``--force`` are the two escape hatches this guard must not eat: both return
here so ``start_gateway`` can act on them (it owns the signalling and the PID claim).
"""
if force:
return
try:
from gateway.host_attach import ATTACH, REFUSE, REPLACE_HOST, decide
decision = decide(get_hermes_home(), replace=replace)
except Exception:
logger.debug("Host gateway attach probe failed", exc_info=True)
decision = None
if decision is not None and decision.outcome == REPLACE_HOST:
return # start_gateway replaces the owner; the config guard below must not pre-empt it
if decision is not None and decision.outcome in (ATTACH, REFUSE):
print(decision.message)
if decision.outcome == REFUSE:
code = _host_decision_exit_code(decision)
# stdout goes to the supervisor's unit log; under launchd a permanent refusal is then
# mapped to a clean exit and the unit is parked. The profile's own logs (errors.log,
# WARNING+) are where a parked fleet is diagnosed, so name the verdict and the remedy there.
logger.warning("gateway run refused (exit %d): %s", code, decision.message)
sys.exit(code)
if _running_under_gateway_supervisor():
sys.exit(_host_decision_exit_code(decision))
sys.exit(0)
# No host record (older gateway, unwritable lock dir): the config-derived refusal still applies.
_guard_named_profile_under_multiplexer(force=force)
def _guard_supervised_gateway_conflict(force: bool = False) -> None:
"""Refuse a foreground gateway when a service manager already supervises one: a shell-launched run
becomes a second dispatcher that escapes the cgroup, survives ``systemctl restart``, and writes the
shared kanban DB concurrently (multi-writer SQLite WAL corruption). ``--force`` starts anyway.
See #35240.
"""
if force or _running_under_gateway_supervisor():
return
try:
snapshot = get_gateway_runtime_snapshot()
except Exception:
logger.debug("Supervised-gateway conflict probe failed", exc_info=True)
return
if not (snapshot.service_installed and snapshot.service_running):
return
print_error(f"A gateway is already running under {snapshot.manager} for this profile.")
print(
" Starting another one from a shell leaves an orphan dispatcher that\n"
" escapes the service, survives restarts, and writes to the same kanban\n"
" DB concurrently — which can corrupt it. Restart the supervised gateway\n"
" instead:"
)
print()
print(" hermes gateway restart")
print()
print(
" Pass --force to start a foreground gateway anyway (not recommended\n"
" while the service is running)."
)
sys.exit(1)
def _guard_existing_gateway_process_conflict(replace: bool = False) -> None:
"""Cheap PID-file preflight before the expensive ``gateway.run`` import (the authoritative lock check):
supervisor loops re-running bare ``gateway run`` burned memory on plugin discovery just to fail
"already running". Same user-facing contract; never scans other HERMES_HOME roots."""
if replace or _running_under_gateway_supervisor():
return
try:
from gateway.status import get_running_pid
pid = get_running_pid()
except Exception:
logger.debug("Existing-gateway process probe failed", exc_info=True)
return
if pid is None:
# get_running_pid() filters by the current profile's HERMES_HOME; warn if the PID file
# belongs to another profile (user switched profiles while the old gateway still runs).
try:
from gateway.status import _read_pid_record, _pid_record_belongs_to_current_profile
stale = _read_pid_record()
if stale is not None and not _pid_record_belongs_to_current_profile(stale):
logger.warning(
"PID file belongs to another profile (hermes_home=%s). "
"The old gateway may still be running under that profile.",
stale.get("hermes_home", "<unknown>"),
)
except Exception:
pass
return
print_error(f"A gateway is already running (PID {pid}), so your bots are most likely online already.")
print(" Check with `hermes gateway status`.")
print(" To restart it: `hermes gateway restart`. To stop it: `hermes gateway stop`.")
print(" To replace it from here: `hermes gateway run --replace`.")
sys.exit(1)
def _guard_official_docker_root_gateway() -> None:
"""Refuse gateway startup when the official Docker privilege drop was bypassed."""
if not hasattr(os, "geteuid") or os.geteuid() != 0 or _truthy_env(os.getenv("HERMES_ALLOW_ROOT_GATEWAY")):
return
if not _is_official_docker_checkout():
return
print_error("Refusing to run the Hermes gateway as root inside the official Docker image.")
print(
" The image entrypoint normally drops privileges to the 'hermes' user. "
"If you override entrypoint in Docker Compose, include "
"/opt/hermes/docker/entrypoint.sh before the Hermes command."
)
print(
" Running the gateway as root can leave root-owned files in "
"$HERMES_HOME and break later non-root dashboard/gateway runs."
)
print(" Set HERMES_ALLOW_ROOT_GATEWAY=1 only if you intentionally accept this risk.")
sys.exit(1)
def _apply_startup_watchdog_config() -> None:
"""Idempotent backstop arming of the startup-liveness watchdog. Must run AFTER the conflict guards (a
--replace loser must not arm one). config.yaml gateway.startup_watchdog* is the user surface; env
vars bridge it because the argv fast-path arms before config loads, and explicit env wins. arm() is
idempotent, so a config timeout needs disarm+re-arm. GatewayRunner disarms once the loop is live."""
try:
from hermes_startup_watchdog import (
ENV_STARTUP_WATCHDOG, ENV_STARTUP_WATCHDOG_TIMEOUT_S, arm_startup_watchdog,
disarm_startup_watchdog, startup_watchdog_disabled,
)
_sw_timeout_bridged = False
try:
from hermes_cli.config import load_config as _sw_load_config
_gw_cfg = (_sw_load_config() or {}).get("gateway", {}) or {}
if ENV_STARTUP_WATCHDOG not in os.environ and not _gw_cfg.get("startup_watchdog", True):
os.environ[ENV_STARTUP_WATCHDOG] = "0"
_sw_timeout = _gw_cfg.get("startup_watchdog_timeout_seconds")
if ENV_STARTUP_WATCHDOG_TIMEOUT_S not in os.environ and _sw_timeout is not None:
os.environ[ENV_STARTUP_WATCHDOG_TIMEOUT_S] = str(_sw_timeout)
_sw_timeout_bridged = True
except Exception:
pass
if startup_watchdog_disabled():
disarm_startup_watchdog()
else:
if _sw_timeout_bridged:
disarm_startup_watchdog()
arm_startup_watchdog()
except Exception:
pass
def _absorb_windows_console_controls() -> None:
"""Make a detached Windows gateway ignore console-control broadcasts from sibling CLIs."""
try:
signal.signal(signal.SIGINT, signal.SIG_IGN)
if hasattr(signal, "SIGBREAK"):
signal.signal(signal.SIGBREAK, signal.SIG_IGN)
except (OSError, ValueError):
pass # SetConsoleCtrlHandler unavailable (rare) — best-effort
# signal only hooks SIGINT/SIGBREAK; SetConsoleCtrlHandler(NULL, TRUE) ignores ALL console
# control events (CTRL_CLOSE/CTRL_LOGOFF included), as background services should.
try:
import ctypes
ctypes.windll.kernel32.SetConsoleCtrlHandler(None, 1) # type: ignore[attr-defined]
except (OSError, AttributeError):
pass
def _make_exit_diag():
"""``_exit_diag(tag, **extra)`` recorder writing ``logs/gateway-exit-diag.log`` — captures every way
``asyncio.run()`` can return, for chasing silent Windows gateway deaths. HERMES_GATEWAY_EXIT_DIAG=0 opts out."""
from datetime import datetime as _dt, timezone as _tz
def _exit_diag(tag: str, **extra: object) -> None:
if os.environ.get("HERMES_GATEWAY_EXIT_DIAG", "1") != "1":
return
try:
from hermes_constants import get_hermes_home as _ghh
log_dir = _ghh() / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
line = {
"ts": _dt.now(_tz.utc).isoformat(), "tag": tag, "pid": os.getpid(),
"python": sys.version.split()[0], "platform": sys.platform, **extra,
}
with open(log_dir / "gateway-exit-diag.log", "a", encoding="utf-8") as f:
f.write(json.dumps(line, default=str) + "\n")
except Exception:
pass # never let the diagnostic itself crash the gateway
return _exit_diag
def _respawn_storm_backoff() -> None:
"""Portable app-level respawn-storm breaker (for supervisors without a floor). Defaults mirror
DEFAULT_CONFIG ``gateway.respawn_storm``; HERMES_GATEWAY_MAX_STARTS / HERMES_GATEWAY_START_WINDOW_S
override; max_starts <= 0 disables. Never blocks startup."""
try:
from gateway.status import record_start_and_check_storm
_max_starts = 5
_win = 120.0
try:
from hermes_cli.config import load_config
_cfg = load_config()
_gw = _cfg.get("gateway") if isinstance(_cfg, dict) else None
_rs = _gw.get("respawn_storm") if isinstance(_gw, dict) else None
if isinstance(_rs, dict):
if isinstance(_rs.get("max_starts"), int):
_max_starts = _rs["max_starts"]
if isinstance(_rs.get("window_seconds"), (int, float)):
_win = float(_rs["window_seconds"])
except Exception:
pass
try:
_max_starts = int(os.environ["HERMES_GATEWAY_MAX_STARTS"])
except (KeyError, ValueError):
pass
try:
_win = float(os.environ["HERMES_GATEWAY_START_WINDOW_S"])
except (KeyError, ValueError):
pass
_storm = record_start_and_check_storm(max_starts=_max_starts, window_s=_win) if _max_starts > 0 else None
if _storm is not None:
logger.warning(
"Gateway (re)started %d times in %.0fs — backing off %.0fs to break a respawn storm.",
_storm.count, _storm.window_s, _storm.backoff_s,
)
# Tell the startup watchdog the backoff sleep is intentional, not a parked deadlock.
try:
from hermes_startup_watchdog import kick_startup_watchdog
kick_startup_watchdog(extra_s=_storm.backoff_s)
except Exception:
pass
time.sleep(_storm.backoff_s)
except Exception as _be:
logger.debug("respawn-storm breaker check failed (non-fatal): %s", _be)
def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, force: bool = False):
"""Run the gateway in foreground. verbose 1=INFO/2+=DEBUG on stderr; quiet: no stderr logs; replace:
kill an existing instance first (avoids systemd restart loops); force: skip the supervised guard."""
_guard_official_docker_root_gateway()
_attach_to_host_gateway_or_guard(force=force, replace=replace)
_guard_supervised_gateway_conflict(force=force)
_guard_existing_gateway_process_conflict(replace=replace)
sys.path.insert(0, str(PROJECT_ROOT))
_apply_startup_watchdog_config()
# Detached Windows runs (HERMES_GATEWAY_DETACHED=1, or non-TTY for older wrappers) ignore
# console-control broadcasts from sibling CLIs; foreground runs keep Ctrl+C-to-stop.
stdin_is_tty = bool(_stdin_is_tty())
_console_window_attached = _windows_console_window_attached()
_breakaway = _windows_gateway_breakaway_state()
_absorb = _windows_gateway_should_absorb_console_controls()
if _absorb:
_absorb_windows_console_controls()
# A system-level unit execs us without XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS; adopt our own
# user bus before any worker env snapshot so `systemd-run --user --scope` works (#104893).
if is_linux() and os.environ.get("INVOCATION_ID"):
_ensure_user_systemd_env()
# Refresh the systemd unit on every boot so restart settings stay current even after an
# exit-code-75 respawn (stale-code or /restart), which bypasses `hermes gateway restart`.
if supports_systemd_services():
try:
refresh_systemd_unit_if_needed(system=False)
except Exception:
pass # best-effort; don't block gateway startup
from gateway.run import start_gateway
print("┌─────────────────────────────────────────────────────────┐")
print("│ ☤ Hermes Gateway Starting... │")
print("├─────────────────────────────────────────────────────────┤")
print("│ Messaging platforms + cron scheduler │")
print("│ Press Ctrl+C to stop │")
print("└─────────────────────────────────────────────────────────┘")
print()
# Exit 1 if no platform connects so systemd Restart=always retries transient errors.
verbosity = None if quiet else verbose
import atexit as _atexit
import traceback as _traceback
_exit_diag = _make_exit_diag()
_exit_diag(
"gateway.start", replace=replace, argv=sys.argv, stdin_is_tty=stdin_is_tty,
console_window_attached=_console_window_attached, detached=_gateway_detached_env(),
breakaway=_breakaway, absorb_windows_console_controls=_absorb,
)
_atexit.register(lambda: _exit_diag("atexit.hook", sys_exc=repr(sys.exc_info())))
_respawn_storm_backoff()
def _hard_exit_after_gateway_teardown(code: int) -> None:
# Mirror gateway.run.main()'s wedge-proof exit: bypass Python finalization so non-daemon
# threads (in-flight cron jobs) can't delay a /restart by minutes.
from gateway.run import _exit_after_graceful_shutdown
_exit_after_graceful_shutdown(code)
success = False
try:
success = asyncio.run(start_gateway(replace=replace, force=force, verbosity=verbosity))
_exit_diag("asyncio.run.returned", success=success)
except KeyboardInterrupt:
# Detached Windows runs absorb SIGINT above; keep the handler for console runs.
_exit_diag("asyncio.run.KeyboardInterrupt", traceback=_traceback.format_exc())
print("\nGateway stopped.")
_hard_exit_after_gateway_teardown(0)
return # unreachable in production (os._exit); guard for test stubs
except SystemExit as e:
_exit_diag("asyncio.run.SystemExit", code=e.code, traceback=_traceback.format_exc())
_hard_exit_after_gateway_teardown(0 if e.code is None else e.code if isinstance(e.code, int) else 1)
except BaseException as e:
# Everything else (CancelledError, exotic BaseExceptions): log the cause, then re-raise.
_exit_diag("asyncio.run.exception", exc_type=type(e).__name__, exc_repr=repr(e), traceback=_traceback.format_exc())
raise
if not success:
_exit_diag("gateway.exit_nonzero")
_hard_exit_after_gateway_teardown(1)
_exit_diag("gateway.exit_clean")
_hard_exit_after_gateway_teardown(0)
# =============================================================================
# Gateway Setup (Interactive Messaging Platform Configuration)
# =============================================================================
from hermes_cli.gateway_setup_wizard import ( # noqa: E402,F401 — facade re-exports; tests patch here
_PLATFORMS,
_all_platforms,
_platform_status,
_set_platform_unauthorized_dm_behavior,
_print_setup_header,
_confirm_reconfigure,
_offer_home_channel,
_save_env_values,
_prompt_csv,
_UNAUTHORIZED_ACCESS_CHOICES,
_prompt_unauthorized_access,
_telegram_auto_setup,
_clean_discord_ids,
_prompt_allowlist_var,
_setup_standard_platform,
_WEIXIN_DM_POLICIES,
_WEIXIN_GROUP_NOTE,
_setup_weixin,
_setup_qqbot,
_signal_line_input,
_setup_signal,
_builtin_setup_fn,
_configure_platform,
_wizard_offer_service_action,
_setup_service_action,
_WIZARD_BANNER,
_WIZARD_BACKEND_LABELS,
_WIZARD_NO_SERVICE_LINES,
_wizard_service_status_block,
_wizard_platform_loop,
_wizard_install_service,
_wizard_post_setup,
gateway_setup,
)
# Operator wording for the out-of-loop watchdog exit reasons stamped by gateway/shutdown_watchdog.py.
_WATCHDOG_EXIT_REASONS = {
"loop_liveness_watchdog": (
"event loop stopped dispatching (housekeeping, cron and the kanban dispatcher froze); "
f"the liveness watchdog exited with code {GATEWAY_SERVICE_RESTART_EXIT_CODE} for the supervisor to restart it"
),
"shutdown_watchdog": "shutdown drain wedged; the shutdown watchdog forced the exit (see logs/gateway-shutdown-watchdog.log)",
}
def _runtime_health_lines() -> list[str]:
"""Summarize the latest persisted gateway runtime health state."""
try:
from gateway.status import (
read_runtime_status, runtime_status_heartbeat_age_s, runtime_status_is_stale, runtime_status_pid_is_live)
except Exception:
return []
state = read_runtime_status()
if not state:
return []
gateway_state = state.get("gateway_state")
exit_reason = state.get("exit_reason")
lines = [
f"⚠ {platform}: {pdata.get('error_message') or 'unknown error'}"
for platform, pdata in (state.get("platforms", {}) or {}).items()
if pdata.get("state") == "fatal"
]
# A live-claiming snapshot can outlive an ungracefully killed gateway (taskkill /F, OOM). Past
# the freshness TTL with the recorded PID gone, say so instead of rendering stale live state.
if gateway_state in ("running", "degraded", "starting", "draining") and runtime_status_is_stale(state):
if not runtime_status_pid_is_live(state):
lines.append(
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
"recorded process is gone (likely an ungraceful shutdown)"
)
return lines
# PID alive but housekeeping stopped re-stamping the file: the reporter's "not a crash" case
# (#113372) — the process looks 'running' while housekeeping/cron/kanban dispatch are frozen.
age = runtime_status_heartbeat_age_s(state)
if gateway_state != "draining" and age is not None:
lines.append(
f"⚠ Gateway heartbeat stale: housekeeping has not refreshed gateway_state.json for {age} s "
f"(event loop or housekeeping wedged; pid {state.get('pid')} alive) — restart the gateway"
)
if gateway_state == "startup_failed" and exit_reason:
lines.append(f"⚠ Last startup issue: {exit_reason}")
elif gateway_state == "degraded" and exit_reason:
# An out-of-loop watchdog hard-exited the process (#113372): the loop stopped dispatching, so
# housekeeping/cron/kanban froze together. Without this arm the file would read 'running'.
lines.append(f"⚠ Gateway exited degraded: {_WATCHDOG_EXIT_REASONS.get(exit_reason, exit_reason)}")
elif gateway_state == "draining":
action = "restart" if state.get("restart_requested") else "shutdown"
from gateway.status import parse_active_agents
count = parse_active_agents(state.get("active_agents"))
lines.append(f"⏳ Gateway draining for {action} ({count} active agent(s))")
work = state.get("active_work")
if isinstance(work, list) and work:
from hermes_cli.update_cmd_drain_report import describe_active_work_unit
lines.extend(f" • {describe_active_work_unit(u)}" for u in work if isinstance(u, dict))
elif gateway_state == "stopped" and exit_reason:
lines.append(f"⚠ Last shutdown reason: {exit_reason}")
return lines
def _print_info_lines(*lines: str) -> None:
for line in lines:
print_info(line)
# WhatsApp/DingTalk/WeCom/Feishu setup flows live in their plugins' adapter.py::interactive_setup.
def _running_under_s6() -> bool:
from hermes_cli.service_manager import detect_service_manager
return detect_service_manager() == "s6"
def _systemd_unit_installed() -> bool:
return supports_systemd_services() and (
get_systemd_unit_path(system=False).exists() or get_systemd_unit_path(system=True).exists()
)
def _is_service_installed() -> bool:
return _installed_service_kind() is not None
def _is_service_running() -> bool:
"""Check if the gateway service is currently running."""
if supports_systemd_services():
return _systemd_unit_is_active(False) or _systemd_unit_is_active(True)
if is_macos() and get_launchd_plist_path().exists():
try:
return _launchd_service_registered(get_launchd_label(), timeout=10)
except subprocess.TimeoutExpired:
return False
# Windows "installed" doesn't mean "running"; like manual runs, a live gateway process decides.
return len(find_gateway_pids()) > 0
def _print_indented(text: str, emit=print) -> None:
for line in text.splitlines():
emit(f" {line}")
def _service_backend(*, windows: bool = True) -> str | None:
"""Host service manager: ``"systemd"`` / ``"launchd"`` / ``"windows"`` / None, in the canonical
predicate order every subcommand routes on. ``windows=False`` never probes ``is_windows()``."""
if supports_systemd_services():
return "systemd"
if is_macos():
return "launchd"
if windows and is_windows():
return "windows"
return None
def _service_call(backend: str, verb: str, system: bool | None = False) -> None:
"""Run ``verb`` (start/stop/restart/uninstall) on ``backend``. Names resolve at call time so tests
can monkeypatch them; only systemd takes a scope, and ``system=None`` omits it (wizard restart)."""
if backend == "windows":
return getattr(_gw_windows(), verb)()
if backend == "launchd":
return globals()[f"launchd_{verb}"]()
fn = globals()[f"systemd_{verb}"]
return fn() if system is None else fn(system=system)
# =============================================================================
# Main Command Handler
# =============================================================================
def _dispatch_via_service_manager_if_s6(action: str, profile: str | None = None) -> bool:
"""Dispatch start/stop/restart via s6 inside an s6 container; True iff dispatched (caller returns).
Profile defaults to the current one; missing slot / s6 errors become actionable CLI messages."""
from hermes_cli.service_manager import (
GatewayNotRegisteredError, detect_service_manager, get_service_manager,
register_unregistered_profile_gateway,
)
if detect_service_manager() != "s6":
return False
if profile is None:
profile = _current_profile_name() # root home (Docker /opt/data included) is gateway-default
mgr = get_service_manager()
if action not in ("start", "stop", "restart"):
return False
service = f"gateway-{profile}"
try:
try:
getattr(mgr, action)(service)
except GatewayNotRegisteredError:
# A profile created from the HOST against a bind-mounted home has a directory but no
# slot (`profile create` cannot reach the container's /run/service). Only `start`
# repairs that; stop/restart on a missing slot stay an error.
if action != "start" or not register_unregistered_profile_gateway(mgr, profile):
raise
print(f"✓ registered the s6 gateway slot for profile {profile!r}")
mgr.start(service)
except (RuntimeError, ValueError, OSError) as exc: # S6Error is a RuntimeError
print(f"✗ {exc}")
sys.exit(1)
return True
def _dispatch_all_via_service_manager_if_s6(action: str) -> bool:
"""Dispatch ``--all`` stop/restart to every registered profile gateway under s6; True iff dispatched.
A bare pkill is seen by s6-supervise as a crash and restarted ~1s later; the service manager flips
``want up``/``want down`` correctly. ``start --all`` is not a CLI surface."""
from hermes_cli.service_manager import (detect_service_manager, get_service_manager)
if detect_service_manager() != "s6" or action not in ("stop", "restart"):
return False
mgr = get_service_manager()
profiles = mgr.list_profile_gateways()
if not profiles:
print("✗ No profile gateways registered under s6")
return True
fn = mgr.stop if action == "stop" else mgr.restart
errors: list[tuple[str, Exception]] = []
for profile in profiles:
try:
fn(f"gateway-{profile}")
except Exception as exc: # noqa: BLE001 — report and continue
errors.append((profile, exc))
succeeded = len(profiles) - len(errors)
verb = "stopped" if action == "stop" else "restarted"
if succeeded:
print(f"✓ {verb.capitalize()} {succeeded} profile gateway(s) under s6")
for profile, exc in errors:
print(f"✗ Could not {action} gateway-{profile}: {exc}")
return True
def gateway_command(args):
"""Handle gateway subcommands."""
try:
return _gateway_command_inner(args)
except UserSystemdUnavailableError as e:
# Actionable message, not a traceback, when the user D-Bus session is unreachable.
print_error("User systemd not reachable:")
_print_indented(str(e))
sys.exit(1)
except SystemScopeRequiresRootError as e:
# System-scope action typed without sudo; the wizard intercepts this earlier with guidance.
print(str(e))
sys.exit(1)
except (subprocess.CalledProcessError, RuntimeError) as e:
# systemctl exited non-zero or is missing entirely: guidance, not a traceback.
from hermes_cli.gateway_command_errors import explain_service_failure
lines = explain_service_failure(e)
if lines is None:
raise
print_error(lines[0])
_print_indented("\n".join(lines[1:]))
sys.exit(1)
def _maybe_redirect_run_to_s6_supervision(args) -> bool:
"""Inside an s6 container, upgrade bare ``gateway run`` to the supervised s6 longrun; True iff dispatched.
``HERMES_S6_SUPERVISED_CHILD`` (set by ``S6ServiceManager._render_run_script``) marks the supervised
child, which must run in foreground or we'd recurse run → start → run; ``--no-supervise`` /
HERMES_GATEWAY_NO_SUPERVISE=1 opts out (CI smoke, debugging)."""
no_supervise = getattr(args, "no_supervise", False) or \
os.environ.get("HERMES_GATEWAY_NO_SUPERVISE", "").lower() in ("1", "true", "yes")
# HERMES_S6_SUPERVISED_CHILD: we ARE the supervised child; fall through so the gateway starts.
if no_supervise or os.environ.get("HERMES_S6_SUPERVISED_CHILD"):
return False
if not _dispatch_via_service_manager_if_s6("start"):
return False
# This process never reaches a GatewayRunner, so the watchdog armed by hermes_cli.main's argv
# fast-path has no other disarm site: the in-process heartbeat below parks with zero CPU and no
# progress lease, which the watchdog reads as a startup deadlock and os._exit(75)s the CMD process.
from hermes_startup_watchdog import disarm_startup_watchdog
disarm_startup_watchdog()
# Breadcrumb on stderr (keep stdout clean for scripts); gateway logs follow via s6-log.
print(
"→ gateway is now running under s6 supervision (auto-restart on crash,\n"
" dashboard supervised alongside if HERMES_DASHBOARD is set).\n"
" This is the recommended setup for the s6 container image — the\n"
" gateway will keep running even if it crashes.\n"
" Use `--no-supervise` (or HERMES_GATEWAY_NO_SUPERVISE=1) to opt out\n"
" and get the pre-s6 foreground behavior instead.",
file=sys.stderr,
flush=True,
)
# Keep the CMD process alive as a heartbeat so the container survives gateway flaps (`docker stop`
# SIGTERMs it). Prefer `sleep infinity` (frees the interpreter); execvp only returns by raising
# (ENOENT with a clobbered PATH / no `sleep`), which used to crash containers.
try:
# The supervised gateway's lifetime is independent of this process — s6-supervise restarts it on
# crash, and we don't want the container to exit when the gateway flaps. The CMD process keeps /init
# alive until `docker stop` sends SIGTERM, at which point /init runs stage 3 shutdown (which tears
# down the supervised gateway cleanly). Prefer `sleep infinity` (matches the static main-hermes
# service's pattern in docker/s6-rc.d/main-hermes/run, and frees the Python interpreter — the
# heartbeat is a tiny `sleep` process, not a resident interpreter). But `os.execvp` does a PATH
# lookup for the `sleep` binary and historically crashed the whole container with FileNotFoundError
# when PATH was empty/truncated/clobbered at this point — e.g. after user customizations rewrote
# PATH, or on minimal images without `sleep` on PATH (issue #36208). Fall back to an in-process
# block (no external binary, can't fail on PATH) so the container keeps running instead of dying
# during boot.
os.execvp("sleep", ["sleep", "infinity"])
except OSError:
print(
"→ `sleep` is unavailable; keeping the s6 CMD process alive "
"in-process until the container is stopped.",
file=sys.stderr,
flush=True,
)
_block_until_terminated()
return True # unreachable on the execvp success path
def _block_until_terminated() -> None:
"""Heartbeat when ``execvp("sleep")`` fails. SIGTERM exits 128+signum so ``docker stop`` is clean;
``Event().wait()`` covers platforms without ``signal.pause()``.
Fallback heartbeat for when ``os.execvp("sleep", ...)`` can't run (``sleep`` missing from PATH — issue
#36208). Installs a SIGTERM handler that exits with the conventional 128+signum code so ``docker stop``
produces a clean, expected exit, then blocks on ``signal.pause()``. Windows) — although this path only
runs inside the s6 Linux container image, the fallback keeps the helper safe to import and unit-test
anywhere.
"""
signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))
pause = getattr(signal, "pause", None)
if pause is not None:
while True:
pause()
else: # pragma: no cover - non-Unix fallback, not exercised in the s6 image
import threading
threading.Event().wait()
def _installed_service_kind_for(windows) -> str | None:
"""``"systemd"`` / ``"launchd"`` when the unit/plist exists, else ``"windows"`` iff ``windows()``
(a thunk so it runs last, like every caller's original ladder), else None."""
if _systemd_unit_installed():
return "systemd"
if is_macos() and get_launchd_plist_path().exists():
return "launchd"
return "windows" if windows() else None
def _installed_service_kind() -> str | None:
"""Installed service kind; stricter than ``_service_backend`` (unit/plist/task must exist)."""
return _installed_service_kind_for(lambda: is_windows() and _gw_windows().is_installed())
def _stop_installed_service(system: bool) -> bool:
"""Stop the installed systemd/launchd/Windows service. Returns True if one was stopped."""
kind = _installed_service_kind()
if kind is None:
return False
# SystemScopeRequiresRootError is a RuntimeError and must propagate from systemd_stop.
try:
_service_call(kind, "stop", system)
return True
except (subprocess.CalledProcessError, *((RuntimeError,) if kind == "windows" else ())):
return False
def _refuse_from_inside_gateway(verb: str, reason: str) -> None:
"""Refuse self-targeting stop/restart/uninstall from inside the gateway process (#92560)."""
from tools.process_registry import _is_supervised_gateway_process
if _is_supervised_gateway_process():
print_error(
f"Refusing to {verb} the gateway from inside the gateway process.\n"
f"This command was blocked to prevent {reason}.\n"
f"Use `hermes gateway {verb}` from a shell outside the running gateway."
)
sys.exit(1)
def _print_lines(*lines: str) -> None:
for line in lines:
print(line)
def _print_runtime_health() -> None:
runtime_lines = _runtime_health_lines()
if runtime_lines:
print()
print("Recent gateway health:")
for line in runtime_lines:
print(f" {line}")
def _cmd_run(args):
if _maybe_redirect_run_to_s6_supervision(args):
return # unreachable; execvp doesn't return
if getattr(args, "external_supervisor", False):
os.environ[EXTERNAL_GATEWAY_SUPERVISOR_ENV] = "1"
run_gateway(
getattr(args, "verbose", 0), quiet=getattr(args, "quiet", False),
replace=getattr(args, "replace", False), force=getattr(args, "force", False),
)
def _cmd_setup(args):
gateway_setup()
_WSL_FOREGROUND_HINT = (
"", " hermes gateway run # direct foreground",
" tmux new -s hermes 'hermes gateway run' # persistent via tmux",
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # background",
)
# ``(exit_code, *lines)`` when a subcommand has no service backend, keyed by (subcommand, reason).
# Reasons in check order: "termux", "wsl" (no operational systemd), "s6" / "container", "unsupported".
# ``None`` exit code means plain return.
_NO_BACKEND_MESSAGES = {
("install", "termux"): (1,
"Gateway service installation is not supported on Termux.", "Run manually: hermes gateway"),
("install", "wsl"): (1,
"WSL detected but systemd is not running.",
"Either enable systemd (add systemd=true to /etc/wsl.conf and restart WSL)",
"or run the gateway in foreground mode:", *_WSL_FOREGROUND_HINT),
("install", "s6"): (None,
"Per-profile gateways are auto-registered when you create a profile.", "",
" hermes profile create <name> # creates the s6 service slot",
" hermes -p <name> gateway start # bring it up via s6",
" hermes status # see currently-supervised gateways"),
("install", "container"): (0,
"Service installation is not needed inside a Docker container.",
"The container runtime is your service manager — use Docker restart policies instead:", "",
" docker run --restart unless-stopped ... # auto-restart on crash/reboot",
" docker restart <container> # manual restart", "",
"To run the gateway: hermes gateway run"),
("install", "unsupported"): (1,
"Service installation not supported on this platform.", "Run manually: hermes gateway run"),
("uninstall", "termux"): (1,
"Gateway service uninstall is not supported on Termux because there is no managed service to remove.",
"Stop manual runs with: hermes gateway stop"),
("uninstall", "s6"): (None,
"Per-profile gateways are auto-unregistered when you delete the profile.", "",
" hermes profile delete <name> # tears down the s6 service slot",
" hermes -p <name> gateway stop # stop without deleting the profile"),
("uninstall", "container"): (0,
"Service uninstall is not applicable inside a Docker container.",
"To stop the gateway, stop or remove the container:", "",
" docker stop <container>", " docker rm <container>"),
("uninstall", "unsupported"): (1,
"Running the gateway as a background service is not available on this platform "
"(no systemd, launchd or Scheduled Tasks), so there is nothing to uninstall.",
"Stop a manually started gateway with: hermes gateway stop"),
("start", "termux"): (1,
"Gateway service start is not supported on Termux because there is no system service manager.",
"Run manually: hermes gateway"),
("start", "wsl"): (1,
"WSL detected but systemd is not available.",
"Run the gateway in foreground mode instead:", *_WSL_FOREGROUND_HINT, "",
"To enable systemd: add systemd=true to /etc/wsl.conf and run 'wsl --shutdown' from PowerShell."),
("start", "container"): (0,
"Service start is not applicable inside a Docker container.",
"The gateway runs as the container's main process.", "",
" docker start <container> # start a stopped container",
" docker restart <container> # restart a running container", "",
"Or run the gateway directly: hermes gateway run"),
("start", "unsupported"): (1,
"Running the gateway as a background service is not available on this platform "
"(no systemd, launchd or Scheduled Tasks).",
"Run it directly with: hermes gateway run"),
}
def _no_backend_exit(subcommand: str, reason: str) -> None:
code, *lines = _NO_BACKEND_MESSAGES[(subcommand, reason)]
_print_lines(*lines)
if code is not None:
sys.exit(code)
def _is_apt_termux_install() -> bool:
"""True when this install is a sealed ``apt-termux`` tree.
A Termux APT package is a sealed tree owned by the package manager with
no service manager behind it. Keyed on the steward stamp
(``sealed_steward``), never a platform probe, so the refusal follows the
INSTALL PROVENANCE (our apt distribution's own stamp) rather than the
ambient environment; the platform-shaped ``is_termux()`` lanes stay
for generic on-device installs. Foreground process management (stop/
status via the PID registry) is NOT gated: it works on every install.
"""
from hermes_cli.steward import STEWARD_APT_TERMUX, sealed_steward
return sealed_steward(PROJECT_ROOT) == STEWARD_APT_TERMUX
def _service_mgmt_blocked() -> bool:
"""Service lanes are unavailable: a Termux device (platform probe) or a
sealed apt-termux install (provenance probe). One predicate so a fourth
service gate cannot forget the disjunction."""
return is_termux() or _is_apt_termux_install()
def _handle_no_backend(subcommand: str, *, wsl: bool, s6: bool) -> None:
"""Fallthrough when no service backend matched. Predicate order: WSL (only when ``wsl``) ->
container (s6 slot hint only when ``s6``; ``start`` reaches here only when s6 isn't running) ->
unsupported."""
if wsl and is_wsl():
reason = "wsl"
elif is_container():
reason = "s6" if s6 and _running_under_s6() else "container"
else:
reason = "unsupported"
_no_backend_exit(subcommand, reason)
def _install_systemd_from_cli(args, *, force: bool, system: bool, run_as_user) -> None:
if is_wsl():
print_warning("WSL detected — systemd services may not survive WSL restarts.")
_print_info_lines(
" Consider running in foreground instead: hermes gateway run",
" Or use tmux/screen for persistence: tmux new -s hermes 'hermes gateway run'",
)
print()
# Honor --start-now/--start-on-login; else prompt on a TTY, default True headless.
non_interactive = not (hasattr(sys.stdin, "isatty") and sys.stdin.isatty())
def _flag(name: str, question: str) -> bool:
value = getattr(args, name, None)
if value is not None:
return value
return prompt_yes_no(question, True) if not non_interactive else True
start_now = _flag("start_now", "Start the gateway now after installing the service?")
start_on_login = _flag("start_on_login", "Start the gateway automatically on login/boot with systemd?")
systemd_install(
force=force, system=system, run_as_user=run_as_user,
enable_on_startup=start_on_login, non_interactive=non_interactive,
)
if start_now:
systemd_start(system=system)
def _cmd_install(args):
if is_managed():
managed_error("install gateway service")
return
if getattr(args, "if_missing", False) and _is_service_installed():
print("✓ Gateway service already installed")
return
force = getattr(args, "force", False)
# `--force` doubles as the reinstall flag here; a served profile's unit would only ever exit 78.
_guard_named_profile_under_multiplexer(force=force)
system = getattr(args, "system", False)
run_as_user = getattr(args, "run_as_user", None)
if _service_mgmt_blocked():
_no_backend_exit("install", "termux")
backend = _service_backend()
if backend == "systemd":
if refuses_container_user_scope_install(system):
sys.exit(1)
_install_systemd_from_cli(args, force=force, system=system, run_as_user=run_as_user)
elif backend == "launchd":
launchd_install(force, start_now=getattr(args, "start_now", None) is not False)
elif backend == "windows":
_gw_windows().install(
force=force,
start_now=getattr(args, 'start_now', None),
start_on_login=getattr(args, 'start_on_login', None),
elevated_handoff=getattr(args, 'elevated_handoff', False),
)
else:
_handle_no_backend("install", wsl=True, s6=True)
def _cmd_uninstall(args):
_refuse_from_inside_gateway("uninstall", "the gateway from terminating itself")
if is_managed():
managed_error("uninstall gateway service")
return
system = getattr(args, "system", False)
if _service_mgmt_blocked():
_no_backend_exit("uninstall", "termux")
backend = _service_backend()
if backend is not None:
_service_call(backend, "uninstall", system)
else:
_handle_no_backend("uninstall", wsl=False, s6=True)
def _host_multiplexer_for_all_verb():
"""The live host gateway a ``--all`` verb must target instead of sweeping every profile.
``--all`` means "the ONE host multiplexer", not "every gateway process on this box": sweeping
killed a multiplexer serving N profiles and started a single gateway in its place, so a
per-profile command caused a host-wide outage and left exactly one profile served.
"""
try:
from gateway.host_attach import host_gateway
return host_gateway()
except Exception:
logger.debug("Host gateway probe failed", exc_info=True)
return None
def _host_multiplexer_is_ours(owner) -> bool:
try:
from gateway.status import _get_process_hermes_home, _same_hermes_home
return bool(_same_hermes_home(owner.home, _get_process_hermes_home()))
except Exception:
return False
def _print_unfolded_gateway_note(owner) -> None:
"""Detect-and-converge: name the per-profile gateways `--all` no longer sweeps, never kill them."""
try:
others = [pid for pid in find_gateway_pids(all_profiles=True) if pid != owner.pid]
except Exception:
return
if not others:
return
print(f" {len(others)} per-profile gateway process(es) still run beside it "
f"(PIDs: {', '.join(str(p) for p in others)}).")
print(" They were left running; fold them in with: hermes gateway migrate --multiplex")
def _cmd_start(args):
from hermes_cli.gateway_profile_lifecycle import profile_lifecycle
if profile_lifecycle("start", args):
return
system = getattr(args, "system", False)
start_all = getattr(args, "all", False)
force = getattr(args, "force", False)
_guard_named_profile_under_multiplexer(force=force)
if not start_all and _dispatch_via_service_manager_if_s6("start"):
return
if start_all:
owner = None if force else _host_multiplexer_for_all_verb()
if owner is not None:
# Already up: `--all` has nothing to start, and the old sweep here SIGTERMed this very
# process before starting a single-profile replacement.
print(f"✓ The host gateway is already running — {owner.describe()}")
print(" One gateway per host serves every profile; nothing to start.")
_print_unfolded_gateway_note(owner)
return
killed = kill_gateway_processes(all_profiles=True)
if killed:
print(f"✓ Killed {killed} stale gateway process(es) across all profiles")
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
_wait_for_api_server_port_free()
if _service_mgmt_blocked():
_no_backend_exit("start", "termux")
backend = _service_backend()
if backend is not None:
_service_call(backend, "start", system)
else:
_handle_no_backend("start", wsl=True, s6=False)
def _cmd_stop(args):
_refuse_from_inside_gateway("stop", "restart loops")
from hermes_cli.gateway_profile_lifecycle import profile_lifecycle
if profile_lifecycle("stop", args):
return
stop_all = getattr(args, "all", False)
system = getattr(args, "system", False)
if not stop_all and not find_gateway_pids() and (
_served_by_another_host_gateway() or named_profile_served_by_running_multiplexer()):
# The launch/default-profile lifecycle still names the whole host.
owner = _served_by_another_host_gateway()
print_error(
f"The host gateway serves profile '{_current_profile_name()}' — there is no separate "
f"gateway for this profile to stop."
)
if owner is not None:
print(f" {owner.describe()}")
print(" Stop or restart the host gateway instead:")
print()
owner_flag = f"-p {owner.profile_label} " if owner is not None else ""
print(f" hermes {owner_flag}gateway stop # takes every served profile offline")
print(f" hermes {owner_flag}gateway restart")
sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
# Under s6 a bare pkill is seen as a crash and restarted; go through the supervisor.
if stop_all and _dispatch_all_via_service_manager_if_s6("stop"):
return
if not stop_all and _dispatch_via_service_manager_if_s6("stop"):
return
service_available = _stop_installed_service(system)
if stop_all:
total = kill_gateway_processes(all_profiles=True) + (1 if service_available else 0)
if total:
print(f"✓ Stopped {total} gateway process(es) across all profiles")
else:
print("✗ No gateway processes found")
elif not service_available:
if stop_profile_gateway():
print("✓ Stopped gateway for this profile")
else:
print("✗ No gateway running for this profile")
else:
print(f"✓ Stopped {get_service_name()} service")
def _stop_host_multiplexer(owner) -> int:
"""SIGTERM the host gateway and nothing else; returns how many processes were signalled."""
stopped = kill_gateway_processes()
if stopped:
return stopped
try:
terminate_pid(owner.pid, force=False)
return 1
except (ProcessLookupError, PermissionError, OSError):
return 0
def _discard_dead_host_record() -> bool:
"""Drop the host rendezvous record once its owner is provably gone (after a confirmed stop)."""
try:
from gateway.host_rendezvous import ROLE_GATEWAY, discard_dead_record
return discard_dead_record(ROLE_GATEWAY)
except Exception:
logger.debug("Host record retraction failed", exc_info=True)
return False
def _restart_all(system: bool) -> None:
owner = _host_multiplexer_for_all_verb()
if owner is not None and not _host_multiplexer_is_ours(owner):
# `--all` means "restart the ONE host multiplexer" — and this profile does not own it.
# Sweeping every profile here took the host process down and replaced it with a gateway
# serving only this profile.
print_error(f"The host gateway runs under profile '{owner.profile_label}'.")
print(f" {owner.describe()}")
print(" `--all` restarts the one host multiplexer, and this profile is not its owner.")
print()
print(f" hermes -p {owner.profile_label} gateway restart --all")
sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
service_stopped = _stop_installed_service(system)
if owner is not None:
# Stop ONLY the host process: its served set comes back with it, and any per-profile
# gateway a pre-migration host still runs is left alone (never auto-SIGTERMed).
stopped = _stop_host_multiplexer(owner)
total = stopped + (1 if service_stopped else 0)
if total:
print(f"✓ Stopped the host gateway (PID {owner.pid}; serves "
f"{', '.join(owner.profiles) or 'unknown'})")
_print_unfolded_gateway_note(owner)
else:
total = kill_gateway_processes(all_profiles=True) + (1 if service_stopped else 0)
if total:
print(f"✓ Stopped {total} gateway process(es) across all profiles")
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
_wait_for_api_server_port_free()
# Retract the stopped owner's rendezvous record. Leaving it made this very function a silent
# no-op: the re-entered `gateway run` below read the corpse's record and ATTACHED to it.
_discard_dead_host_record()
print("Starting gateway...")
# Even without a registered task, gateway_windows.start() uses the detached launcher.
kind = _installed_service_kind_for(is_windows)
if kind is None:
# replace=True: if the old owner is still draining (a long drain, an ineffective SIGKILL, a
# foreign-home owner the scan never saw), take the host over instead of attaching to it.
run_gateway(verbose=0, replace=True)
else:
_service_call(kind, "start", system)
def _cmd_restart(args):
_refuse_from_inside_gateway("restart", "restart loops")
from hermes_cli.gateway_profile_lifecycle import profile_lifecycle
if profile_lifecycle("restart", args):
return
system = getattr(args, "system", False)
restart_all = getattr(args, "all", False)
force = getattr(args, "force", False)
# `--all` targets the ONE host multiplexer and _restart_all does its own ownership check with
# the right one-liner; running the generic named-profile guard first made that branch
# unreachable for `-p X gateway restart --all` (it printed a bare `gateway restart` instead).
if not restart_all:
_guard_named_profile_under_multiplexer(force=force)
if restart_all and _dispatch_all_via_service_manager_if_s6("restart"):
return
if not restart_all and _dispatch_via_service_manager_if_s6("restart"):
return
if restart_all:
_restart_all(system)
return
# The Windows restart path handles both registered installs and detached restarts.
kind = _installed_service_kind_for(is_windows)
service_configured = kind is not None and (kind != "windows" or _gw_windows().is_installed())
if kind is not None:
swallow = (RuntimeError, OSError) if kind == "windows" else ()
try:
_service_call(kind, "restart", system)
return
except (subprocess.CalledProcessError, *swallow):
pass
# Linger only explains a FAILED systemd unit restart. Without an installed unit the
# detached run below is the restart; bailing here left `hermes gateway restart` a
# silent exit-0 no-op on any Linux login session (Desktop read it as success).
if kind == "systemd" and supports_systemd_services():
linger_ok, _detail = get_systemd_linger_status()
if linger_ok is not True:
import getpass
_print_lines(
"", "⚠ Cannot restart gateway as a service — linger is not enabled.",
" The gateway user service requires linger to function on headless servers.", "",
f" Run: sudo loginctl enable-linger {getpass.getuser()}", "",
" Then restart the gateway:", " hermes gateway restart",
)
return
if service_configured:
_print_lines(
"", "✗ Gateway service restart failed.",
" The service definition exists, but the service manager did not recover it.",
" Fix the service, then retry: hermes gateway start",
)
sys.exit(1)
# A gateway that declares an external supervisor (custom launchd agent / unit running
# `gateway run --external-supervisor`) restarts by exiting back to it: the stop + foreground
# run below would stamp this CLI's PID as the gateway and wedge every respawn (#110637).
from gateway.status import get_running_pid
from hermes_cli.gateway_supervised_restart import (
gateway_declares_external_supervisor, restart_externally_supervised_gateway,
)
supervised_pid = get_running_pid()
if supervised_pid and gateway_declares_external_supervisor(supervised_pid):
restart_externally_supervised_gateway(supervised_pid)
return
if stop_profile_gateway():
print("✓ Stopped gateway for this profile")
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
_wait_for_api_server_port_free()
print("Starting gateway...")
run_gateway(verbose=0, force=force)
# ``hermes gateway status`` hints for a manually-run / stopped gateway, keyed by host kind.
_STATUS_RUNNING_HINTS = {
"termux": ("Termux note:", " Android may stop background jobs when Termux is suspended"),
"wsl": (
"WSL note:", " The gateway is running in foreground/manual mode (recommended for WSL).",
" Use tmux or screen for persistence across terminal closes.",
),
"windows": ("To install as a Windows Scheduled Task (auto-start on login):", " hermes gateway install"),
"other": (
"To install as a service:", " hermes gateway install", " sudo hermes gateway install --system",
),
}
_STATUS_STOPPED_HINTS = {
"termux": (
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # Best-effort background start",
),
"wsl": (
" tmux new -s hermes 'hermes gateway run' # persistent via tmux",
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # background",
),
"windows": (" hermes gateway install # Install as Windows Scheduled Task (auto-start on login)",),
"other": (
" hermes gateway install # Install as user service",
" sudo hermes gateway install --system # Install as boot-time system service",
),
}
def _status_host_kind() -> str:
if is_termux():
return "termux"
if is_wsl():
return "wsl"
return "windows" if is_windows() else "other"
def _cmd_status(args):
from hermes_cli.gateway_profile_lifecycle import print_parked_status
if print_parked_status():
return
deep = getattr(args, "deep", False)
full = getattr(args, "full", False)
system = getattr(args, "system", False)
snapshot = get_gateway_runtime_snapshot(system=system)
from hermes_cli.profiles import get_active_profile_name, profile_is_standalone
active_standalone = ((get_active_profile_name() or "default") != "default"
and profile_is_standalone(get_hermes_home()))
if active_standalone:
from hermes_cli.gateway_multiplex_mode import STANDALONE_DEPRECATION_NOTICE
print("standalone by config (gateway.standalone: true) — temporary compatibility shim")
print(f" {STANDALONE_DEPRECATION_NOTICE}")
_windows_service_installed = is_windows() and _gw_windows().is_installed()
if not active_standalone and not snapshot.running and named_profile_served_by_running_multiplexer():
# Satellite profile: the default multiplexer is the live inbound process for it.
print("✓ Gateway is running via the default-profile multiplexer")
print(" Manage it from the default profile: hermes gateway status")
_print_served_ingress_urls(get_active_profile_name())
_print_unserved_shared_ingress(get_active_profile_name())
elif (kind := _installed_service_kind_for(lambda: _windows_service_installed)) is not None:
if kind == "systemd":
systemd_status(deep, system=system, full=full)
elif kind == "launchd":
launchd_status(deep)
else:
_gw_windows().status(deep=deep)
_print_gateway_process_mismatch(snapshot)
_print_multiplex_standalone_reason()
_print_served_ingress_urls()
else:
pids = list(snapshot.gateway_pids)
if pids:
print(f"✓ Gateway is running (PID: {', '.join(map(str, pids))})")
print(" (Running manually, not as a system service)")
_print_runtime_health()
_print_multiplex_standalone_reason()
_print_served_ingress_urls()
print()
_print_lines(*_STATUS_RUNNING_HINTS[_status_host_kind()])
else:
print("✗ Gateway is not running")
_print_runtime_health()
print()
print("To start:")
print(" hermes gateway run # Run in foreground")
_print_lines(*_STATUS_STOPPED_HINTS[_status_host_kind()])
_print_duplicate_credential_warnings()
_print_other_profiles_gateway_status()
_print_standalone_by_config()
def _print_standalone_by_config() -> None:
"""Default-profile status: name the profiles that opted out of the host multiplexer by config,
so the served set the host gateway reports is not mistaken for the installed roster."""
from hermes_cli.profiles import get_active_profile_name, profiles_to_serve
if (get_active_profile_name() or "default") != "default":
return
roster = {name for name, _home in profiles_to_serve(True, include_standalone=True)}
served = {name for name, _home in profiles_to_serve(True)}
names = sorted(roster - served - {"default"})
if names:
print(f"standalone by config (temporary compatibility shim): {', '.join(names)}")
def _cmd_list(args):
_gateway_list()
def _cmd_migrate_legacy(args):
"""Stop, disable, and remove legacy Hermes gateway unit files (e.g. hermes.service)."""
dry_run = getattr(args, "dry_run", False)
yes = getattr(args, "yes", False)
if not supports_systemd_services() and not is_macos():
print("Legacy unit migration only applies to systemd-based Linux hosts.")
return
remove_legacy_hermes_units(interactive=not yes, dry_run=dry_run)
def _cmd_migrate(args):
from hermes_cli.gateway_migrate import cmd_migrate
cmd_migrate(args)
_GATEWAY_SUBCOMMANDS = {
None: _cmd_run, "run": _cmd_run, "setup": _cmd_setup, "install": _cmd_install,
"uninstall": _cmd_uninstall, "start": _cmd_start, "stop": _cmd_stop, "restart": _cmd_restart,
"status": _cmd_status, "list": _cmd_list, "migrate-legacy": _cmd_migrate_legacy, "migrate": _cmd_migrate,
}
def _gateway_command_inner(args):
handler = _GATEWAY_SUBCOMMANDS.get(getattr(args, "gateway_command", None))
if handler is not None:
handler(args)
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----
# Names external plugins imported from this module before the Sep 2026 decomposition.
# Internal code MUST NOT use these (scripts/check_compat_pointers.py fails CI if it does).
# The whole block is removed by reverting the commit that added it.
def print_systemd_linger_guidance() -> None:
"""Print the current linger status and the fix when it is disabled."""
linger_enabled, linger_detail = get_systemd_linger_status()
if linger_enabled is True:
print("✓ Systemd linger is enabled (service survives logout)")
elif linger_enabled is False:
print("⚠ Systemd linger is disabled (gateway may stop when you log out)")
print(" Run: sudo loginctl enable-linger $USER")
else:
print(f"⚠ Could not verify systemd linger ({linger_detail})")
print(" If you want the gateway user service to survive logout, run:")
print(" sudo loginctl enable-linger $USER")
_PLUGIN_COMPAT_LAZY = {
'DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT': ('gateway.restart', 'DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT'),
}
def __getattr__(name): # PEP 562 — lazy so no import cycles
target = _PLUGIN_COMPAT_LAZY.get(name)
if target is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
import importlib
from hermes_cli.plugin_compat import warn_once
warn_once(__name__, name, *target)
return getattr(importlib.import_module(target[0]), target[1])
# ---- END PLUGIN-COMPAT ----
def _pm_runtime_venv_dir(project_root: Path | None = None) -> Path | None:
"""The venv pm provisioned for this install, resolved through
``runtime_paths.selected_venv`` — the committed-environment contract —
never from interpreter state.
Under no-boot-through-venv the gateway runs the store python with the
venv's site-packages on PYTHONPATH, so ``sys.prefix`` always equals
``sys.base_prefix`` and ``VIRTUAL_ENV`` is unset in bundled installs;
prefix/env probing silently degrades. Resolution follows the committed-
environment contract: a committed selection is returned as-is, a host
with nothing committed yields nothing launchable, and a malformed
selection raises — fail closed, never a silently wrong venv.
"""
root = Path(project_root) if project_root is not None else PROJECT_ROOT
from pm.environments import selected_venv
venv = selected_venv(root) # a malformed committed selection raises: fail closed
return venv if venv.is_dir() else None