The seventeen remaining literals are container-side paths, AF_UNIX socket-path-limit candidates on darwin, detection needles, guard regexes and guidance text that tells the model to avoid /tmp. Each carries an inline `no-tmp: ok — <why>` so the reason lives next to the line; the baseline keeps only a fenced tree listing where a marker would render.
6714 lines
306 KiB
Python
6714 lines
306 KiB
Python
"""Gateway subcommand for hermes CLI.
|
|
|
|
Handles: hermes gateway [run|start|stop|restart|status|install|uninstall|setup]
|
|
"""
|
|
|
|
import asyncio
|
|
import contextlib
|
|
from hermes_cli.cli_output import line_input
|
|
import json
|
|
import logging
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from hermes_cli import setup_platforms
|
|
|
|
# UV's bundled Python ships a minimal PATH; ensure launchctl/systemctl are discoverable.
|
|
if os.name == "posix":
|
|
_sys_dirs = {"/bin", "/usr/bin", "/usr/sbin", "/sbin"}
|
|
_path_dirs = set(os.environ.get("PATH", "").split(os.pathsep))
|
|
_missing = _sys_dirs - _path_dirs
|
|
if _missing:
|
|
os.environ["PATH"] = os.environ.get("PATH", "") + os.pathsep + os.pathsep.join(sorted(_missing))
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent.resolve()
|
|
|
|
from gateway.config import coerce_systemd_watchdog_seconds, load_gateway_config
|
|
from gateway.status import terminate_pid
|
|
from gateway.restart import (
|
|
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
|
|
EXTERNAL_GATEWAY_SUPERVISOR_ENV,
|
|
GATEWAY_FATAL_CONFIG_EXIT_CODE,
|
|
GATEWAY_SERVICE_RESTART_EXIT_CODE,
|
|
is_gateway_supervisor_process,
|
|
parse_cron_drain_timeout,
|
|
parse_restart_after_turn_timeout,
|
|
parse_restart_drain_timeout,
|
|
resolve_restart_exit_wait_budget,
|
|
resolve_systemd_timeout_stop_sec,
|
|
)
|
|
from hermes_cli.config import (
|
|
get_env_value,
|
|
get_hermes_home,
|
|
is_managed,
|
|
managed_error,
|
|
read_raw_config,
|
|
save_env_value,
|
|
write_platform_config_field,
|
|
)
|
|
|
|
# display_hermes_home is imported lazily: hermes_constants may be a cached pre-update version.
|
|
from hermes_cli.setup import (
|
|
print_header,
|
|
print_info,
|
|
print_success,
|
|
print_warning,
|
|
print_error,
|
|
prompt,
|
|
prompt_choice,
|
|
prompt_yes_no,
|
|
)
|
|
from hermes_cli.colors import Colors, color
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Shared ``subprocess.run`` kwargs for text-mode probes (stdout/stderr captured, decode-tolerant).
|
|
_CAPTURE_TEXT = dict(capture_output=True, text=True, encoding="utf-8", errors="replace")
|
|
|
|
# =============================================================================
|
|
# Process Management (for manual gateway runs)
|
|
# =============================================================================
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GatewayRuntimeSnapshot:
|
|
manager: str
|
|
service_installed: bool = False
|
|
service_running: bool = False
|
|
gateway_pids: tuple[int, ...] = ()
|
|
service_scope: str | None = None
|
|
|
|
@property
|
|
def running(self) -> bool:
|
|
return self.service_running or bool(self.gateway_pids)
|
|
|
|
@property
|
|
def has_process_service_mismatch(self) -> bool:
|
|
return self.service_installed and self.running and not self.service_running
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ProfileGatewayProcess:
|
|
profile: str
|
|
path: Path
|
|
pid: int
|
|
create_time: float = 0.0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WindowsGatewayService:
|
|
"""A real Windows service supervising a profile gateway process tree."""
|
|
|
|
name: str
|
|
profile: str
|
|
service_pid: int
|
|
gateway_pid: int
|
|
descendant_pids: frozenset[int]
|
|
descendant_identities: tuple[tuple[int, float], ...]
|
|
service_create_time: float = 0.0
|
|
gateway_create_time: float = 0.0
|
|
|
|
|
|
def _get_service_pids(all_profiles: bool = False) -> set:
|
|
"""PIDs managed by systemd/launchd gateway services (excluded from stale-process sweeps).
|
|
|
|
Relies on the service manager committing the new PID before the restart command returns.
|
|
``all_profiles`` widens the current profile's unit/label to the whole ``hermes-gateway*`` /
|
|
``ai.hermes.gateway*`` fleet so update/reaper never kill a sibling's service gateway as "manual".
|
|
|
|
``all_profiles`` widens the launchd branch to every installed ``ai.hermes.gateway*`` LaunchAgent — the
|
|
update path needs the whole fleet excluded from its sweep (#41403, #73626): sibling-profile launchd
|
|
gateways found by the (BSD-fixed) ps scan must not be misclassified as manual processes and killed.
|
|
Default-scope callers (``gateway status``, cron checks) keep seeing only the current profile's service;
|
|
the orphan reaper passes all_profiles=True for the same friendly-fire reason. The systemd branch mirrors
|
|
this: default scope filters to the current profile's exact unit name; ``all_profiles=True`` widens to
|
|
the ``hermes-gateway*`` fleet glob.
|
|
"""
|
|
pids: set = set()
|
|
|
|
# --- systemd (Linux): user and system scopes ---
|
|
if supports_systemd_services():
|
|
pattern = "hermes-gateway*" if all_profiles else get_service_name()
|
|
for scope_args in [["systemctl", "--user"], ["systemctl"]]:
|
|
try:
|
|
# Belt-and-suspenders for the EXCLUDE use case (#74075): a bare ``launchctl list`` prefix
|
|
# scan also catches ai.hermes.gateway* agents the label derivation can't map (renamed
|
|
# profiles, other installs sharing this user). Over-inclusion is safe here — these PIDs are
|
|
# only ever protected from the kill sweep, never targeted. Restart paths use the
|
|
# label-derived set only.
|
|
result = subprocess.run(
|
|
scope_args
|
|
+ ["list-units", pattern, "--plain", "--no-legend", "--no-pager"],
|
|
timeout=5,
|
|
**_CAPTURE_TEXT,
|
|
)
|
|
for line in result.stdout.strip().splitlines():
|
|
parts = line.split()
|
|
if not parts or not parts[0].endswith(".service"):
|
|
continue
|
|
svc = parts[0]
|
|
try:
|
|
show = subprocess.run(
|
|
scope_args + ["show", svc, "--property=MainPID", "--value"],
|
|
timeout=5,
|
|
**_CAPTURE_TEXT,
|
|
)
|
|
pid = int(show.stdout.strip())
|
|
if pid > 0:
|
|
pids.add(pid)
|
|
except (ValueError, subprocess.TimeoutExpired):
|
|
pass
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
pass
|
|
|
|
# --- launchd (macOS) ---
|
|
if is_macos():
|
|
labels = {get_launchd_label()}
|
|
if all_profiles:
|
|
# Whole fleet, mirroring the systemd ``hermes-gateway*`` glob above.
|
|
# Every gateway LaunchAgent, not just the invoking profile's — mirrors the systemd branch's
|
|
# ``hermes-gateway*`` pattern above. The update path restarts the whole fleet, and its
|
|
# stale-process sweep must not mistake a sibling service's fresh PID for a manual gateway it
|
|
# should kill (#41403).
|
|
labels.update(launchd_gateway_labels_for_install())
|
|
for label in sorted(labels):
|
|
try:
|
|
_domain, pid = _locate_launchd_gateway_service(label)
|
|
except subprocess.TimeoutExpired:
|
|
continue
|
|
if pid is not None and pid > 0:
|
|
pids.add(pid)
|
|
if all_profiles:
|
|
# Prefix scan also catches ai.hermes.gateway* agents the label derivation can't map
|
|
# (renamed profiles, other installs). Over-inclusion is safe: PIDs are only protected.
|
|
try:
|
|
result = subprocess.run(["launchctl", "list"], timeout=5, **_CAPTURE_TEXT)
|
|
if result.returncode == 0:
|
|
for line in result.stdout.strip().splitlines():
|
|
parts = line.split()
|
|
if len(parts) >= 3 and parts[-1].startswith("ai.hermes.gateway"):
|
|
try:
|
|
pid = int(parts[0])
|
|
if pid > 0:
|
|
pids.add(pid)
|
|
except ValueError:
|
|
pass
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
pass
|
|
|
|
return pids
|
|
|
|
|
|
def _get_parent_pid(pid: int) -> int | None:
|
|
"""Parent PID for ``pid``, or None. psutil first (works on Windows, where ``ps`` doesn't)."""
|
|
if pid <= 1:
|
|
return None
|
|
try:
|
|
import psutil # type: ignore
|
|
return psutil.Process(pid).ppid() or None
|
|
except ImportError:
|
|
pass
|
|
except Exception:
|
|
return None
|
|
# ps fallback, POSIX only: Git Bash's ps.exe would flash a console from the windowless backend.
|
|
if is_windows() or not shutil.which("ps"):
|
|
return None
|
|
try:
|
|
result = subprocess.run(["ps", "-o", "ppid=", "-p", str(pid)], timeout=5, **_CAPTURE_TEXT)
|
|
except (FileNotFoundError, subprocess.TimeoutExpired):
|
|
return None
|
|
raw = result.stdout.strip()
|
|
if result.returncode != 0 or not raw:
|
|
return None
|
|
try:
|
|
parent_pid = int(raw.splitlines()[-1].strip())
|
|
except ValueError:
|
|
return None
|
|
return parent_pid if parent_pid > 0 else None
|
|
|
|
|
|
def _is_pid_ancestor_of_current_process(target_pid: int) -> bool:
|
|
"""Return True when ``target_pid`` is this process or one of its ancestors."""
|
|
if target_pid <= 0:
|
|
return False
|
|
|
|
pid = os.getpid()
|
|
seen: set[int] = set()
|
|
while pid and pid not in seen:
|
|
if pid == target_pid:
|
|
return True
|
|
seen.add(pid)
|
|
pid = _get_parent_pid(pid) or 0
|
|
return False
|
|
|
|
|
|
def _request_gateway_self_restart(pid: int) -> bool:
|
|
"""Ask a running gateway ancestor to restart itself asynchronously."""
|
|
if not hasattr(signal, "SIGUSR1") or not _is_pid_ancestor_of_current_process(pid):
|
|
return False
|
|
try:
|
|
os.kill(pid, signal.SIGUSR1) # windows-footgun: ok — POSIX signal, guarded by hasattr(signal, 'SIGUSR1') above
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _graceful_restart_via_sigusr1(pid: int, drain_timeout: float, *, on_progress=None) -> bool:
|
|
"""SIGUSR1 (drain-aware restart) a gateway PID and wait for exit; False if unsent or it outlived the timeout.
|
|
|
|
gateway/run.py maps SIGUSR1 to ``request_restart(via_service=True)``: refuse new turns, drain,
|
|
``stop()``, exit; the supervisor relaunches. ``drain_timeout`` must cover after-turn wait + drain
|
|
— pass ``resolve_restart_exit_wait_budget(...)``. ``on_progress`` (zero-arg) runs on every poll so
|
|
a long wait can report what the gateway is still holding for (``update_cmd_drain_report``).
|
|
"""
|
|
if not hasattr(signal, "SIGUSR1") or pid <= 0:
|
|
return False
|
|
try:
|
|
os.kill(pid, signal.SIGUSR1) # windows-footgun: ok — POSIX signal, guarded by hasattr(signal, 'SIGUSR1') above
|
|
except ProcessLookupError:
|
|
return True
|
|
except (PermissionError, OSError):
|
|
return False
|
|
|
|
return _wait_for_pid_exit(pid, max(drain_timeout, 1.0), on_progress=on_progress)
|
|
|
|
|
|
def _wait_for_pid_exit(pid: int, timeout: float, *, on_progress=None) -> bool:
|
|
"""Wait up to ``timeout``s for ``pid`` to exit; True once gone. (``launchctl bootstrap`` fails EIO
|
|
while the previous instance still drains, so teardown callers must wait for the real exit.)"""
|
|
if pid <= 0:
|
|
return True
|
|
# ``os.kill(pid, 0)`` hard-kills on Windows (TerminateProcess); use _pid_exists instead.
|
|
from gateway.status import _pid_exists
|
|
deadline = time.monotonic() + max(timeout, 0.0)
|
|
while True:
|
|
if not _pid_exists(pid):
|
|
return True
|
|
if time.monotonic() >= deadline:
|
|
return False
|
|
if on_progress is not None:
|
|
on_progress()
|
|
time.sleep(0.5)
|
|
|
|
|
|
# --- Wedged-gateway detection + bounded escalation ---------------------------
|
|
# A gateway whose asyncio loop is stalled cannot handle SIGTERM/SIGUSR1, so the drain wait burns
|
|
# its full budget and `hermes update` can deadlock. Two witnesses classify the loop BEFORE any
|
|
# drain wait: the heartbeat file ``state/gateway.heartbeat`` (rewritten every 30s on a thread, so
|
|
# staleness alone is not proof) and the loop-tick socket ``state/gateway.loop-tick.<pid>.sock``
|
|
# answered by the loop itself; the payload records whether the socket is armed (``loop_tick_socket``).
|
|
# ``alive``: socket answered, or fresh file not contradicted -> normal graceful drain. ``wedged``:
|
|
# heartbeat is this PID's, stale past several beats, AND the armed socket stays silent across
|
|
# ``tick_strikes`` consecutive misses -> callers may ``_escalate_wedged_gateway``; one silent probe
|
|
# is never authority. ``unknown``: no/unreadable heartbeat, PID mismatch, or witness conflict ->
|
|
# treated as alive; never escalate on ambiguity. Legacy payloads (no ``loop_tick_socket`` flag)
|
|
# wrote on-loop, so staleness alone remains proof.
|
|
|
|
# --- Wedged-gateway detection + bounded escalation (#81642) ----------------- A gateway whose asyncio loop
|
|
# is stalled (e.g. an in-loop compression pass, #72707) cannot process SIGTERM/SIGUSR1 shutdown: the drain
|
|
# wait then burns the full drain budget (180s by default), warns "still running after 180.0s — restart may
|
|
# fail", and `hermes update` can deadlock behind it. The loop publishes a liveness signal precisely for this
|
|
# case: an asyncio task rewrites ``state/gateway.heartbeat`` every 30s (#66892), so a frozen loop stops
|
|
# refreshing the file while a busy-but-alive loop keeps refreshing it. Since #90502 the heartbeat write runs
|
|
# on a thread (a stalling filesystem must not be able to block the loop the watchdog watches), which costs
|
|
# the file its status as *proof*: a stalled write or a saturated executor can age the file while the loop
|
|
# runs, and an off-loop write can land after the loop froze, keeping the file fresh for a dead loop. The
|
|
# loop therefore also arms a second witness — ``state/gateway.loop-tick.<pid>.sock``, a UNIX socket answered
|
|
# by the loop itself — and records whether it is armed in the heartbeat payload (``loop_tick_socket``).
|
|
# ``probe_gateway_loop_liveness`` reads both signals (a local stat + JSON read + a bounded socket ping,
|
|
# repeated up to ``tick_strikes`` times when a wedge is suspected — worst case ~3.4s, still far inside the
|
|
# 10s query tier of the subprocess timeout doc) and classifies the gateway BEFORE any drain wait begins: -
|
|
# ``alive`` — the loop answered the tick socket, or the file is fresh and the loop is not contradicted by
|
|
# the socket. Callers must take the normal graceful-drain path, which honours the in-flight cron drain floor
|
|
# (#86684). - ``wedged`` — the heartbeat belongs to this PID, is stale well past several missed beats, AND
|
|
# the tick socket is armed but stays silent across a sustained window of consecutive misses (default 3):
|
|
# both witnesses agree, sustained, that the loop is provably dead. One silent probe is never destructive
|
|
# authority — a transient synchronous stall can outlast a single recv timeout, so a lone miss falls to
|
|
# ``unknown``. Draining is pointless for a provably dead loop (nothing can run the drain), so callers may
|
|
# escalate immediately via ``_escalate_wedged_gateway``. - ``unknown`` — no heartbeat / unreadable / PID
|
|
# mismatch / witness conflict (fresh file with a silent loop, armed socket unreachable). Treated like
|
|
# ``alive``: never escalate on ambiguity. The distinction matters: only a *provably dead* loop may bypass
|
|
# the cron drain floor. A merely busy gateway still answers the probe (socket ping) and keeps its full drain
|
|
# budget — even when the filesystem is stalling the heartbeat write (the incident that motivated #90502).
|
|
# Legacy gateways (no ``loop_tick_socket`` flag in the payload) wrote the file on-loop, so their staleness
|
|
# remains proof and the old single-witness contract is unchanged.
|
|
GATEWAY_LOOP_ALIVE = "alive"
|
|
GATEWAY_LOOP_WEDGED = "wedged"
|
|
GATEWAY_LOOP_UNKNOWN = "unknown"
|
|
|
|
# 3 missed 30s beats (gateway.shutdown_watchdog.DEFAULT_HEARTBEAT_INTERVAL_S): decisive, not one slow write.
|
|
DEFAULT_LOOP_LIVENESS_STALE_AFTER_S = 90.0
|
|
|
|
# Sentinel for "the producer never wrote the witness flag" (legacy payload).
|
|
_LOOP_TICK_ABSENT = object()
|
|
|
|
|
|
def _probe_loop_tick_socket(pid: int, home: Path | None, timeout: float = 1.0) -> bool | None:
|
|
"""Ping the loop-tick witness socket: True answered, False node present but silent, None no node (not evidence)."""
|
|
try:
|
|
from gateway.shutdown_watchdog import get_loop_tick_socket_path
|
|
path = get_loop_tick_socket_path(home, pid)
|
|
if not path.is_socket():
|
|
return None
|
|
except Exception:
|
|
return None
|
|
return _ping_loop_tick_witness(socket.AF_UNIX, str(path), timeout)
|
|
|
|
|
|
def _ping_loop_tick_witness(family: int, address, timeout: float) -> bool:
|
|
"""Connect to a loop-tick witness and expect one byte ``"1"``; False on refusal/timeout/any error."""
|
|
sock = None
|
|
try:
|
|
sock = socket.socket(family, socket.SOCK_STREAM)
|
|
sock.settimeout(max(float(timeout), 0.0))
|
|
sock.connect(address)
|
|
return sock.recv(1) == b"1"
|
|
except Exception:
|
|
return False
|
|
finally:
|
|
if sock is not None:
|
|
with contextlib.suppress(Exception):
|
|
sock.close()
|
|
|
|
|
|
def _probe_loop_tick_tcp(port: int, timeout: float = 1.0) -> bool | None:
|
|
"""TCP-loopback variant of the tick probe for Windows (no AF_UNIX in asyncio); same semantics, None
|
|
on invalid port."""
|
|
try:
|
|
port_num = int(port)
|
|
if port_num <= 0 or port_num > 65535:
|
|
return None
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return _ping_loop_tick_witness(socket.AF_INET, ("127.0.0.1", port_num), timeout)
|
|
|
|
|
|
def _probe_loop_tick_socket_sustained(
|
|
pid: int, home: Path | None, *, timeout: float = 1.0, strikes: int = 3, gap_s: float = 0.2,
|
|
tcp_port: int | None = None,
|
|
) -> bool | None:
|
|
"""Probe the tick socket up to ``strikes`` times, ``gap_s`` apart: True once answered, False if a node
|
|
stayed silent the whole window, None if the node vanished (not evidence). One silent probe is not
|
|
destructive evidence — a transient synchronous stall can outlast one recv timeout.
|
|
|
|
A single silent probe is NOT destructive evidence: the loop may be in a short transient synchronous
|
|
stall (a reconnect storm, a heavy synchronous callback, scheduler delay) that outlasts one recv timeout.
|
|
Killing a gateway on that would be a false wedge — the exact class of false positive #90502 exists to
|
|
prevent. Destructive authority therefore requires the loop to fail to answer across a bounded window of
|
|
``strikes`` consecutive misses, ``gap_s`` apart; any answer inside the window proves the loop is
|
|
dispatching and returns ``True``.
|
|
"""
|
|
total = max(int(strikes), 0)
|
|
for attempt in range(total):
|
|
if tcp_port is not None:
|
|
result = _probe_loop_tick_tcp(tcp_port, timeout=timeout)
|
|
else:
|
|
result = _probe_loop_tick_socket(pid, home, timeout=timeout)
|
|
if result is True:
|
|
return True
|
|
if result is None:
|
|
# No node: ambiguity, never a wedge — absence is not a miss.
|
|
return None
|
|
if attempt < total - 1 and gap_s > 0:
|
|
time.sleep(gap_s)
|
|
return False
|
|
|
|
|
|
def probe_gateway_loop_liveness(
|
|
pid: int, *, stale_after: float = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S, home: Path | None = None,
|
|
tick_timeout: float = 1.0, tick_strikes: int = 3, tick_gap_s: float = 0.2,
|
|
) -> str:
|
|
"""Classify a gateway PID's event loop as alive / wedged / unknown (see block comment above).
|
|
Stale heartbeat is ``wedged`` only when the payload declares the tick socket armed AND it stays
|
|
silent across ``tick_strikes`` misses; any answer is ``alive``; ambiguity is ``unknown``.
|
|
|
|
- the loop-tick socket (``state/gateway.loop-tick.<pid>.sock``): answered by the gateway loop itself, so
|
|
a reply is direct proof that the loop is dispatching. It is never refreshed by the heartbeat executor
|
|
thread and never stalled by a filesystem that is slow to fsync. - the heartbeat file
|
|
(``state/gateway.heartbeat``): rewritten every 30s on a thread since #90502, so freshness alone is no
|
|
longer proof of loop schedulability — a stalled write (measured at 112.6s max on the incident box) or a
|
|
saturated executor can age the file while the loop runs, and a write can land after the loop froze.
|
|
"""
|
|
try:
|
|
stale_budget = max(float(stale_after), 0.0)
|
|
except (TypeError, ValueError):
|
|
stale_budget = DEFAULT_LOOP_LIVENESS_STALE_AFTER_S
|
|
try:
|
|
from gateway.shutdown_watchdog import get_loop_heartbeat_path
|
|
path = get_loop_heartbeat_path(home)
|
|
mtime = path.stat().st_mtime
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
heartbeat_pid = int(payload.get("pid", 0))
|
|
except Exception:
|
|
return GATEWAY_LOOP_UNKNOWN
|
|
if heartbeat_pid <= 0 or int(pid) <= 0 or heartbeat_pid != int(pid):
|
|
# Heartbeat is not this process's (old version, starting up, stale file): not evidence.
|
|
return GATEWAY_LOOP_UNKNOWN
|
|
|
|
# TCP loopback witness (Windows) takes priority when published; else the AF_UNIX socket.
|
|
tcp_port = payload.get("loop_tick_tcp_port")
|
|
try:
|
|
tcp_port_int = int(tcp_port) if tcp_port is not None else None
|
|
except (TypeError, ValueError):
|
|
tcp_port_int = None
|
|
|
|
if tcp_port_int is not None and tcp_port_int > 0:
|
|
witness = _probe_loop_tick_tcp(tcp_port_int, timeout=tick_timeout)
|
|
tick_armed = True
|
|
else:
|
|
witness = _probe_loop_tick_socket(pid, home, timeout=tick_timeout)
|
|
tick_armed = payload.get("loop_tick_socket", _LOOP_TICK_ABSENT)
|
|
if witness is True:
|
|
# Loop answered: a stale file is a stalled write, not a wedge.
|
|
return GATEWAY_LOOP_ALIVE
|
|
# The loop answered a ping — it is dispatching right now. See #90502.
|
|
age = time.time() - mtime
|
|
if age <= stale_budget:
|
|
if witness is False:
|
|
# Fresh file but silent loop: an off-loop write can land after the loop froze.
|
|
return GATEWAY_LOOP_UNKNOWN
|
|
return GATEWAY_LOOP_ALIVE
|
|
|
|
# Stale past the budget; the verdict depends on what the producer promised about its witness.
|
|
if tick_armed is _LOOP_TICK_ABSENT:
|
|
# Legacy on-loop writer: staleness proves the loop stopped scheduling.
|
|
return GATEWAY_LOOP_WEDGED
|
|
if tick_armed is not True:
|
|
# Witness could not be armed (bind failed); off-loop write means staleness is not proof.
|
|
return GATEWAY_LOOP_UNKNOWN
|
|
if witness is False:
|
|
# First miss. The probe above is miss #1, so ``tick_strikes - 1`` more attempts follow.
|
|
# One silent probe is NOT destructive authority: a short transient synchronous stall can outlast a
|
|
# single recv timeout, and killing a live gateway on it would be the exact false wedge #90502 exists
|
|
# to prevent.
|
|
sustained = _probe_loop_tick_socket_sustained(
|
|
pid, home, timeout=tick_timeout, strikes=tick_strikes - 1, gap_s=tick_gap_s, tcp_port=tcp_port_int
|
|
)
|
|
if sustained is False:
|
|
return GATEWAY_LOOP_WEDGED
|
|
if sustained is True:
|
|
return GATEWAY_LOOP_ALIVE # Transient stall, not a wedge.
|
|
return GATEWAY_LOOP_UNKNOWN # Witness vanished mid-window: ambiguity — never kill on it.
|
|
return GATEWAY_LOOP_UNKNOWN # Armed but unreachable socket: ambiguity — never kill on it.
|
|
|
|
|
|
def _escalate_wedged_gateway(pid: int, *, term_grace: float = 5.0, kill_wait: float = 5.0) -> bool:
|
|
"""Bounded stop (SIGTERM, ``term_grace``, SIGKILL, ``kill_wait``) for a provably dead loop; True once gone.
|
|
Callers MUST have classified ``GATEWAY_LOOP_WEDGED`` first: escalating a merely busy gateway
|
|
bypasses the cron drain floor and SIGKILLs live work.
|
|
|
|
See #86684.
|
|
"""
|
|
from gateway.status import get_process_start_time
|
|
expected_start_time = get_process_start_time(pid)
|
|
try:
|
|
terminate_pid(pid, force=False)
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
return _wait_for_pid_exit(pid, 1.0)
|
|
if _wait_for_pid_exit(pid, max(float(term_grace), 0.0)):
|
|
return True
|
|
try:
|
|
terminate_pid(pid, force=True, expected_start_time=expected_start_time)
|
|
print(f"⚠ Gateway PID {pid} unresponsive to SIGTERM; sent SIGKILL")
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
pass
|
|
return _wait_for_pid_exit(pid, max(float(kill_wait), 0.0))
|
|
|
|
|
|
def _get_ancestor_pids() -> set[int]:
|
|
"""PIDs of this process and its ancestors, so scans never count the invoking ``hermes`` CLI as a gateway.
|
|
|
|
Walks from the current PID up to PID 1 (init) so that process-table scans never match the calling CLI
|
|
process or any of its parents. This prevents ``hermes gateway status`` from falsely counting the
|
|
``hermes`` CLI that invoked it as a running gateway instance (see #13242).
|
|
"""
|
|
ancestors: set[int] = set()
|
|
pid = os.getpid()
|
|
for _ in range(64):
|
|
ancestors.add(pid)
|
|
parent = _get_parent_pid(pid)
|
|
if parent is None or parent <= 0 or parent in ancestors:
|
|
break
|
|
pid = parent
|
|
return ancestors
|
|
|
|
|
|
def _append_unique_pid(pids: list[int], pid: int | None, exclude_pids: set[int]) -> None:
|
|
if pid and pid > 0 and pid != os.getpid() and pid not in exclude_pids and pid not in pids:
|
|
pids.append(pid)
|
|
|
|
|
|
def _iter_proc_cmdlines(exclude_pids: set[int]):
|
|
"""Yield ``(pid, cmdline)`` from ``/proc`` (Docker without procps); raises if /proc is unusable."""
|
|
my_pid = os.getpid()
|
|
for entry in os.listdir("/proc"):
|
|
if not entry.isdigit():
|
|
continue
|
|
pid = int(entry)
|
|
if pid == my_pid or pid in exclude_pids:
|
|
continue
|
|
try:
|
|
with open(f"/proc/{pid}/cmdline", "rb") as _f:
|
|
cmdline = _f.read().decode("utf-8", errors="replace")
|
|
except (OSError, PermissionError):
|
|
continue
|
|
yield pid, cmdline.replace("\x00", " ")
|
|
|
|
|
|
def _scan_gateway_pids(
|
|
exclude_pids: set[int], all_profiles: bool = False, include_restart_managers: bool = False
|
|
) -> list[int]:
|
|
"""Best-effort process-table scan for gateway PIDs (backs up a stale/missing PID file; ``--all`` sweeps)."""
|
|
# Exclude the entire ancestor chain so the CLI process that invoked this scan (e.g. ``hermes gateway
|
|
# status``) is never mistaken for a running gateway. See #13242.
|
|
exclude_pids = exclude_pids | _get_ancestor_pids()
|
|
pids: list[int] = []
|
|
# Strict matcher shared with gateway.status: requires a real ``gateway run`` argv, so
|
|
# ``gateway status``/``dashboard`` siblings and ``python -m tui_gateway`` don't match.
|
|
from gateway.status import (
|
|
looks_like_gateway_command_line, looks_like_gateway_runtime_command_line, profile_flag_value,
|
|
)
|
|
current_home = str(get_hermes_home().resolve())
|
|
# Forward slashes on both sides of the HERMES_HOME= match (mirrors gateway.status).
|
|
current_home_lc = current_home.lower().replace("\\", "/")
|
|
current_profile_arg = _profile_arg(current_home)
|
|
current_profile_name = current_profile_arg.split()[-1] if current_profile_arg else ""
|
|
current_profile_name_lc = current_profile_name.lower()
|
|
|
|
def _matches_current_profile(command: str) -> bool:
|
|
command_lc = command.lower().replace("\\", "/")
|
|
if current_profile_name:
|
|
# Token equality, not substring: `-p ops` must not claim (or SIGTERM) an `-p ops-2` gateway.
|
|
return (
|
|
profile_flag_value(command_lc) == current_profile_name_lc
|
|
or f"hermes_home={current_home_lc}" in command_lc
|
|
)
|
|
|
|
# Default profile: accept unless argv advertises another profile in any spelling the CLI
|
|
# pre-parser accepts (``--profile=ops`` slipped past a substring test, so a default-profile
|
|
# fallback stop could SIGTERM the named gateway). HERMES_HOME may come via env (invisible to
|
|
# wmic/CIM), so only a non-matching explicit HERMES_HOME= disqualifies.
|
|
if profile_flag_value(command_lc) is not None:
|
|
return False
|
|
return not ("hermes_home=" in command_lc and f"hermes_home={current_home_lc}" not in command_lc)
|
|
|
|
def _consider(pid: int, command: str) -> None:
|
|
matches_runtime = looks_like_gateway_command_line(command) or (
|
|
include_restart_managers and looks_like_gateway_runtime_command_line(command)
|
|
)
|
|
if matches_runtime and (all_profiles or _matches_current_profile(command)):
|
|
_append_unique_pid(pids, pid, exclude_pids)
|
|
|
|
try:
|
|
if is_windows():
|
|
listing = _windows_process_listing()
|
|
if listing is None:
|
|
return []
|
|
for pid, command in _iter_windows_list_processes(listing):
|
|
_consider(pid, command)
|
|
else:
|
|
# /proc first (Docker without procps), then `ps -Aww`.
|
|
_found_via_proc = False
|
|
if os.path.isdir("/proc"):
|
|
try:
|
|
for pid, command in _iter_proc_cmdlines(exclude_pids):
|
|
_consider(pid, command)
|
|
_found_via_proc = True
|
|
except Exception:
|
|
pass
|
|
|
|
if not _found_via_proc:
|
|
# ``-Aww`` not ``-A eww``: BSD/macOS ps rejects ``e``; ``-ww`` = unlimited width.
|
|
result = subprocess.run(["ps", "-Aww", "-o", "pid=,command="], timeout=10, **_CAPTURE_TEXT)
|
|
if result.returncode != 0:
|
|
return []
|
|
for line in result.stdout.split("\n"):
|
|
parsed = _parse_ps_line(line)
|
|
if parsed is not None:
|
|
_consider(*parsed)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return []
|
|
|
|
# Windows: a venv ``pythonw.exe`` is a launcher stub that spawns the base Python with the same
|
|
# command line, so each gateway yields two matched PIDs. Drop a matched PID that parents another.
|
|
if is_windows() and len(pids) > 1:
|
|
pids = _filter_venv_launcher_stubs(pids)
|
|
|
|
return pids
|
|
|
|
|
|
def _parse_ps_line(line: str) -> tuple[int, str] | None:
|
|
"""``(pid, command)`` from one ``ps -o pid=,command=`` line; also accepts ``ps aux`` rows."""
|
|
stripped = line.strip()
|
|
if not stripped or "grep" in stripped:
|
|
return None
|
|
parts = stripped.split(None, 1)
|
|
if len(parts) == 2:
|
|
with contextlib.suppress(ValueError):
|
|
return int(parts[0]), parts[1]
|
|
aux_parts = stripped.split()
|
|
if len(aux_parts) > 10 and aux_parts[1].isdigit():
|
|
return int(aux_parts[1]), " ".join(aux_parts[10:])
|
|
return None
|
|
|
|
|
|
def _iter_windows_list_processes(listing: str):
|
|
"""Yield ``(pid, command_line)`` from wmic/CIM ``/FORMAT:LIST`` output."""
|
|
current_cmd = ""
|
|
for line in listing.split("\n"):
|
|
line = line.strip()
|
|
if line.startswith("CommandLine="):
|
|
current_cmd = line[len("CommandLine=") :]
|
|
elif line.startswith("ProcessId="):
|
|
with contextlib.suppress(ValueError):
|
|
yield int(line[len("ProcessId=") :]), current_cmd
|
|
current_cmd = ""
|
|
|
|
|
|
def _windows_process_listing() -> str | None:
|
|
"""``CommandLine=``/``ProcessId=`` LIST output for every Windows process (wmic, else Get-CimInstance), or None.
|
|
``bounded_probe_run``, NOT ``subprocess.run(timeout=...)``: run()'s post-timeout cleanup joins pipe
|
|
readers unbounded and a conhost.exe holding duplicated handles wedges the caller forever; it also
|
|
hides the console window this windowless pythonw backend would flash."""
|
|
# Prefer wmic when present (fast, stable output format). On modern Windows 11 / Win 10 late builds, wmic
|
|
# has been removed as part of the WMIC deprecation — fall back to PowerShell's Get-CimInstance. A spawn
|
|
# failure or timeout (result is None) trips the fallback. ``hermes update`` hung exactly there on
|
|
# slow-WMI machines where the full Win32_Process scan exceeds its budget (#87134). bounded_probe_run
|
|
# also hides the console window: this scan runs inside the windowless pythonw.exe gateway/desktop
|
|
# backend, so a bare wmic/powershell spawn would flash a conhost window on every watchdog probe.
|
|
from hermes_cli._subprocess_compat import bounded_probe_run
|
|
wmic_path = shutil.which("wmic")
|
|
result = None
|
|
if wmic_path is not None:
|
|
result = bounded_probe_run(
|
|
[wmic_path, "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"], timeout=10, errors="ignore"
|
|
)
|
|
if result is None or result.returncode != 0 or not (result.stdout or ""):
|
|
powershell = shutil.which("powershell") or shutil.which("pwsh")
|
|
if powershell is None:
|
|
return None
|
|
ps_cmd = (
|
|
"Get-CimInstance Win32_Process | "
|
|
"ForEach-Object { "
|
|
" 'CommandLine=' + ($_.CommandLine -replace \"`r`n\",' ' -replace \"`n\",' '); "
|
|
" 'ProcessId=' + $_.ProcessId; "
|
|
" '' "
|
|
"}"
|
|
)
|
|
result = bounded_probe_run([powershell, "-NoProfile", "-Command", ps_cmd], timeout=15, errors="ignore")
|
|
if result is None:
|
|
return None
|
|
return None if result.returncode != 0 or result.stdout is None else result.stdout
|
|
|
|
|
|
def _filter_venv_launcher_stubs(pids: list[int]) -> list[int]:
|
|
"""Drop venv-launcher ``pythonw.exe`` stubs that parent another matched PID (see ``_scan_gateway_pids``)."""
|
|
try:
|
|
import psutil # type: ignore
|
|
except ImportError:
|
|
return pids
|
|
|
|
pid_set = set(pids)
|
|
drop: set[int] = set()
|
|
for pid in pids:
|
|
try:
|
|
ppid = psutil.Process(pid).ppid()
|
|
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
|
continue
|
|
if ppid is not None and ppid in pid_set:
|
|
drop.add(ppid)
|
|
return [p for p in pids if p not in drop]
|
|
|
|
|
|
def find_gateway_pids(exclude_pids: set | None = None, all_profiles: bool = False) -> list:
|
|
"""Find running gateway PIDs for the current profile, or every profile with ``all_profiles`` (``hermes update``)."""
|
|
_exclude = set(exclude_pids or set())
|
|
pids: list[int] = []
|
|
if not all_profiles:
|
|
try:
|
|
from gateway.status import get_running_pid
|
|
_append_unique_pid(pids, get_running_pid(), _exclude)
|
|
except Exception:
|
|
pass
|
|
for pid in _get_service_pids(all_profiles=all_profiles):
|
|
_append_unique_pid(pids, pid, _exclude)
|
|
try:
|
|
include_restart_managers = not supports_systemd_services()
|
|
except Exception:
|
|
include_restart_managers = False
|
|
for pid in _scan_gateway_pids(_exclude, all_profiles=all_profiles, include_restart_managers=include_restart_managers):
|
|
_append_unique_pid(pids, pid, _exclude)
|
|
return pids
|
|
|
|
|
|
def find_profile_gateway_processes(exclude_pids: set | None = None, *, strict: bool = False) -> list[ProfileGatewayProcess]:
|
|
"""Return running gateway PIDs mapped to Hermes profiles via PID files."""
|
|
_exclude = set(exclude_pids or set())
|
|
processes: list[ProfileGatewayProcess] = []
|
|
try:
|
|
from gateway.status import get_running_pid, get_running_pid_identity_strict
|
|
from hermes_cli.profiles import list_profiles
|
|
except Exception:
|
|
if strict:
|
|
raise
|
|
return processes
|
|
|
|
seen: set[int] = set()
|
|
try:
|
|
profiles = list_profiles()
|
|
except Exception:
|
|
if strict:
|
|
raise
|
|
return processes
|
|
for profile in profiles:
|
|
try:
|
|
if strict:
|
|
identity = get_running_pid_identity_strict(profile.path / "gateway.pid")
|
|
pid = identity[0] if identity else None
|
|
create_time = identity[1] if identity else 0.0
|
|
else:
|
|
pid = get_running_pid(profile.path / "gateway.pid", cleanup_stale=False)
|
|
create_time = 0.0
|
|
except Exception as exc:
|
|
if strict:
|
|
raise RuntimeError(f"Could not inspect gateway PID for profile {profile.name}") from exc
|
|
continue
|
|
if pid is None or pid <= 0 or pid in _exclude or pid in seen:
|
|
continue
|
|
seen.add(pid)
|
|
processes.append(ProfileGatewayProcess(profile=profile.name, path=profile.path, pid=pid, create_time=create_time))
|
|
return processes
|
|
|
|
|
|
def _scm_service_field(service, field: str):
|
|
"""psutil ``WindowsService`` exposes getters as methods; ``as_dict()`` covers objects without them."""
|
|
getter = getattr(service, field, None)
|
|
return getter() if callable(getter) else service.as_dict().get(field)
|
|
|
|
|
|
def find_windows_gateway_services(
|
|
*, psutil_module=None, profile_processes: list[ProfileGatewayProcess] | None = None
|
|
) -> list[WindowsGatewayService]:
|
|
"""Profile gateways supervised by real, Hermes-owned Windows services. Service-logon processes may
|
|
hide their command lines, so identity = Hermes's own PID file + a parent chain ending at a running
|
|
SCM service PID whose name or binary path is Hermes's (``gateway_windows.hermes_owns_windows_service``).
|
|
The whole service subtree is returned so the Desktop preflight exempts exactly what the updater stops
|
|
through the SCM; a gateway under any other service (a Scheduled Task's svchost) is a plain process."""
|
|
if sys.platform != "win32":
|
|
return []
|
|
try:
|
|
if psutil_module is None:
|
|
import psutil as psutil_module # type: ignore[no-redef] # noqa: PLC0415
|
|
if profile_processes is None:
|
|
profile_processes = find_profile_gateway_processes(strict=True)
|
|
from hermes_cli.gateway_windows import hermes_owns_windows_service, hermes_service_roots
|
|
|
|
hermes_roots = hermes_service_roots()
|
|
service_names_by_pid: dict[int, set[str]] = {}
|
|
indeterminate_services_by_pid: dict[int, list[tuple[str, object]]] = {}
|
|
for service in psutil_module.win_service_iter():
|
|
try:
|
|
service_name = str(_scm_service_field(service, "name") or "")
|
|
if not service_name:
|
|
raise RuntimeError("SCM service has an empty name")
|
|
# Ownership before state: an OS service above the gateway (Task Scheduler's svchost for a
|
|
# task-launched gateway, BITS mid-transition) is never its supervisor, so neither its
|
|
# PID nor its status may steer the pause. Only Hermes-owned services reach the guards below.
|
|
# The name alone settles Hermes-named services; binpath (QueryServiceConfig) is asked only
|
|
# for the rest, and a service that refuses even that to this user is one this user could
|
|
# not `sc stop` either — never Hermes's, never a reason to abort the enumeration.
|
|
owned = hermes_owns_windows_service(service_name, "", hermes_roots)
|
|
if not owned:
|
|
try:
|
|
service_binpath = str(_scm_service_field(service, "binpath") or "")
|
|
except psutil_module.AccessDenied:
|
|
continue
|
|
owned = hermes_owns_windows_service(service_name, service_binpath, hermes_roots)
|
|
if not owned:
|
|
continue
|
|
service_status = _scm_service_field(service, "status")
|
|
service_pid = int(_scm_service_field(service, "pid") or 0)
|
|
except FileNotFoundError:
|
|
# Deleted between enumeration and inspection.
|
|
continue
|
|
except Exception as exc:
|
|
raise RuntimeError("SCM service inspection failed") from exc
|
|
if service_status == "stopped":
|
|
continue
|
|
if service_status != "running":
|
|
if service_pid > 0:
|
|
indeterminate_services_by_pid.setdefault(service_pid, []).append((service_name, service_status))
|
|
continue
|
|
if service_pid <= 0:
|
|
raise RuntimeError(f"Running SCM service {service_name} has no valid process ID")
|
|
service_names_by_pid.setdefault(service_pid, set()).add(service_name)
|
|
except Exception as exc:
|
|
raise RuntimeError("SCM service enumeration failed") from exc
|
|
|
|
found: dict[str, WindowsGatewayService] = {}
|
|
for profile_process in profile_processes:
|
|
try:
|
|
gateway_process = psutil_module.Process(int(profile_process.pid))
|
|
gateway_create_time = float(gateway_process.create_time())
|
|
if profile_process.create_time <= 0 or abs(gateway_create_time - profile_process.create_time) > 0.001:
|
|
raise RuntimeError("Gateway process identity changed during SCM discovery")
|
|
ancestor_pids = [int(parent.pid) for parent in gateway_process.parents()]
|
|
for pid in ancestor_pids:
|
|
indeterminate_services = indeterminate_services_by_pid.get(pid, [])
|
|
if indeterminate_services:
|
|
service_name, service_status = indeterminate_services[0]
|
|
raise RuntimeError(f"SCM service {service_name} has indeterminate status: {service_status}")
|
|
shared_service_pids = [pid for pid in ancestor_pids if len(service_names_by_pid.get(pid, set())) > 1]
|
|
if shared_service_pids:
|
|
raise RuntimeError(
|
|
"Gateway ownership is ambiguous under shared SCM host PID(s): "
|
|
+ ", ".join(str(pid) for pid in shared_service_pids)
|
|
)
|
|
service_pid = next((pid for pid in ancestor_pids if len(service_names_by_pid.get(pid, set())) == 1), None)
|
|
if service_pid is None:
|
|
continue
|
|
service_name = next(iter(service_names_by_pid[service_pid]))
|
|
service_process = psutil_module.Process(service_pid)
|
|
service_create_time = float(service_process.create_time())
|
|
descendant_processes = service_process.children(recursive=True)
|
|
descendants = frozenset(int(child.pid) for child in descendant_processes)
|
|
if int(profile_process.pid) not in descendants:
|
|
continue
|
|
descendant_identities = tuple(
|
|
sorted((int(child.pid), float(child.create_time())) for child in descendant_processes)
|
|
)
|
|
found[service_name] = WindowsGatewayService(
|
|
name=service_name,
|
|
profile=str(profile_process.profile),
|
|
service_pid=service_pid,
|
|
gateway_pid=int(profile_process.pid),
|
|
descendant_pids=descendants,
|
|
descendant_identities=descendant_identities,
|
|
service_create_time=service_create_time,
|
|
gateway_create_time=gateway_create_time,
|
|
)
|
|
except RuntimeError:
|
|
raise
|
|
except Exception as exc:
|
|
raise RuntimeError(f"Could not determine SCM ownership for gateway profile {profile_process.profile}") from exc
|
|
return [found[name] for name in sorted(found)]
|
|
|
|
|
|
def _gateway_run_args_for_profile(profile: str) -> list[str]:
|
|
args = [get_python_path(), "-m", "hermes_cli.main"]
|
|
if profile != "default":
|
|
args.extend(["--profile", profile])
|
|
args.extend(["gateway", "run", "--replace"])
|
|
return args
|
|
|
|
|
|
def _capture_gateway_argv(pid: int) -> list[str] | None:
|
|
"""Live argv of a running gateway (snapshotted before update kills so unmapped gateways can respawn);
|
|
None if psutil is unavailable, the process is gone/denied, or the argv isn't a gateway command."""
|
|
if pid <= 1:
|
|
return None
|
|
try:
|
|
import psutil # type: ignore
|
|
except ImportError:
|
|
return None
|
|
try:
|
|
argv = list(psutil.Process(pid).cmdline() or [])
|
|
except Exception: # NoSuchProcess / AccessDenied / ZombieProcess included
|
|
return None
|
|
if not argv:
|
|
return None
|
|
# Never respawn an unrelated process the scan happened to report.
|
|
try:
|
|
from gateway.status import looks_like_gateway_command_line
|
|
if not looks_like_gateway_command_line(" ".join(argv)):
|
|
return None
|
|
except Exception:
|
|
pass
|
|
return argv
|
|
|
|
|
|
def _prepare_profile_gateway_update_restart(profile: str, pid: int) -> str | None:
|
|
"""Choose who relaunches a profile gateway after ``hermes update``: ``--external-supervisor`` gateways
|
|
exit back to their manager (a detached watcher would race its replacement); otherwise arm the
|
|
profile-derived detached watcher, falling back to replaying the captured command line.
|
|
|
|
When the profile-derived relaunch cannot be armed -- typically because ``_gateway_run_args_for_profile``
|
|
cannot rebuild a run argv for this profile -- fall back to replaying the process's own captured command
|
|
line, which is what ``launch_detached_gateway_restart_by_cmdline`` exists for and what the Windows
|
|
post-update path already does for its unmapped gateways. Without this the caller has no way to relaunch
|
|
the process and (before #88654) silently left it running pre-update modules against post-update code on
|
|
disk. ``argv`` is already captured above, so the fallback costs nothing extra.
|
|
"""
|
|
argv = _capture_gateway_argv(pid)
|
|
if argv and "--external-supervisor" in argv:
|
|
return "external-supervisor"
|
|
if launch_detached_profile_gateway_restart(profile, pid):
|
|
return "detached"
|
|
if argv and launch_detached_gateway_restart_by_cmdline(pid, list(argv)):
|
|
return "detached-cmdline"
|
|
return None
|
|
|
|
|
|
def launch_detached_gateway_restart_by_cmdline(old_pid: int, run_argv: list[str]) -> bool:
|
|
"""Relaunch a gateway with no profile→PID-file mapping by replaying its captured argv after exit."""
|
|
return old_pid > 0 and bool(run_argv) and _spawn_gateway_restart_watcher(old_pid, list(run_argv))
|
|
|
|
|
|
def launch_detached_profile_gateway_restart(profile: str, old_pid: int) -> bool:
|
|
"""Relaunch a manually-run profile gateway after its current PID exits."""
|
|
return old_pid > 0 and _spawn_gateway_restart_watcher(old_pid, _gateway_run_args_for_profile(profile))
|
|
|
|
|
|
def _spawn_gateway_restart_watcher(old_pid: int, run_argv: list[str]) -> bool:
|
|
"""Spawn the detached watcher that respawns ``run_argv`` once ``old_pid`` exits. Watcher and respawn
|
|
both need platform-appropriate detach: POSIX setsid; on Windows ``start_new_session`` does NOT detach
|
|
(the watcher would die with the CLI console), so ``windows_detach_popen_kwargs()`` supplies flags."""
|
|
if old_pid <= 0 or not run_argv:
|
|
return False
|
|
from hermes_cli._subprocess_compat import windows_detach_flags_without_breakaway, windows_detach_popen_kwargs
|
|
|
|
# Windows: ``run_argv`` leads with the venv's console ``python.exe`` — the interpreter we want:
|
|
# the watcher respawns it under CREATE_NO_WINDOW detach flags so the gateway owns one hidden
|
|
# console all descendants inherit and nothing flashes (#54220/#56747). The spec helper
|
|
# normalizes the interpreter and captures a stable cwd + env overlay (HERMES_HOME,
|
|
# VIRTUAL_ENV, PYTHONPATH) so the respawn doesn't depend on the watcher's cwd. No-op on POSIX.
|
|
respawn_cwd = ""
|
|
# See gateway_windows.windowless_gateway_restart_spec. See #54220, #56747.
|
|
respawn_env_overlay: dict[str, str] = {}
|
|
if sys.platform == "win32":
|
|
try:
|
|
from hermes_cli.gateway_windows import windowless_gateway_restart_spec
|
|
run_argv, respawn_cwd, respawn_env_overlay = windowless_gateway_restart_spec(list(run_argv))
|
|
except Exception:
|
|
# Fall back to the original argv: a visible window beats a failed respawn.
|
|
respawn_cwd = ""
|
|
respawn_env_overlay = {}
|
|
|
|
# cwd/env overlay are embedded as JSON literals in the watcher source (no extra argv plumbing).
|
|
watcher = textwrap.dedent(
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from hermes_cli._subprocess_compat import (
|
|
_WINDOWS_GATEWAY_BREAKAWAY_ENV, windows_detach_flags, windows_detach_flags_without_breakaway,
|
|
)
|
|
|
|
pid = int(sys.argv[1])
|
|
cmd = sys.argv[2:]
|
|
_respawn_cwd = {respawn_cwd_literal}
|
|
_respawn_env_overlay = {respawn_env_literal}
|
|
deadline = time.monotonic() + 120
|
|
while time.monotonic() < deadline:
|
|
# ``os.kill(pid, 0)`` is not a no-op on Windows — use the cross-platform existence check.
|
|
from gateway.status import _pid_exists
|
|
if not _pid_exists(pid):
|
|
break
|
|
time.sleep(0.2)
|
|
|
|
# Route the respawned gateway's stray stdout/stderr to the same sidecar log _spawn_detached
|
|
# uses: with DEVNULL a gateway killed moments after respawn (parent Job Object teardown when
|
|
# breakaway is denied) left ZERO trace. Best-effort: DEVNULL when the log dir is unavailable.
|
|
_stdio_target = subprocess.DEVNULL
|
|
_stdio_fh = None
|
|
try:
|
|
from hermes_cli.config import get_hermes_home
|
|
from pathlib import Path
|
|
_log_dir = Path(get_hermes_home()) / "logs"
|
|
_log_dir.mkdir(parents=True, exist_ok=True)
|
|
_stdio_fh = open(_log_dir / "gateway-stdio.log", "ab", buffering=0)
|
|
_stdio_target = _stdio_fh
|
|
except Exception:
|
|
pass
|
|
|
|
# Platform-appropriate detach for the respawned gateway: POSIX start_new_session (setsid);
|
|
# Windows needs explicit creationflags. CREATE_BREAKAWAY_FROM_JOB is critical: the watcher may
|
|
# itself sit inside a job object (Electron/Tauri parent) and without breakaway the respawned
|
|
# gateway dies when that job tears down. See _subprocess_compat.windows_detach_flags().
|
|
_popen_kwargs = {{"stdout": _stdio_target, "stderr": _stdio_target}}
|
|
# Anchor at the stable working dir and overlay the env (VIRTUAL_ENV / PYTHONPATH /
|
|
# HERMES_HOME) the windowless base interpreter needs to import hermes_cli. Empty on POSIX.
|
|
if _respawn_cwd:
|
|
_popen_kwargs["cwd"] = _respawn_cwd
|
|
_base_env = {{**os.environ, **_respawn_env_overlay}}
|
|
try:
|
|
if sys.platform == "win32":
|
|
try:
|
|
_popen_kwargs["creationflags"] = windows_detach_flags()
|
|
# Stamp the breakaway state exactly like gateway_windows._spawn_detached so the
|
|
# respawned gateway's exit-diag / lifecycle records show whether it escaped the
|
|
# parent Job Object (a job-teardown kill is otherwise indistinguishable).
|
|
_popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "1"}}
|
|
subprocess.Popen(cmd, **_popen_kwargs)
|
|
except OSError:
|
|
# CREATE_BREAKAWAY_FROM_JOB is rejected with ERROR_ACCESS_DENIED when the parent's
|
|
# job object refuses breakaway; retry without it (mirrors _spawn_detached).
|
|
_popen_kwargs["creationflags"] = windows_detach_flags_without_breakaway()
|
|
_popen_kwargs["env"] = {{**_base_env, _WINDOWS_GATEWAY_BREAKAWAY_ENV: "0"}}
|
|
subprocess.Popen(cmd, **_popen_kwargs)
|
|
else:
|
|
if _respawn_env_overlay:
|
|
_popen_kwargs["env"] = _base_env
|
|
_popen_kwargs["start_new_session"] = True
|
|
subprocess.Popen(cmd, **_popen_kwargs)
|
|
finally:
|
|
if _stdio_fh is not None:
|
|
try:
|
|
_stdio_fh.close()
|
|
except OSError:
|
|
pass
|
|
"""
|
|
).strip().format(respawn_cwd_literal=json.dumps(respawn_cwd), respawn_env_literal=json.dumps(respawn_env_overlay))
|
|
|
|
watcher_argv = [sys.executable, "-c", watcher, str(old_pid), *run_argv]
|
|
devnull = {"stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL}
|
|
# Same detach for the watcher itself, so closing the terminal doesn't kill it.
|
|
try:
|
|
subprocess.Popen(watcher_argv, **devnull, **windows_detach_popen_kwargs())
|
|
except OSError:
|
|
# Parent job object rejected CREATE_BREAKAWAY_FROM_JOB; retry without it (Windows only —
|
|
# ``start_new_session=True`` cannot raise OSError on POSIX).
|
|
fallback_kwargs: dict = (
|
|
{"creationflags": windows_detach_flags_without_breakaway()} if sys.platform == "win32"
|
|
else {"start_new_session": True}
|
|
)
|
|
try:
|
|
subprocess.Popen(watcher_argv, **devnull, **fallback_kwargs)
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _systemd_unit_is_active(system: bool) -> bool:
|
|
"""``systemctl is-active`` == "active" for the installed unit in ``system`` scope, else False."""
|
|
if not get_systemd_unit_path(system=system).exists():
|
|
return False
|
|
try:
|
|
result = _run_systemctl(["is-active", get_service_name()], system=system, timeout=10, **_CAPTURE_TEXT)
|
|
except (RuntimeError, subprocess.TimeoutExpired):
|
|
return False
|
|
return result.stdout.strip() == "active"
|
|
|
|
|
|
def _probe_systemd_service_running(system: bool = False) -> tuple[bool, bool]:
|
|
selected_system = _select_systemd_scope(system)
|
|
return selected_system, _systemd_unit_is_active(selected_system)
|
|
|
|
|
|
def _parse_kv_pairs(items) -> dict[str, str]:
|
|
"""``{key: value}`` from ``KEY=VALUE`` strings (later keys win; values stripped)."""
|
|
return {k: v.strip() for k, v in (item.split("=", 1) for item in items if "=" in item)}
|
|
|
|
|
|
def _systemctl_show(properties: tuple[str, ...], *, system: bool) -> dict[str, str]:
|
|
"""``systemctl show --property a,b`` for the gateway unit as ``{key: value}``; {} on failure."""
|
|
try:
|
|
result = _run_systemctl(
|
|
["show", get_service_name(), "--no-pager", "--property", ",".join(properties)],
|
|
system=_select_systemd_scope(system), timeout=10, **_CAPTURE_TEXT,
|
|
)
|
|
except (RuntimeError, subprocess.TimeoutExpired, OSError):
|
|
return {}
|
|
return _parse_kv_pairs(result.stdout.splitlines()) if result.returncode == 0 else {}
|
|
|
|
|
|
def _hermes_home_pinned_by_unit(unit_path: Path) -> str | None:
|
|
"""``HERMES_HOME`` pinned by the unit file at *unit_path*, or None when absent/unreadable."""
|
|
try:
|
|
text = unit_path.read_text(encoding="utf-8")
|
|
except OSError:
|
|
return None
|
|
for line in text.splitlines():
|
|
body = line.strip()
|
|
if body.startswith("Environment="):
|
|
body = body[len("Environment=") :].strip().strip('"')
|
|
if body.startswith("HERMES_HOME="):
|
|
return body.split("=", 1)[1].strip().strip('"') or None
|
|
return None
|
|
|
|
|
|
def _hermes_home_from_systemd_unit_file(system: bool = False) -> str | None:
|
|
"""``HERMES_HOME`` from the on-disk unit file - what refresh/compare already read, and reliable under ``sudo``."""
|
|
return _hermes_home_pinned_by_unit(get_systemd_unit_path(system=system))
|
|
|
|
|
|
def _sync_hermes_home_from_systemd_unit(system: bool) -> None:
|
|
"""Adopt a system-scope unit's ``HERMES_HOME``: under ``sudo`` it is stripped and HOME=/root, so
|
|
get_hermes_home() would pick the wrong profile for runtime-status/PID reads."""
|
|
if not system:
|
|
return
|
|
# On-disk unit first; ``systemctl show`` for units that only exist in the manager.
|
|
unit_home = (_hermes_home_from_systemd_unit_file(system=True) or "").strip()
|
|
if not unit_home:
|
|
env_line = _systemctl_show(("Environment",), system=True).get("Environment", "")
|
|
unit_home = _parse_kv_pairs(env_line.split()).get("HERMES_HOME", "").strip()
|
|
if unit_home and os.environ.get("HERMES_HOME", "").strip() != unit_home:
|
|
os.environ["HERMES_HOME"] = unit_home
|
|
|
|
|
|
def _read_systemd_unit_properties(
|
|
system: bool = False,
|
|
properties: tuple[str, ...] = ("ActiveState", "SubState", "Result", "ExecMainStatus", "MainPID"),
|
|
) -> dict[str, str]:
|
|
"""Return selected ``systemctl show`` properties for the gateway unit."""
|
|
return _systemctl_show(properties, system=system)
|
|
|
|
|
|
def _positive_pid(value) -> int | None:
|
|
"""``int(value)`` when it parses and is > 0, else None."""
|
|
try:
|
|
pid = int(value or 0)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return pid if pid > 0 else None
|
|
|
|
|
|
def _systemd_main_pid_from_props(props: dict[str, str]) -> int | None:
|
|
return _positive_pid(props.get("MainPID", "0") or "0")
|
|
|
|
|
|
def _runtime_state_pid(state: dict | None) -> int:
|
|
"""``pid`` recorded in a runtime-status dict; 0 when absent, unparsable, or non-positive."""
|
|
return _positive_pid((state or {}).get("pid", 0)) or 0
|
|
|
|
|
|
def _systemd_main_pid(system: bool = False) -> int | None:
|
|
return _systemd_main_pid_from_props(_read_systemd_unit_properties(system=system))
|
|
|
|
|
|
def _read_gateway_runtime_status() -> dict | None:
|
|
try:
|
|
from gateway.status import read_runtime_status
|
|
state = read_runtime_status()
|
|
except Exception:
|
|
return None
|
|
return state if isinstance(state, dict) else None
|
|
|
|
|
|
def _systemd_cli_bits(system: bool) -> tuple[str, str, str]:
|
|
"""``(sudo_prefix, scope_flag, user_flag)`` for printed hints: ``("sudo ", " --system", "")`` in
|
|
system scope, ``("", "", "--user ")`` in user scope."""
|
|
return ("sudo ", " --system", "") if system else ("", "", "--user ")
|
|
|
|
|
|
def _wait_for_systemd_service_restart(
|
|
*,
|
|
system: bool = False,
|
|
previous_pid: int | None = None,
|
|
timeout: float | None = None,
|
|
replacement_observed: list[bool] | None = None,
|
|
) -> bool:
|
|
"""Wait for the gateway service to become active after a restart handoff."""
|
|
svc = get_service_name()
|
|
scope_label = _service_scope_label(system).capitalize()
|
|
if timeout is None:
|
|
timeout = _systemd_restart_wait_timeout(system=system)
|
|
deadline = time.monotonic() + timeout
|
|
printed_runtime_wait = False
|
|
|
|
while time.monotonic() < deadline:
|
|
props = _read_systemd_unit_properties(system=system)
|
|
active_state = props.get("ActiveState", "")
|
|
sub_state = props.get("SubState", "")
|
|
try:
|
|
from gateway.status import get_running_pid
|
|
new_pid = get_running_pid()
|
|
except Exception:
|
|
new_pid = None
|
|
new_pid = new_pid or _systemd_main_pid_from_props(props)
|
|
|
|
runtime_state = _read_gateway_runtime_status()
|
|
runtime_pid = _runtime_state_pid(runtime_state)
|
|
if (
|
|
previous_pid is not None
|
|
and replacement_observed is not None
|
|
and not replacement_observed
|
|
and any(p > 0 and p != previous_pid for p in (new_pid or 0, runtime_pid))
|
|
):
|
|
replacement_observed.append(True)
|
|
|
|
if active_state == "active" and new_pid and (previous_pid is None or new_pid != previous_pid):
|
|
if runtime_pid != new_pid:
|
|
runtime_state = _read_gateway_runtime_status()
|
|
if runtime_state and _runtime_state_pid(runtime_state) != new_pid:
|
|
runtime_state = None
|
|
gateway_state = (runtime_state or {}).get("gateway_state")
|
|
if gateway_state in ("running", "degraded"):
|
|
print(f"✓ {scope_label} service restarted (PID {new_pid})")
|
|
if gateway_state == "degraded":
|
|
# Serving, but a configured platform is parked or retrying: a real restart, not a
|
|
# failure — say so instead of waiting out the timeout and reporting one.
|
|
print(f"⚠ {scope_label} gateway is DEGRADED — see `hermes gateway status`")
|
|
return True
|
|
if gateway_state == "startup_failed":
|
|
reason = (runtime_state or {}).get("exit_reason") or "startup failed"
|
|
print(
|
|
f"⚠ {scope_label} service process restarted (PID {new_pid}), but gateway startup failed: {reason}"
|
|
)
|
|
return False
|
|
if not printed_runtime_wait:
|
|
print(f"⏳ {scope_label} service process started (PID {new_pid}); waiting for gateway runtime...")
|
|
printed_runtime_wait = True
|
|
|
|
if active_state == "activating" and sub_state == "auto-restart":
|
|
time.sleep(1)
|
|
continue
|
|
|
|
if _systemd_unit_is_start_limited(props):
|
|
_print_systemd_start_limit_wait(system=system)
|
|
return False
|
|
|
|
time.sleep(2)
|
|
|
|
sudo, _, user_flag = _systemd_cli_bits(system)
|
|
print(
|
|
f"⚠ {scope_label} service did not become active within {int(timeout)}s.\n"
|
|
f" Check status: {sudo}hermes gateway status\n"
|
|
f" Check logs: journalctl {user_flag}-u {svc} -l --since '2 min ago'"
|
|
)
|
|
return False
|
|
|
|
|
|
def _systemd_restart_wait_timeout(system: bool = False) -> float:
|
|
"""Cover systemd's relaunch delays before applying the runtime wait floor."""
|
|
from gateway.shutdown_forensics import parse_systemd_duration_to_us
|
|
props = _read_systemd_unit_properties(system=system, properties=("RestartUSec", "TimeoutStartUSec"))
|
|
supervisor_budget = 0.0
|
|
for name in ("RestartUSec", "TimeoutStartUSec"):
|
|
raw = props.get(name, "")
|
|
duration_us = int(raw) if raw.isdigit() else parse_systemd_duration_to_us(raw)
|
|
if duration_us is not None:
|
|
supervisor_budget += duration_us / 1_000_000
|
|
return 60.0 + supervisor_budget
|
|
|
|
|
|
def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool:
|
|
return "start-limit-hit" in (props.get("Result", "").lower(), props.get("SubState", "").lower())
|
|
|
|
|
|
def _systemd_error_indicates_start_limit(exc: subprocess.CalledProcessError) -> bool:
|
|
parts: list[str] = []
|
|
for attr in ("stderr", "stdout", "output"):
|
|
value = getattr(exc, attr, None)
|
|
if value:
|
|
parts.append(value.decode(errors="replace") if isinstance(value, bytes) else str(value))
|
|
text = "\n".join(parts).lower()
|
|
return "start-limit-hit" in text or "start request repeated too quickly" in text or "start-limit" in text
|
|
|
|
|
|
def _systemd_service_is_start_limited(system: bool = False) -> bool:
|
|
return _systemd_unit_is_start_limited(_read_systemd_unit_properties(system=system))
|
|
|
|
|
|
def _print_systemd_start_limit_wait(system: bool = False) -> None:
|
|
svc = get_service_name()
|
|
scope_label = _service_scope_label(system).capitalize()
|
|
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
|
|
print(f"⏳ {scope_label} service is temporarily rate-limited by systemd.")
|
|
print(" systemd is refusing another immediate start after repeated exits.")
|
|
print(f" Wait for the start-limit window to expire, then run: {sudo}hermes gateway restart{scope_flag}")
|
|
print(f" Or clear the failed state manually: systemctl {user_flag}reset-failed {svc}")
|
|
print(f" Check logs: journalctl {user_flag}-u {svc} -l --since '5 min ago'")
|
|
|
|
|
|
def _recover_pending_systemd_restart(system: bool = False, previous_pid: int | None = None) -> bool:
|
|
"""Recover a planned service restart that is stuck in systemd state."""
|
|
props = _read_systemd_unit_properties(system=system)
|
|
if not props:
|
|
return False
|
|
|
|
try:
|
|
from gateway.status import read_runtime_status
|
|
except Exception:
|
|
return False
|
|
|
|
if not (read_runtime_status() or {}).get("restart_requested"):
|
|
return False
|
|
|
|
active_state = props.get("ActiveState", "")
|
|
if active_state == "activating" and props.get("SubState", "") == "auto-restart":
|
|
print("⏳ Service restart already pending — waiting for systemd relaunch...")
|
|
return _wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
|
|
|
|
if active_state == "failed" and (
|
|
props.get("ExecMainStatus", "") == str(GATEWAY_SERVICE_RESTART_EXIT_CODE)
|
|
or props.get("Result", "") == "exit-code"
|
|
):
|
|
svc = get_service_name()
|
|
print(f"↻ Clearing failed state for pending {_service_scope_label(system)} service restart...")
|
|
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
|
|
_run_systemctl(["start", svc], system=system, check=False, timeout=90)
|
|
return _wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
|
|
|
|
return False
|
|
|
|
|
|
def _parse_launchd_pid_from_list_output(output: str) -> int | None:
|
|
"""PID from ``launchctl list <label>`` (``"PID" = <n>;``); None if absent (registered, not running)
|
|
or non-positive (crashed)."""
|
|
for line in output.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith(('"PID"', "PID")) and "=" in stripped:
|
|
return _positive_pid(stripped.split("=", 1)[1].strip().rstrip(";").strip('"'))
|
|
return None
|
|
|
|
|
|
def _parse_launchd_pid_from_print_output(output: str) -> int | None:
|
|
"""Live PID from ``launchctl print`` (first ``pid = <N>`` line wins); None if absent or non-positive."""
|
|
for line in output.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("pid = "):
|
|
return _positive_pid(stripped[len("pid = "):].strip())
|
|
return None
|
|
|
|
|
|
def _launchd_print_service_pid(domain: str, label: str) -> tuple[bool, int | None]:
|
|
"""``(loaded, pid)`` for ``domain/label`` via ``launchctl print`` (domain-explicit; ``launchctl list``
|
|
infers it from caller context). ``TimeoutExpired`` propagates: a wedged launchctl is not "unloaded".
|
|
|
|
Domain-explicit on purpose: legacy ``launchctl list`` infers its domain from the caller's execution
|
|
context, which is exactly the ambiguity that sank the first fleet-restart attempt (#41403 review).
|
|
``TimeoutExpired`` propagates — fleet-restart callers own per-label failure accounting (a wedged
|
|
launchctl call must be reported, not read as "unloaded").
|
|
"""
|
|
try:
|
|
result = subprocess.run(["launchctl", "print", f"{domain}/{label}"], timeout=5, **_CAPTURE_TEXT)
|
|
except FileNotFoundError:
|
|
return (False, None)
|
|
if result.returncode != 0:
|
|
return (False, None)
|
|
return (True, _parse_launchd_pid_from_print_output(result.stdout))
|
|
|
|
|
|
def _launchd_service_registered(label: str, *, timeout: int = 5) -> bool:
|
|
"""True when launchd knows ``label`` (``launchctl list`` exit 0). Domain-agnostic, so still true on
|
|
macOS 26+ hosts whose per-user domains reject management. FileNotFoundError/TimeoutExpired propagate."""
|
|
result = subprocess.run(["launchctl", "list", label], timeout=timeout, **_CAPTURE_TEXT)
|
|
return result.returncode == 0
|
|
|
|
|
|
def _locate_launchd_gateway_service(label: str) -> tuple[str | None, int | None]:
|
|
"""``(domain, pid)`` for ``label``, probing ``gui/<uid>`` then ``user/<uid>``. Never uses the current
|
|
profile's cached ``_launchd_domain()`` — a fleet can mix domains. ``TimeoutExpired`` propagates."""
|
|
uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
|
|
for domain in (f"gui/{uid}", f"user/{uid}"):
|
|
loaded, pid = _launchd_print_service_pid(domain, label)
|
|
if loaded:
|
|
return (domain, pid)
|
|
return (None, None)
|
|
|
|
|
|
def _probe_launchd_service_running() -> bool:
|
|
"""True when the plist exists AND launchd is running a process for the current label."""
|
|
return get_launchd_plist_path().exists() and _launchctl_label_supervising_process(get_launchd_label())
|
|
|
|
|
|
def _s6_gateway_snapshot(gateway_pids: tuple[int, ...]) -> GatewayRuntimeSnapshot | None:
|
|
"""Snapshot for an s6-supervised container gateway, or None when s6 isn't the service manager."""
|
|
from hermes_cli.service_manager import detect_service_manager, get_service_manager
|
|
if detect_service_manager() != "s6":
|
|
return None
|
|
service_name = f"gateway-{_current_profile_name()}"
|
|
mgr = get_service_manager()
|
|
service_installed = service_running = False
|
|
try:
|
|
service_dir = getattr(mgr, "scandir", None)
|
|
if service_dir is not None:
|
|
service_installed = (service_dir / service_name).is_dir()
|
|
except Exception:
|
|
service_installed = False
|
|
if service_installed:
|
|
try:
|
|
service_running = bool(mgr.is_running(service_name))
|
|
except Exception:
|
|
service_running = False
|
|
return GatewayRuntimeSnapshot(
|
|
manager="s6 (container supervisor)",
|
|
service_installed=service_installed,
|
|
service_running=service_running,
|
|
gateway_pids=gateway_pids,
|
|
service_scope="s6",
|
|
)
|
|
|
|
|
|
def get_gateway_runtime_snapshot(system: bool = False) -> GatewayRuntimeSnapshot:
|
|
"""Return a unified view of gateway liveness for the current profile."""
|
|
gateway_pids = tuple(find_gateway_pids())
|
|
if is_termux():
|
|
return GatewayRuntimeSnapshot(manager="Termux / manual process", gateway_pids=gateway_pids)
|
|
|
|
from hermes_constants import is_container
|
|
if is_linux() and is_container():
|
|
# Report s6 supervision under our /init; other container runtimes keep "docker (foreground)".
|
|
try:
|
|
snapshot = _s6_gateway_snapshot(gateway_pids)
|
|
if snapshot is not None:
|
|
return snapshot
|
|
except Exception:
|
|
pass # Fall through to the legacy label on any detection error.
|
|
return GatewayRuntimeSnapshot(manager="docker (foreground)", gateway_pids=gateway_pids)
|
|
|
|
if supports_systemd_services():
|
|
selected_system, service_running = _probe_systemd_service_running(system=system)
|
|
scope_label = _service_scope_label(selected_system)
|
|
return GatewayRuntimeSnapshot(
|
|
manager=f"systemd ({scope_label})",
|
|
service_installed=get_systemd_unit_path(system=selected_system).exists(),
|
|
service_running=service_running,
|
|
gateway_pids=gateway_pids,
|
|
service_scope=scope_label,
|
|
)
|
|
|
|
if is_macos():
|
|
return GatewayRuntimeSnapshot(
|
|
manager="launchd",
|
|
service_installed=get_launchd_plist_path().exists(),
|
|
service_running=_probe_launchd_service_running(),
|
|
gateway_pids=gateway_pids,
|
|
service_scope="launchd",
|
|
)
|
|
|
|
return GatewayRuntimeSnapshot(manager="manual process", gateway_pids=gateway_pids)
|
|
|
|
|
|
def _format_gateway_pids(pids: tuple[int, ...] | list[int], *, limit: int | None = 3) -> str:
|
|
rendered = [str(pid) for pid in (pids if limit is None else pids[:limit]) if pid > 0]
|
|
if limit is not None and len(pids) > limit:
|
|
rendered.append("...")
|
|
return ", ".join(rendered)
|
|
|
|
|
|
def _print_gateway_process_mismatch(snapshot: GatewayRuntimeSnapshot) -> None:
|
|
if not snapshot.has_process_service_mismatch:
|
|
return
|
|
print()
|
|
pids_line = f" PID(s): {_format_gateway_pids(snapshot.gateway_pids, limit=None)}"
|
|
# Managed detached fallback (launchd exit-5 path) vs. a genuinely manual run.
|
|
if _launchd_unsupported_marker_exists():
|
|
print("⚠ Gateway is running as a detached fallback process — launchd cannot supervise it")
|
|
print(pids_line)
|
|
print(" Auto-start at login and auto-restart on crash are NOT available.")
|
|
print(" Stop it with: hermes gateway stop")
|
|
else:
|
|
print("⚠ Gateway process is running for this profile, but the service is not active")
|
|
print(pids_line)
|
|
print(" This is usually a manual foreground/tmux/nohup run, so `hermes gateway`")
|
|
print(" can refuse to start another copy until this process stops.")
|
|
|
|
|
|
def _print_multiplex_standalone_reason() -> None:
|
|
"""The boot guard kept an unset-default gateway standalone: say so in status, with the remedy."""
|
|
try:
|
|
from gateway.status import read_runtime_status
|
|
reason = (read_runtime_status() or {}).get("multiplex_standalone_reason")
|
|
except Exception:
|
|
return
|
|
if reason:
|
|
print(f"⚠ Serving the default profile only (gateway.multiplex_profiles unset): {reason}")
|
|
print(" Fold every profile onto this gateway: hermes gateway migrate --multiplex")
|
|
print(" Keep per-profile gateways: hermes config set gateway.multiplex_profiles false")
|
|
|
|
|
|
def _print_served_ingress_urls(profile: str | None = None) -> None:
|
|
"""Callback URLs of inbound-port platforms the live multiplexer serves for secondary profiles
|
|
(the value to paste into the Twilio / LINE / Teams / BlueBubbles console)."""
|
|
try:
|
|
from hermes_cli.gateway_multiplex_served import format_ingress_url_lines, served_profile_ingress_urls
|
|
urls = served_profile_ingress_urls(profile)
|
|
except Exception:
|
|
return
|
|
if not urls:
|
|
return
|
|
print()
|
|
print("Inbound callback URLs on the shared listener:")
|
|
for name, per_platform in sorted(urls.items()):
|
|
for line in format_ingress_url_lines(per_platform, indent=f" {name}/" if not profile else " "):
|
|
print(line)
|
|
|
|
|
|
def _print_unserved_shared_ingress(profile: str | None) -> None:
|
|
"""Shared-ingress platforms (WhatsApp/Relay) this served profile enabled that the multiplexer runs
|
|
only on the default profile — the ``whatsapp: not served under multiplex`` line."""
|
|
try:
|
|
from hermes_cli.gateway_multiplex_served import served_profile_unserved_platforms
|
|
unserved = served_profile_unserved_platforms(profile or "")
|
|
except Exception:
|
|
return
|
|
if not unserved:
|
|
return
|
|
print()
|
|
for platform, reason in sorted(unserved.items()):
|
|
print(f" ⚠ {platform}: {reason}")
|
|
print(" Enable it on the default profile (shared ingress serves every profile), or disable it here.")
|
|
|
|
|
|
def _print_other_profiles_gateway_status() -> None:
|
|
"""Print other profiles' running gateways at the bottom of ``hermes gateway status``."""
|
|
try:
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
current = get_active_profile_name()
|
|
other_processes = [p for p in find_profile_gateway_processes() if p.profile != current]
|
|
if not other_processes:
|
|
return
|
|
print()
|
|
print("Other profiles:")
|
|
for proc in other_processes:
|
|
print(f" ✓ {proc.profile:<16s} — PID {proc.pid}")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _gateway_list() -> None:
|
|
"""List every profile and whether its gateway is running."""
|
|
try:
|
|
from hermes_cli.profiles import list_profiles, get_active_profile_name
|
|
except Exception:
|
|
print("Unable to list profiles.")
|
|
return
|
|
|
|
profiles = list_profiles()
|
|
if not profiles:
|
|
print("No profiles found.")
|
|
return
|
|
|
|
current = get_active_profile_name()
|
|
|
|
print("Gateways:")
|
|
for prof in profiles:
|
|
marker = "✓" if prof.gateway_running else "✗"
|
|
label = prof.name + (" (current)" if prof.name == current else "")
|
|
parts = [f" {marker} {label:<24s}"]
|
|
if prof.gateway_running:
|
|
pid = None
|
|
try:
|
|
from gateway.status import get_running_pid
|
|
pid = get_running_pid(prof.path / "gateway.pid", cleanup_stale=False)
|
|
except Exception:
|
|
pass
|
|
if pid:
|
|
parts.append(f"PID {pid}")
|
|
elif named_profile_served_by_running_multiplexer(prof.name):
|
|
parts.append("served by the default multiplexer")
|
|
else:
|
|
parts.append("not running")
|
|
print(" — ".join(parts))
|
|
|
|
|
|
def kill_gateway_processes(force: bool = False, exclude_pids: set | None = None, all_profiles: bool = False) -> int:
|
|
"""Kill running gateway processes (force-kill if ``force``); ``exclude_pids`` skips e.g. just-
|
|
restarted service PIDs. Returns count killed."""
|
|
killed = 0
|
|
for pid in find_gateway_pids(exclude_pids=exclude_pids, all_profiles=all_profiles):
|
|
try:
|
|
expected_start_time = None
|
|
if force:
|
|
# Re-verify the LIVE cmdline at kill time: a PID recycled since the scan must never be tree-killed.
|
|
# Re-verify at kill time, not just scan time: the cmdline match inside find_gateway_pids()
|
|
# is stale by the time we get here, and a recycled PID could otherwise be tree-killed
|
|
# (#89614 class). _capture_gateway_argv re-reads the LIVE cmdline and returns None for
|
|
# anything that no longer looks like a gateway — refuse those.
|
|
if _capture_gateway_argv(pid) is None:
|
|
continue
|
|
from gateway.status import get_process_start_time
|
|
expected_start_time = get_process_start_time(pid)
|
|
terminate_pid(pid, force=force, expected_start_time=expected_start_time)
|
|
killed += 1
|
|
except ProcessLookupError:
|
|
pass
|
|
except PermissionError:
|
|
print(f"⚠ Permission denied to kill PID {pid}")
|
|
except OSError as exc:
|
|
print(f"Failed to kill PID {pid}: {exc}")
|
|
return killed
|
|
|
|
|
|
_REAPER_SUPERVISOR_WALK_LIMIT = 12
|
|
|
|
|
|
def _reaper_candidate_is_supervisor_owned(pid: int) -> bool:
|
|
"""True when ``pid``'s parent chain reaches ``services.exe`` (Task Scheduler-owned gateway). Windows-only
|
|
reaper backstop: ``_get_service_pids()`` is empty there, so a Scheduled-Task gateway with a stale
|
|
pidfile would look like an orphan. Fail-open once the Task's bootstrap parent exits. Not applied
|
|
on POSIX, where everything descends from PID 1 and would look supervised.
|
|
|
|
See #83683, #86098.
|
|
This check is deliberately NOT applied on POSIX: there, every process has PID 1 (launchd / init /
|
|
systemd) in its ancestry — and a genuine orphan is *reparented directly to PID 1* — so supervisor-name
|
|
ancestry carries zero signal and would spare every orphan the reaper exists to kill (#51325, 75936).
|
|
POSIX supervised gateways are already covered pidfile- independently by the ``_get_service_pids()``
|
|
exclusion.
|
|
"""
|
|
if not is_windows():
|
|
return False
|
|
try:
|
|
import psutil # type: ignore
|
|
parent = psutil.Process(pid).parent()
|
|
for _ in range(_REAPER_SUPERVISOR_WALK_LIMIT):
|
|
if parent is None:
|
|
break
|
|
with contextlib.suppress(Exception):
|
|
if (parent.name() or "").lower() == "services.exe":
|
|
return True
|
|
parent = parent.parent()
|
|
except Exception:
|
|
pass
|
|
return False
|
|
|
|
|
|
def _reap_unsupervised_gateway_orphans(extra_exclude: set | None = None) -> bool:
|
|
"""Kill no-supervisor gateway orphans the pidfile/runtime record can't see. On WSL/no-systemd hosts
|
|
the restart fallback runs the gateway in-process under a ``gateway restart`` argv; a stale pidfile
|
|
then lets a live orphan keep the webhook port while a restart stacks a duplicate. No-op where a
|
|
supervisor exists (there ``gateway restart`` is a transient command). ``extra_exclude``: already killed."""
|
|
try:
|
|
supervised_host = supports_systemd_services()
|
|
except Exception:
|
|
supervised_host = True
|
|
if supervised_host:
|
|
return False
|
|
|
|
# Task Scheduler is a supervisor too; its state beats a parent-chain walk (broken once the bootstrap exits).
|
|
# A Scheduled Task gateway whose conhost/VBS bootstrap has already exited is invisible to
|
|
# `_reaper_candidate_is_supervisor_owned` (the parent chain breaks before services.exe, fail-open), yet
|
|
# it is alive and supervised. After that launcher exits the task is typically Ready, not Running —
|
|
# treating only Running as supervised still kills the detached gateway on every desktop serve start
|
|
# (#86098, #87001).
|
|
if is_windows():
|
|
try:
|
|
from hermes_cli.gateway_windows import get_task_name # profile-aware task name
|
|
_task_name = get_task_name()
|
|
except Exception:
|
|
_task_name = "Hermes_Gateway"
|
|
if _windows_scheduled_task_supervises(_task_name):
|
|
return False
|
|
|
|
from gateway.status import _pid_exists, get_process_start_time, write_planned_stop_marker
|
|
own = _reaper_exclusion_pids(extra_exclude)
|
|
try:
|
|
# On Windows also drop Task Scheduler-owned candidates (the pidfile-less gap).
|
|
orphans = [
|
|
p for p in find_gateway_pids(exclude_pids=own) if p and p > 0 and not _reaper_candidate_is_supervisor_owned(p)
|
|
]
|
|
except Exception:
|
|
return False
|
|
if not orphans:
|
|
return False
|
|
|
|
# Pin each orphan's start time now: the delayed SIGKILL must never hit a recycled PID.
|
|
# Pin each orphan's identity NOW: the cmdline scan above matched at scan-time only, and the SIGKILL
|
|
# escalation below fires seconds later. A PID recycled inside that window must never be force-killed
|
|
# (#89614 class). Fingerprint capture is best-effort — SIGTERM below proceeds regardless (it targets the
|
|
# process verified by the scan an instant ago), but the delayed SIGKILL requires a still-matching
|
|
# fingerprint.
|
|
orphan_identity: dict[int, int] = {}
|
|
for pid in orphans:
|
|
start = get_process_start_time(pid)
|
|
if start is not None:
|
|
orphan_identity[pid] = start
|
|
|
|
reaped = False
|
|
for pid in orphans:
|
|
with contextlib.suppress(Exception):
|
|
write_planned_stop_marker(pid)
|
|
# ``os.kill(..., SIGTERM)`` maps to TerminateProcess on Windows, so it
|
|
# would kill the gateway before its marker watcher can drain and close
|
|
# state cleanly. Let the bounded survivor wait below escalate instead.
|
|
if is_windows():
|
|
reaped = True
|
|
continue
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
continue
|
|
except PermissionError:
|
|
print(f"⚠ Permission denied to kill orphaned gateway PID {pid}")
|
|
continue
|
|
reaped = True
|
|
|
|
# Wait, then force-kill survivors so the replacement can bind the port cleanly.
|
|
# Fail-closed: SIGKILL only a PID that still names the process fingerprinted at scan time.
|
|
_force_kill_survivors([
|
|
pid for pid in _await_gateway_exit(orphans, pid_exists=_pid_exists)
|
|
if pid in orphan_identity and get_process_start_time(pid) == orphan_identity[pid]
|
|
])
|
|
return reaped
|
|
|
|
|
|
def _reaper_exclusion_pids(extra_exclude: set | None) -> set[int]:
|
|
"""PIDs the orphan reaper must never kill: self, caller extras, service-managed, recorded."""
|
|
own = {os.getpid()} | (extra_exclude or set())
|
|
# Service-managed gateways are never orphans (on macOS supports_systemd_services() is False, so a
|
|
# launchd gateway would otherwise be SIGTERM'd); all_profiles because the scan sees siblings too.
|
|
with contextlib.suppress(Exception):
|
|
# This covers macOS launchd (supports_systemd_services() is False there, so without this the launchd
|
|
# gateway looks like an unsupervised orphan and gets SIGTERM'd, causing launchd to restart it — or
|
|
# leaving it down under KeepAlive.SuccessfulExit=false) and any systemd unit reachable from a host
|
|
# that got past the gate above (#83683, #85344).
|
|
# all_profiles=True: the reaper's process scan sees every profile's gateway (and on macOS the
|
|
# now-working ps fallback surfaces sibling launchd gateways, #73626), so the service exclusion must
|
|
# cover the whole ai.hermes.gateway* fleet — not just the current profile's label — or a sibling
|
|
# profile's launchd gateway is misclassified as an unsupervised orphan and reaped. Same class as the
|
|
# update-sweep fix in #74075.
|
|
own |= _get_service_pids(all_profiles=True)
|
|
# Exempt the recorded gateway PID and its parent chain (on Windows the Scheduled-Task bootstrap's
|
|
# ``gateway run`` argv matches the scan; killing it takes the gateway down). Use the RAW pidfile +
|
|
# lock records, not only the validated probe: get_running_pid returns None on any validation
|
|
# hiccup — exactly when a healthy standalone gateway would be hard-killed (Windows SIGTERM is
|
|
# TerminateProcess, no drain). For a KILL exclusion list a stale PID at worst spares one process;
|
|
# a false negative kills a live gateway. The probe still supplies the runtime-status fallback PID.
|
|
try:
|
|
from gateway.status import _pid_from_record, _read_gateway_lock_record, _read_pid_record, get_running_pid
|
|
recorded_pids = {_pid_from_record(rec) for rec in (_read_pid_record(), _read_gateway_lock_record())}
|
|
recorded_pids.add(get_running_pid(cleanup_stale=False))
|
|
for recorded in recorded_pids:
|
|
if not recorded or recorded <= 0:
|
|
continue
|
|
own.add(recorded)
|
|
try:
|
|
import psutil # type: ignore
|
|
parent = psutil.Process(recorded).parent()
|
|
while parent is not None:
|
|
own.add(parent.pid)
|
|
parent = parent.parent()
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
return own
|
|
|
|
|
|
# A retiring gateway runs a PASSIVE WAL checkpoint in ``SessionDB.close()``; a SIGKILL mid-checkpoint
|
|
# corrupts ``state.db``. It keeps serving while we wait, so a long grace only delays the port bind.
|
|
_ORPHAN_EXIT_GRACE_SECONDS = 30.0
|
|
_ORPHAN_EXIT_POLL_SECONDS = 0.2
|
|
|
|
|
|
def _await_gateway_exit(
|
|
pids, *, pid_exists, sleep=None, grace_s: float = _ORPHAN_EXIT_GRACE_SECONDS, poll_s: float = _ORPHAN_EXIT_POLL_SECONDS
|
|
):
|
|
"""Poll up to *grace_s* for *pids* to exit; return survivors. ``pid_exists``/``sleep`` injectable for tests."""
|
|
if sleep is None:
|
|
sleep = time.sleep
|
|
survivors = list(pids)
|
|
for _ in range(max(1, int(grace_s / poll_s))):
|
|
survivors = [p for p in survivors if pid_exists(p)]
|
|
if not survivors:
|
|
break
|
|
sleep(poll_s)
|
|
else:
|
|
# Re-check after the LAST sleep, or a recycled PID could get the SIGKILL.
|
|
survivors = [p for p in survivors if pid_exists(p)]
|
|
return survivors
|
|
|
|
|
|
def _force_kill_survivors(survivors, *, kill=None) -> None:
|
|
"""SIGKILL processes that outlasted the grace period, loudly — a force-kill can tear the store, so
|
|
it must leave a trace."""
|
|
kill = kill or os.kill
|
|
for pid in survivors:
|
|
logger.warning(
|
|
"Gateway PID %s did not exit within %.0fs of the stop request (SIGTERM, or the planned-stop "
|
|
"marker on Windows) — sending "
|
|
"SIGKILL. A kill during a WAL checkpoint can corrupt state.db; "
|
|
"the next start will run an integrity check.",
|
|
pid, _ORPHAN_EXIT_GRACE_SECONDS,
|
|
)
|
|
with contextlib.suppress((ProcessLookupError, PermissionError, OSError)):
|
|
kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
|
|
|
|
|
|
def _mark_planned_stop(pid: int | None = None) -> None:
|
|
"""Best-effort planned-stop marker for ``pid`` (default: the recorded gateway PID)."""
|
|
try:
|
|
from gateway.status import get_running_pid, write_planned_stop_marker
|
|
if pid is None:
|
|
pid = get_running_pid(cleanup_stale=False)
|
|
if pid is not None:
|
|
write_planned_stop_marker(pid)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def stop_profile_gateway() -> bool:
|
|
"""Stop only this profile's gateway via its PID file; True if a process was stopped. Without a
|
|
supervisor the pidfile can be stale while a live orphan holds the webhook port, so fall back to
|
|
the orphan-aware scan rather than stacking a duplicate.
|
|
|
|
Even when the pid file is valid and points to the current gateway, older orphans may linger from prior
|
|
restarts that overwrote the pid file before the old process exited. After killing the recorded PID, also
|
|
sweep for any remaining orphans so each restart produces at most one live gateway (#75936).
|
|
"""
|
|
try:
|
|
from gateway.status import get_running_pid, remove_pid_file
|
|
except ImportError:
|
|
return False
|
|
|
|
pid = get_running_pid()
|
|
if pid is None:
|
|
return _reap_unsupervised_gateway_orphans()
|
|
|
|
if is_windows():
|
|
# Windows maps SIGTERM to TerminateProcess. The marker watcher is the
|
|
# gateway's graceful-stop IPC, so wait for it before force-killing a
|
|
# wedged process.
|
|
from gateway.status import get_process_start_time
|
|
from hermes_cli.gateway_windows import (
|
|
_drain_gateway_pid,
|
|
_force_terminate_known_gateway_pids,
|
|
_windows_stop_drain_timeout,
|
|
)
|
|
|
|
# Capture identity BEFORE the drain (as _escalate_wedged_gateway does): if the PID is
|
|
# recycled during the wait, terminate_pid's start-time mismatch refuses the taskkill.
|
|
expected_start_time = get_process_start_time(pid)
|
|
if not _drain_gateway_pid(pid, _windows_stop_drain_timeout()):
|
|
_force_terminate_known_gateway_pids({pid: expected_start_time})
|
|
else:
|
|
_mark_planned_stop(pid)
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass # Already gone
|
|
except PermissionError:
|
|
print(f"⚠ Permission denied to kill PID {pid}")
|
|
return False
|
|
|
|
# ``_pid_exists``, NOT ``os.kill(pid, 0)`` (TerminateProcess on Windows).
|
|
from gateway.status import _pid_exists
|
|
for _ in range(20):
|
|
if not _pid_exists(pid):
|
|
break
|
|
time.sleep(0.5)
|
|
|
|
if get_running_pid() is None:
|
|
remove_pid_file()
|
|
|
|
# Reap orphans from prior restarts whose pidfile entry was overwritten; skip the PID just killed.
|
|
try:
|
|
# Exclude the PID we just killed so the sweep doesn't double-kill a process that's still tearing
|
|
# down — _reap_unsupervised_gateway_orphans already excludes our own PID. See #75936.
|
|
_reap_unsupervised_gateway_orphans(extra_exclude={pid} if pid else None)
|
|
except Exception as exc:
|
|
logger.debug("orphan reap after stop_profile_gateway failed: %s", exc)
|
|
return True
|
|
|
|
|
|
def is_linux() -> bool:
|
|
return sys.platform.startswith("linux")
|
|
|
|
|
|
from hermes_constants import is_container, is_termux, is_wsl
|
|
|
|
|
|
def _wsl_systemd_operational() -> bool:
|
|
"""WSL2 with ``systemd=true`` in wsl.conf has working systemd; WSL1/without it does not."""
|
|
return _systemd_operational(system=True)
|
|
|
|
|
|
def _systemd_operational(system: bool = False) -> bool:
|
|
"""Return True when the requested systemd scope is usable."""
|
|
try:
|
|
result = _run_systemctl(["is-system-running"], system=system, timeout=5, **_CAPTURE_TEXT)
|
|
except (RuntimeError, subprocess.TimeoutExpired, OSError):
|
|
return False
|
|
# "running", "degraded", "starting" all mean systemd is PID 1
|
|
return result.stdout.strip().lower() in {"running", "degraded", "starting", "initializing"}
|
|
|
|
|
|
def supports_systemd_services() -> bool:
|
|
if not is_linux() or is_termux() or shutil.which("systemctl") is None:
|
|
return False
|
|
if is_wsl():
|
|
return _wsl_systemd_operational()
|
|
if is_container():
|
|
# A container whose init is systemd (nspawn, some k8s pods) behaves like a host.
|
|
return _systemd_operational(system=False) or _systemd_operational(system=True)
|
|
return True
|
|
|
|
|
|
def is_macos() -> bool:
|
|
return sys.platform == "darwin"
|
|
|
|
|
|
def is_windows() -> bool:
|
|
return sys.platform == "win32"
|
|
|
|
|
|
def _gw_windows():
|
|
"""Lazily import :mod:`hermes_cli.gateway_windows` (Windows-only service backend)."""
|
|
from hermes_cli import gateway_windows
|
|
return gateway_windows
|
|
|
|
|
|
# Task Scheduler states meaning "still supervised" (Ready = steady state after the launcher exits).
|
|
# Task Scheduler states that mean "this profile still has an official supervisor". Queued is a rare
|
|
# in-between. Disabled / MISSING are not supervisors. See #87001.
|
|
_WINDOWS_TASK_SUPERVISOR_STATES = frozenset({"Running", "Ready", "Queued"})
|
|
|
|
|
|
def _windows_scheduled_task_state(task_name: str) -> str | None:
|
|
"""Locale-independent Task Scheduler state, or None on failure.
|
|
|
|
Query the COM API directly: Get-ScheduledTask auto-loads the CIM module,
|
|
which can stall desktop backend startup for the entire ten-second timeout.
|
|
Keep the existing supervisor semantics (Ready and Queued count as owned).
|
|
"""
|
|
if not is_windows():
|
|
return None
|
|
quoted_name = task_name.replace("'", "''")
|
|
ps_cmd = (
|
|
"$ErrorActionPreference = 'Stop'; "
|
|
"$s = [Activator]::CreateInstance([type]::GetTypeFromProgID('Schedule.Service')); "
|
|
"$s.Connect(); "
|
|
"try { "
|
|
f"$t = $s.GetFolder('\\').GetTask('{quoted_name}'); "
|
|
# TASK_STATE values are stable, unlike localized schtasks.exe output.
|
|
"@('Unknown', 'Disabled', 'Queued', 'Ready', 'Running')[[int]$t.State] "
|
|
"} catch { "
|
|
"$e = $_.Exception; while ($e.InnerException) { $e = $e.InnerException }; "
|
|
"if ($e.HResult -in @(-2147024894, -2147024893)) { 'MISSING' } else { throw } "
|
|
"}"
|
|
)
|
|
try:
|
|
powershell = shutil.which("powershell") or shutil.which("pwsh")
|
|
if powershell is None:
|
|
return None
|
|
result = subprocess.run(
|
|
[powershell, "-NoProfile", "-NonInteractive", "-Command", ps_cmd],
|
|
capture_output=True, text=True, encoding="utf-8", errors="ignore", timeout=10,
|
|
)
|
|
if result.returncode != 0:
|
|
return None
|
|
return (result.stdout or "").strip() or None
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
|
|
|
|
def _windows_scheduled_task_supervises(task_name: str) -> bool:
|
|
"""True when Task Scheduler still owns this profile's gateway (Ready counts: the task is Ready, not
|
|
Running, after bootstrap exits). Any failure returns False so callers fall back to pidfile / parent-chain.
|
|
|
|
Used to treat Task Scheduler as a gateway supervisor on Windows: the orphan-reap sweep must not kill a
|
|
gateway that a scheduled task launched and left detached. After the bootstrap exits the task is Ready,
|
|
not Running; a Running-only check still writes the planned-stop marker, the gateway exits cleanly with
|
|
code 0, and the scheduler never restarts it — silently killing A2A/messaging on every desktop-app launch
|
|
(#86098, #87001).
|
|
"""
|
|
return _windows_scheduled_task_state(task_name) in _WINDOWS_TASK_SUPERVISOR_STATES
|
|
|
|
|
|
def _gateway_detached_env() -> bool:
|
|
return _truthy_env(os.getenv("HERMES_GATEWAY_DETACHED"))
|
|
|
|
|
|
def _stdin_is_tty() -> bool | None:
|
|
"""``sys.stdin.isatty()``; None when stdin is closed/invalid."""
|
|
try:
|
|
return bool(sys.stdin and sys.stdin.isatty())
|
|
except (ValueError, OSError):
|
|
return None
|
|
|
|
|
|
def _windows_gateway_should_absorb_console_controls() -> bool:
|
|
"""True for detached Windows gateway runs that should ignore Ctrl+C (``HERMES_GATEWAY_DETACHED=1``
|
|
or no interactive stdin); foreground runs stay interruptible."""
|
|
if not is_windows():
|
|
return False
|
|
if _gateway_detached_env():
|
|
return True
|
|
return not _stdin_is_tty()
|
|
|
|
|
|
def _windows_console_window_attached() -> bool | None:
|
|
"""Return whether Windows assigned this process a console window."""
|
|
if not is_windows():
|
|
return None
|
|
try:
|
|
import ctypes
|
|
return bool(ctypes.windll.kernel32.GetConsoleWindow()) # type: ignore[attr-defined]
|
|
except (OSError, AttributeError):
|
|
return None
|
|
|
|
|
|
def _windows_gateway_breakaway_state() -> bool | None:
|
|
"""Consume private spawn metadata without guessing for older launchers."""
|
|
if not is_windows():
|
|
return None
|
|
from hermes_cli._subprocess_compat import _WINDOWS_GATEWAY_BREAKAWAY_ENV
|
|
return {"1": True, "0": False}.get(os.environ.pop(_WINDOWS_GATEWAY_BREAKAWAY_ENV, None))
|
|
|
|
|
|
# =============================================================================
|
|
# Service Configuration
|
|
# =============================================================================
|
|
|
|
_SERVICE_BASE = "hermes-gateway"
|
|
SERVICE_DESCRIPTION = "Hermes Agent Gateway - Messaging Platform Integration"
|
|
|
|
_SYSTEM_UNIT_DIR = Path("/etc/systemd/system")
|
|
|
|
|
|
def _profile_name_from_home(home: Path, default: Path) -> str | None:
|
|
"""Profile name when ``home`` is ``<default>/profiles/<name>`` with a service-safe name, else None."""
|
|
import re
|
|
try:
|
|
parts = home.relative_to((default / "profiles").resolve()).parts
|
|
except ValueError:
|
|
return None
|
|
if len(parts) == 1 and re.match(r"^[a-z0-9][a-z0-9_-]{0,63}$", parts[0]):
|
|
return parts[0]
|
|
return None
|
|
|
|
|
|
def _native_service_homes() -> set[Path]:
|
|
"""This process's native default home plus, when root under sudo, the invoking user's (see
|
|
``_profile_suffix`` for why sudo matters)."""
|
|
from hermes_constants import _get_platform_default_hermes_home, sudo_invoker_default_home
|
|
|
|
homes = {_get_platform_default_hermes_home().resolve()}
|
|
sudo_home = sudo_invoker_default_home()
|
|
if sudo_home is not None:
|
|
homes.add(sudo_home.resolve())
|
|
return homes
|
|
|
|
|
|
def _bare_unit_pinned_home() -> Path | None:
|
|
"""Resolved ``HERMES_HOME`` pinned by an installed ``hermes-gateway.service``, or None. The unit is the
|
|
one naming basis that holds still across the sudo mid-command switch (see ``_profile_suffix``) and it
|
|
covers every elevated identity — ``sudo -i`` and cron included, where SUDO_USER is absent.
|
|
|
|
Linux- and root-gated: a systemd unit is not an identity authority for launchd labels, Windows
|
|
scheduled tasks, or s6 slots, which share ``_profile_suffix()``, and only an elevated process ever
|
|
operates the system unit — an unprivileged user-scope command must keep naming its own units, or a
|
|
bare system unit pinning ``profiles/<name>`` would alias that profile onto the user's default unit.
|
|
``is_linux()`` is a plain ``sys.platform`` test; ``supports_systemd_services()`` would be wrong here,
|
|
since it can shell out to ``systemctl is-system-running`` on WSL/containers and this runs on every
|
|
name resolution.
|
|
"""
|
|
if not is_linux() or os.geteuid() != 0: # windows-footgun: ok — behind is_linux()
|
|
return None
|
|
pinned = _hermes_home_pinned_by_unit(_SYSTEM_UNIT_DIR / f"{_SERVICE_BASE}.service")
|
|
if not pinned:
|
|
return None
|
|
try:
|
|
return Path(pinned).expanduser().resolve()
|
|
except (RuntimeError, ValueError): # hand-edited unit: ``~nouser`` or an embedded NUL
|
|
return None
|
|
|
|
|
|
def _profile_suffix() -> str:
|
|
"""Service-name suffix for HERMES_HOME: "" for a home that owns the bare name, the profile name for
|
|
``<root>/profiles/<name>``, else a short hash of the path.
|
|
|
|
Bare-name owners: this process's platform-native default (``~/.hermes``), under sudo the invoking
|
|
user's native default, and the home pinned by an installed ``hermes-gateway.service``. Under sudo the
|
|
naming basis moves MID-COMMAND — sudo strips HERMES_HOME and sets HOME=/root, then
|
|
``_sync_hermes_home_from_systemd_unit()`` adopts the unit's own HERMES_HOME into ``os.environ`` — so a
|
|
basis derived from the process alone names one unit before the adoption and another after it. The
|
|
unit-pinned check must precede the profile branch: ``sudo hermes gateway install --system`` resolves
|
|
the BARE name from root's default, then pins the invoking user's remapped home, so the bare unit
|
|
legitimately carries a ``<root>/profiles/<name>`` home.
|
|
|
|
The bare name is deliberately NOT tied to ``get_default_hermes_root()``: that helper treats any
|
|
HERMES_HOME outside ``~/.hermes`` (Docker ``/opt/data``, a temp dir) as "the root itself", which let a
|
|
temp-home harness resolve to the default profile's ``hermes-gateway`` unit and uninstall the
|
|
production gateway. Service names are host-wide identities; a home with no installed bare unit and
|
|
no native default keeps its own suffix.
|
|
"""
|
|
import hashlib
|
|
from hermes_constants import get_default_hermes_root
|
|
home = get_hermes_home().resolve()
|
|
if home in _native_service_homes() or home == _bare_unit_pinned_home():
|
|
return ""
|
|
name = _profile_name_from_home(home, get_default_hermes_root().resolve())
|
|
return name or hashlib.sha256(str(home).encode()).hexdigest()[:8]
|
|
|
|
|
|
def _current_profile_name() -> str:
|
|
"""Profile id relative to the profile ROOT: ``default`` for the root itself (Docker's ``/opt/data``
|
|
included), ``<name>`` for ``<root>/profiles/<name>``, else the service hash. s6 slots and the
|
|
multiplexer ask which PROFILE this is; ``_profile_suffix()`` answers which HOST SERVICE this is."""
|
|
from hermes_constants import profile_name_for_home
|
|
return profile_name_for_home(get_hermes_home()) or _profile_suffix()
|
|
|
|
|
|
def _profile_arg(hermes_home: str | None = None, default_root: str | Path | None = None) -> str:
|
|
"""``--profile <name>`` for ``<root>/profiles/<name>``, else "". *hermes_home*/*default_root* let a
|
|
sudo/root process generate a unit for another user (the defaults would refer to root)."""
|
|
from hermes_constants import get_default_hermes_root
|
|
home = Path(hermes_home or str(get_hermes_home())).resolve()
|
|
default = Path(default_root).resolve() if default_root else get_default_hermes_root().resolve()
|
|
if home == default:
|
|
return ""
|
|
name = _profile_name_from_home(home, default)
|
|
return f"--profile {name}" if name else ""
|
|
|
|
|
|
def get_service_name() -> str:
|
|
"""Systemd service name: ``hermes-gateway`` for default HERMES_HOME, ``hermes-gateway-<profile>``
|
|
or ``-<hash>`` otherwise."""
|
|
suffix = _profile_suffix()
|
|
return f"{_SERVICE_BASE}-{suffix}" if suffix else _SERVICE_BASE
|
|
|
|
|
|
def get_systemd_unit_path(system: bool = False) -> Path:
|
|
name = get_service_name()
|
|
if system:
|
|
return _SYSTEM_UNIT_DIR / f"{name}.service"
|
|
return Path.home() / ".config" / "systemd" / "user" / f"{name}.service"
|
|
|
|
|
|
class UserSystemdUnavailableError(RuntimeError):
|
|
"""``systemctl --user`` cannot reach the user D-Bus session (fresh SSH sessions with linger off,
|
|
so ``/run/user/$UID/bus`` never exists). ``args[0]`` is a user-facing remediation message."""
|
|
|
|
|
|
class SystemScopeRequiresRootError(RuntimeError):
|
|
"""System-scope gateway operation attempted as non-root. Typed (not ``sys.exit(1)``) so the setup
|
|
wizard can print remediation; ``args`` = (message, action) and ``str(e)`` is the message only."""
|
|
|
|
def __str__(self) -> str:
|
|
return self.args[0] if self.args else ""
|
|
|
|
|
|
def _user_runtime_dir() -> Path:
|
|
"""``$XDG_RUNTIME_DIR`` or ``/run/user/<uid>`` (regardless of existence)."""
|
|
return Path(os.environ.get("XDG_RUNTIME_DIR") or f"/run/user/{os.getuid()}") # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
|
|
|
|
def _user_dbus_socket_path() -> Path:
|
|
"""Return the expected per-user D-Bus socket path (regardless of existence)."""
|
|
return _user_runtime_dir() / "bus"
|
|
|
|
|
|
def _user_systemd_private_socket_path() -> Path:
|
|
"""Return the per-user systemd private socket path (regardless of existence)."""
|
|
return _user_runtime_dir() / "systemd" / "private"
|
|
|
|
|
|
def _path_exists_safe(path: Path) -> bool:
|
|
"""``Path.exists()`` treating an inaccessible path as absent: a leaked ``XDG_RUNTIME_DIR`` from
|
|
another user (``/run/user/0`` is 0700) would otherwise crash the preflight with EACCES.
|
|
|
|
``Path.exists()`` only swallows a subset of ``OSError`` (ENOENT/ENOTDIR/ EBADF/ELOOP); ``EACCES`` still
|
|
propagates. When ``XDG_RUNTIME_DIR`` leaks from another user — the classic ``su``/``sudo -u`` from a
|
|
root shell case, where ``/run/user/0`` is ``0700 root:root`` — stat-ing a socket underneath it raises
|
|
``PermissionError`` that escapes the systemd preflight as a raw traceback (#86558). An unreadable path
|
|
is, for our purposes, not reachable.
|
|
"""
|
|
try:
|
|
return path.exists()
|
|
except OSError: # e.g. EACCES on another user's runtime dir
|
|
return False
|
|
|
|
|
|
def _runtime_dir_is_ours(runtime_dir: str) -> bool:
|
|
"""True when *runtime_dir* exists and is owned by our uid (a leaked foreign XDG_RUNTIME_DIR must not be trusted)."""
|
|
try:
|
|
return Path(runtime_dir).stat().st_uid == os.getuid() # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _user_systemd_socket_ready() -> bool:
|
|
"""True when the user D-Bus socket OR the per-user systemd private socket exists (some distros
|
|
expose only the latter and ``systemctl --user`` still works). Inaccessible counts as not-ready."""
|
|
return _path_exists_safe(_user_dbus_socket_path()) or _path_exists_safe(_user_systemd_private_socket_path())
|
|
|
|
|
|
def _ensure_user_systemd_env() -> None:
|
|
"""Set XDG_RUNTIME_DIR / DBUS_SESSION_BUS_ADDRESS so ``systemctl --user`` works on headless (SSH)
|
|
hosts; an XDG_RUNTIME_DIR leaked from another user is replaced with our own ``/run/user/{uid}``.
|
|
|
|
An ``XDG_RUNTIME_DIR`` that leaked from another user (``su``/``sudo -u`` from root, where the env still
|
|
points at ``/run/user/0``) is dropped in favour of our own ``/run/user/{uid}`` so ``systemctl --user``
|
|
targets the right instance instead of an unreadable foreign socket (#86558).
|
|
"""
|
|
uid = os.getuid() # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
xdg = os.environ.get("XDG_RUNTIME_DIR")
|
|
if (not xdg or not _runtime_dir_is_ours(xdg)) and _runtime_dir_is_ours(f"/run/user/{uid}"):
|
|
os.environ["XDG_RUNTIME_DIR"] = f"/run/user/{uid}"
|
|
|
|
if "DBUS_SESSION_BUS_ADDRESS" not in os.environ:
|
|
bus_path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{uid}")) / "bus"
|
|
if _path_exists_safe(bus_path):
|
|
os.environ["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path={bus_path}"
|
|
|
|
|
|
def _wait_for_user_dbus_socket(timeout: float = 3.0) -> bool:
|
|
"""Poll up to ``timeout`` s for a user systemd control socket (user@.service takes a moment after enable-linger)."""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if _user_systemd_socket_ready():
|
|
_ensure_user_systemd_env()
|
|
return True
|
|
time.sleep(0.2)
|
|
return _user_systemd_socket_ready()
|
|
|
|
|
|
def _wait_for_target_user_bus(uid: int, timeout: float = 5.0) -> bool:
|
|
"""Poll for ``/run/user/<uid>/bus`` of ANOTHER account (the system unit's ``User=`` while root installs).
|
|
Only the D-Bus socket counts — ``systemd/private`` alone is enough for ``systemctl --user`` but not for
|
|
the ``systemd-run --user`` that restart-safe workers need. Never adopts anything into our env."""
|
|
bus = Path(f"/run/user/{uid}/bus")
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if _path_exists_safe(bus):
|
|
return True
|
|
time.sleep(0.2)
|
|
return _path_exists_safe(bus)
|
|
|
|
|
|
def _loginctl_enable_linger(username: str) -> subprocess.CompletedProcess:
|
|
"""``loginctl enable-linger <username>`` (check=False, 30s); exceptions propagate to the caller."""
|
|
return subprocess.run(["loginctl", "enable-linger", username], check=False, timeout=30, **_CAPTURE_TEXT)
|
|
|
|
|
|
def _completed_process_detail(result) -> str:
|
|
"""stderr, else stdout, else ``exit <rc>`` — stripped."""
|
|
return (result.stderr or result.stdout or f"exit {result.returncode}").strip()
|
|
|
|
|
|
def _preflight_user_systemd(*, auto_enable_linger: bool = True) -> None:
|
|
"""Ensure ``systemctl --user`` can reach user-scope systemd; raise UserSystemdUnavailableError otherwise.
|
|
No-op when a control socket exists; else wait briefly if linger is on, or (``auto_enable_linger``)
|
|
try ``loginctl enable-linger`` (non-root works when polkit permits)."""
|
|
_ensure_user_systemd_env()
|
|
if _user_systemd_socket_ready():
|
|
return
|
|
|
|
import getpass
|
|
username = getpass.getuser()
|
|
linger_enabled, linger_detail = get_systemd_linger_status()
|
|
sudo_hint = f" sudo loginctl enable-linger {username}"
|
|
|
|
if linger_enabled is True:
|
|
if _wait_for_user_dbus_socket(timeout=3.0):
|
|
return
|
|
# Linger is on but socket still missing — unusual; fall through to error.
|
|
_raise_user_systemd_unavailable(
|
|
username,
|
|
reason="User systemd control sockets are missing even though linger is enabled.",
|
|
fix_hint=(
|
|
f" systemctl start user@{os.getuid()}.service\n" # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
" (may require sudo; try again after the command succeeds)"
|
|
),
|
|
)
|
|
|
|
if auto_enable_linger and shutil.which("loginctl"):
|
|
try:
|
|
result = _loginctl_enable_linger(username)
|
|
except Exception as exc:
|
|
_raise_user_systemd_unavailable(
|
|
username, reason=f"loginctl enable-linger failed ({exc}).", fix_hint=sudo_hint
|
|
)
|
|
else:
|
|
if result.returncode == 0:
|
|
if _wait_for_user_dbus_socket(timeout=5.0):
|
|
print(f"✓ Enabled linger for {username} — user D-Bus now available")
|
|
return
|
|
# enable-linger succeeded but the socket never appeared.
|
|
_raise_user_systemd_unavailable(
|
|
username,
|
|
reason="Linger was enabled, but the user D-Bus socket did not appear.",
|
|
fix_hint=(
|
|
" Log out and log back in, then re-run the command.\n"
|
|
f" Or reboot and run: systemctl --user start {get_service_name()}"
|
|
),
|
|
)
|
|
_raise_user_systemd_unavailable(
|
|
username,
|
|
reason=f"loginctl enable-linger was denied: {_completed_process_detail(result)}",
|
|
fix_hint=sudo_hint,
|
|
)
|
|
|
|
_raise_user_systemd_unavailable(
|
|
username,
|
|
reason=f"User D-Bus session is not available ({linger_detail or 'linger disabled'}).",
|
|
fix_hint=sudo_hint,
|
|
)
|
|
|
|
|
|
def _raise_user_systemd_unavailable(username: str, *, reason: str, fix_hint: str) -> None:
|
|
"""Build a user-facing error message and raise UserSystemdUnavailableError."""
|
|
msg = (
|
|
f"{reason}\n"
|
|
" systemctl --user cannot reach the user D-Bus session in this shell.\n"
|
|
"\n"
|
|
" To fix:\n"
|
|
f"{fix_hint}\n"
|
|
"\n"
|
|
" Alternative: run the gateway in the foreground (stays up until\n"
|
|
" you exit / close the terminal):\n"
|
|
" hermes gateway run"
|
|
)
|
|
raise UserSystemdUnavailableError(msg)
|
|
|
|
|
|
def _systemctl_cmd(system: bool = False) -> list[str]:
|
|
if not system:
|
|
_ensure_user_systemd_env()
|
|
return ["systemctl"] if system else ["systemctl", "--user"]
|
|
|
|
|
|
def _run_systemctl(args: list[str], *, system: bool = False, **kwargs) -> subprocess.CompletedProcess:
|
|
"""Run systemctl; raise RuntimeError (not raw FileNotFoundError) if missing, for callers bypassing
|
|
``supports_systemd_services()``."""
|
|
try:
|
|
return subprocess.run(_systemctl_cmd(system) + args, **kwargs)
|
|
except FileNotFoundError:
|
|
from hermes_cli.gateway_command_errors import SystemctlUnavailableError
|
|
raise SystemctlUnavailableError() from None
|
|
|
|
|
|
def _service_scope_label(system: bool = False) -> str:
|
|
return "system" if system else "user"
|
|
|
|
|
|
def get_installed_systemd_scopes() -> list[str]:
|
|
scopes: list[str] = []
|
|
seen_paths: set[Path] = set()
|
|
for system, label in ((False, "user"), (True, "system")):
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
if unit_path not in seen_paths and unit_path.exists():
|
|
scopes.append(label)
|
|
seen_paths.add(unit_path)
|
|
return scopes
|
|
|
|
|
|
def has_conflicting_systemd_units() -> bool:
|
|
return len(get_installed_systemd_scopes()) > 1
|
|
|
|
|
|
# Legacy pre-rename names: explicit allowlist (NOT a glob) so profile and third-party units never match.
|
|
_LEGACY_SERVICE_NAMES: tuple[str, ...] = ("hermes.service",)
|
|
|
|
# ExecStart markers identifying a unit as running our gateway; a legacy unit is flagged only if one matches.
|
|
_LEGACY_UNIT_EXECSTART_MARKERS: tuple[str, ...] = (
|
|
"hermes_cli.main gateway",
|
|
"hermes_cli/main.py gateway",
|
|
"gateway/run.py",
|
|
" hermes gateway ",
|
|
"/hermes gateway ",
|
|
)
|
|
|
|
|
|
def _legacy_unit_search_paths() -> list[tuple[bool, Path]]:
|
|
"""``[(is_system, base_dir), ...]`` to scan for legacy units; factored out so tests can monkeypatch."""
|
|
return [(False, Path.home() / ".config" / "systemd" / "user"), (True, _SYSTEM_UNIT_DIR)]
|
|
|
|
|
|
def _find_legacy_hermes_units() -> list[tuple[str, Path, bool]]:
|
|
"""``[(unit_name, unit_path, is_system)]`` for legacy gateway units (e.g. ``hermes.service``), which
|
|
fight the current unit for the bot token (SIGTERM flap loop). Explicit name allowlist + ExecStart
|
|
marker check so profile/third-party units never match; no mutation.
|
|
|
|
Detects unit files installed by older Hermes versions that used a different service name (e.g. When both
|
|
a legacy unit and the current ``hermes-gateway.service`` are active, they fight over the same bot token
|
|
— the PR #5646 signal-recovery change turns this into a 30-second SIGTERM flap loop.
|
|
"""
|
|
results: list[tuple[str, Path, bool]] = []
|
|
for is_system, base in _legacy_unit_search_paths():
|
|
for name in _LEGACY_SERVICE_NAMES:
|
|
unit_path = base / name
|
|
try:
|
|
if not unit_path.exists():
|
|
continue
|
|
text = unit_path.read_text(encoding="utf-8", errors="ignore")
|
|
except (OSError, PermissionError):
|
|
continue
|
|
if any(marker in text for marker in _LEGACY_UNIT_EXECSTART_MARKERS):
|
|
results.append((name, unit_path, is_system))
|
|
return results
|
|
|
|
|
|
def has_legacy_hermes_units() -> bool:
|
|
"""Return True when any legacy Hermes gateway unit files exist."""
|
|
return bool(_find_legacy_hermes_units())
|
|
|
|
|
|
def print_legacy_unit_warning() -> None:
|
|
"""Warn about installed legacy gateway units; prints nothing when there are none."""
|
|
legacy = _find_legacy_hermes_units()
|
|
if not legacy:
|
|
return
|
|
print_warning("Legacy Hermes gateway unit(s) detected from an older install:")
|
|
for name, path, is_system in legacy:
|
|
print_info(f" {path} ({_service_scope_label(is_system)} scope)")
|
|
print_info(" These run alongside the current hermes-gateway service and")
|
|
print_info(" cause SIGTERM flap loops — both try to use the same bot token.")
|
|
print_info(" Remove them with:")
|
|
print_info(" hermes gateway migrate-legacy")
|
|
|
|
|
|
def remove_legacy_hermes_units(interactive: bool = True, dry_run: bool = False) -> tuple[int, list[Path]]:
|
|
"""Stop, disable, and remove legacy gateway units. ``interactive=False`` skips the prompt; ``dry_run``
|
|
only lists. Returns ``(removed_count, remaining_paths)`` (remaining: e.g. system-scope when not root)."""
|
|
legacy = _find_legacy_hermes_units()
|
|
if not legacy:
|
|
print("No legacy Hermes gateway units found.")
|
|
return 0, []
|
|
|
|
print()
|
|
print("Legacy Hermes gateway unit(s) found:")
|
|
for name, path, is_system in legacy:
|
|
print(f" {path} ({_service_scope_label(is_system)} scope)")
|
|
print()
|
|
|
|
if dry_run:
|
|
print("(dry-run — nothing removed)")
|
|
return 0, [p for _, p, _ in legacy]
|
|
|
|
if interactive and not prompt_yes_no("Remove these legacy units?", True):
|
|
print("Skipped. Run again with: hermes gateway migrate-legacy")
|
|
return 0, [p for _, p, _ in legacy]
|
|
|
|
removed = 0
|
|
remaining: list[Path] = []
|
|
|
|
def _remove_units(units: list[tuple[str, Path]], *, system: bool) -> None:
|
|
nonlocal removed
|
|
for name, path in units:
|
|
try:
|
|
_run_systemctl(["stop", name], system=system, check=False, timeout=90)
|
|
_run_systemctl(["disable", name], system=system, check=False, timeout=30)
|
|
path.unlink(missing_ok=True)
|
|
print(f" ✓ Removed {path}")
|
|
removed += 1
|
|
except (OSError, RuntimeError) as e:
|
|
print(f" ⚠ Could not remove {path}: {e}")
|
|
remaining.append(path)
|
|
with contextlib.suppress(RuntimeError):
|
|
_run_systemctl(["daemon-reload"], system=system, check=False, timeout=30)
|
|
|
|
user_units = [(n, p) for n, p, is_sys in legacy if not is_sys]
|
|
system_units = [(n, p) for n, p, is_sys in legacy if is_sys]
|
|
if user_units:
|
|
_remove_units(user_units, system=False)
|
|
|
|
# System-scope removal (needs root)
|
|
if system_units:
|
|
if os.geteuid() != 0: # windows-footgun: ok — Linux systemd removal path, guarded by `if system == "Linux"` / systemd-only branch
|
|
print()
|
|
print_warning("System-scope legacy units require root to remove.")
|
|
print_info(" Re-run with: sudo hermes gateway migrate-legacy")
|
|
remaining.extend(path for _, path in system_units)
|
|
else:
|
|
_remove_units(system_units, system=True)
|
|
|
|
print()
|
|
if remaining:
|
|
print_warning(f"{len(remaining)} legacy unit(s) still present — see messages above.")
|
|
else:
|
|
print_success(f"Removed {removed} legacy unit(s).")
|
|
|
|
return removed, remaining
|
|
|
|
|
|
def print_systemd_scope_conflict_warning() -> None:
|
|
scopes = get_installed_systemd_scopes()
|
|
if len(scopes) < 2:
|
|
return
|
|
|
|
print_warning(f"Both user and system gateway services are installed ({' + '.join(scopes)}).")
|
|
print_info(" This is confusing and can make start/stop/status behavior ambiguous.")
|
|
print_info(" Default gateway commands target the user service unless you pass --system.")
|
|
print_info(" Keep one of these:")
|
|
print_info(" hermes gateway uninstall")
|
|
print_info(" sudo hermes gateway uninstall --system")
|
|
|
|
|
|
def refuses_container_user_scope_install(system: bool) -> bool:
|
|
"""True (after printing the guidance) when a fresh USER-scope unit was requested inside a container.
|
|
|
|
A systemd container passes ``supports_systemd_services()`` on purpose so ``--system`` keeps working,
|
|
but a user unit there is not container-scoped: the unit file and its ``default.target.wants`` symlink
|
|
land in ``~/.config/systemd/user`` — commonly the host's own home bind-mounted in — so the host's
|
|
``systemd --user`` enables it too and a second gateway polls the same bot token outside the container.
|
|
Callers decide between ``sys.exit(1)`` (CLI) and skipping the install (wizard)."""
|
|
if system or not is_container():
|
|
return False
|
|
print_error("Refusing to install a user-scope systemd gateway service inside a container.")
|
|
_print_info_lines(
|
|
"The unit file and its enable symlink would be written to the home directory, which is",
|
|
"commonly the host's own home bind-mounted in — the host's user manager then enables and",
|
|
"starts the same unit, so a second gateway polls the same bot token outside the container",
|
|
"(Telegram: 'Conflict: terminated by other getUpdates request').",
|
|
"",
|
|
" hermes gateway run # run as the container's main process",
|
|
" docker run --restart unless-stopped ... # container restart policy",
|
|
"",
|
|
"If systemd manages this container (systemd as PID 1), install an isolated system service instead:",
|
|
" sudo hermes gateway install --system --run-as-user <user>",
|
|
)
|
|
return True
|
|
|
|
|
|
def _require_root_for_system_service(action: str) -> None:
|
|
if os.geteuid() != 0: # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
raise SystemScopeRequiresRootError(f"System gateway {action} requires root. Re-run with sudo.", action)
|
|
|
|
|
|
def _system_service_identity(run_as_user: str | None = None) -> tuple[str, str, str, int]:
|
|
import getpass
|
|
import grp
|
|
import pwd
|
|
username = (
|
|
run_as_user or os.getenv("SUDO_USER") or os.getenv("USER") or os.getenv("LOGNAME") or getpass.getuser()
|
|
).strip()
|
|
if not username:
|
|
raise ValueError("Could not determine which user the gateway service should run as")
|
|
if username == "root" and not run_as_user:
|
|
raise ValueError(
|
|
"Refusing to install the gateway system service as root; pass --run-as-user root to override (e.g. in LXC containers)"
|
|
)
|
|
if username == "root":
|
|
print_warning("Installing gateway service to run as root.")
|
|
print_info(" This is fine for LXC/container environments but not recommended on bare-metal hosts.")
|
|
|
|
try:
|
|
user_info = pwd.getpwnam(username)
|
|
except KeyError as e:
|
|
raise ValueError(f"Unknown user: {username}") from e
|
|
return username, grp.getgrgid(user_info.pw_gid).gr_name, user_info.pw_dir, user_info.pw_uid
|
|
|
|
|
|
def _read_systemd_user_from_unit(unit_path: Path) -> str | None:
|
|
if not unit_path.exists():
|
|
return None
|
|
for line in unit_path.read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("User="):
|
|
return line.split("=", 1)[1].strip() or None
|
|
return None
|
|
|
|
|
|
def _default_system_service_user() -> str | None:
|
|
for candidate in (os.getenv("SUDO_USER"), os.getenv("USER"), os.getenv("LOGNAME")):
|
|
candidate = (candidate or "").strip()
|
|
if candidate and candidate != "root":
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def prompt_linux_gateway_install_scope() -> str | None:
|
|
# Only root can create a boot-time system service; never hand a non-root user a "re-run under sudo" recipe.
|
|
is_root = os.geteuid() == 0 # windows-footgun: ok — Linux systemd install wizard, never invoked on Windows
|
|
options = ["User service (no sudo; best for laptops/dev boxes; may need linger after logout)"]
|
|
values: list[str | None] = ["user"]
|
|
if is_root:
|
|
options.append("System service (starts on boot; runs as your chosen user)")
|
|
values.append("system")
|
|
options.append("Skip service install for now")
|
|
values.append(None)
|
|
choice = prompt_choice(" Choose how the gateway should run in the background:", options, default=0)
|
|
if not is_root and choice == 0:
|
|
print_info(" Tip: for a boot-time system service, re-run setup as root (e.g. from a root shell or `sudo -i`).")
|
|
return values[choice]
|
|
|
|
|
|
def install_linux_gateway_from_setup(force: bool = False, enable_on_startup: bool = True) -> tuple[str | None, bool]:
|
|
scope = prompt_linux_gateway_install_scope()
|
|
if scope is None:
|
|
return None, False
|
|
|
|
if scope == "system":
|
|
run_as_user = _default_system_service_user()
|
|
if os.geteuid() != 0: # windows-footgun: ok — Linux systemd install wizard, never invoked on Windows
|
|
# Unreachable from the wizard (system scope only offered to root); defensive guard for direct callers.
|
|
print_warning(
|
|
" System service install requires root. Re-run setup from a "
|
|
"root shell, or install a user service instead: hermes gateway install"
|
|
)
|
|
return scope, False
|
|
|
|
while not run_as_user:
|
|
run_as_user = (prompt(" Run the system gateway service as which user?", default="") or "").strip()
|
|
if not run_as_user:
|
|
print_error(" Enter a username.")
|
|
|
|
systemd_install(force=force, system=True, run_as_user=run_as_user, enable_on_startup=enable_on_startup)
|
|
return scope, True
|
|
|
|
if refuses_container_user_scope_install(system=False):
|
|
return scope, False
|
|
systemd_install(force=force, system=False, enable_on_startup=enable_on_startup)
|
|
return scope, True
|
|
|
|
|
|
def ensure_gateway_service(context: str = "setup") -> bool:
|
|
"""Install and start a user-scope gateway service without prompting (``hermes setup``/``import``).
|
|
A zero-platform gateway is a supported degraded mode (cron runs), so this never gates on messaging
|
|
config. Never raises; True when a service is installed and running."""
|
|
from hermes_constants import is_container
|
|
if is_container():
|
|
# Containers use restart policies, not service managers.
|
|
print_info("Start the gateway to bring your bots online:")
|
|
print_info(" hermes gateway run # Run as container main process")
|
|
print_info("")
|
|
print_info("For automatic restarts, use a Docker restart policy:")
|
|
print_info(" docker run --restart unless-stopped ...")
|
|
return False
|
|
|
|
supports_systemd = supports_systemd_services()
|
|
if not (supports_systemd or is_macos() or is_windows()):
|
|
print_info(" No supported service manager found on this host.")
|
|
print_info(" Run the gateway in the foreground with: hermes gateway")
|
|
return False
|
|
|
|
try:
|
|
if _is_service_running():
|
|
return True
|
|
if _served_profile_needs_no_service():
|
|
return True
|
|
if not _is_service_installed():
|
|
if supports_systemd and has_conflicting_systemd_units():
|
|
# Both units would fight over bot tokens; don't pile a fresh install onto a conflicted state.
|
|
print_systemd_scope_conflict_warning()
|
|
return False
|
|
print_info(" Installing the gateway background service ...")
|
|
if supports_systemd:
|
|
systemd_install(force=False, non_interactive=True)
|
|
elif is_macos():
|
|
launchd_install(force=False)
|
|
else:
|
|
_gw_windows().install(force=False) # Registers the Scheduled Task AND starts it.
|
|
print_success(" Gateway service installed and started.")
|
|
return True
|
|
if supports_systemd:
|
|
systemd_start()
|
|
elif is_macos():
|
|
launchd_start()
|
|
else:
|
|
_gw_windows().start()
|
|
print_success(" Gateway service running (cron jobs + messaging platforms).")
|
|
return True
|
|
except UserSystemdUnavailableError as e:
|
|
print_warning(" Could not reach user systemd to start the gateway service:")
|
|
_print_indented(str(e), print_info)
|
|
except SystemScopeRequiresRootError as e:
|
|
print_warning(f" Gateway service needs root for this scope: {e}")
|
|
_print_system_scope_remediation("start")
|
|
except SystemExit:
|
|
# Some install/start paths sys.exit() on hard failures (temp-HOME guard); never abort setup/import.
|
|
print_warning(" Gateway service install did not complete.")
|
|
print_info(" You can retry manually: hermes gateway install")
|
|
except Exception as e:
|
|
print_warning(f" Gateway service install failed: {e}")
|
|
print_info(" You can retry manually: hermes gateway install")
|
|
return False
|
|
|
|
|
|
def get_systemd_linger_status(username: str | None = None) -> tuple[bool | None, str]:
|
|
"""Linger status for *username* or the current user when omitted.
|
|
|
|
System-scope gateway installation runs as root but the service runs as a
|
|
configured target user, so querying the caller would validate the wrong
|
|
user manager.
|
|
"""
|
|
if is_termux():
|
|
return None, "not supported in Termux"
|
|
if not is_linux():
|
|
return None, "not supported on this platform"
|
|
if not shutil.which("loginctl"):
|
|
return None, "loginctl not found"
|
|
|
|
if username is None:
|
|
username = os.getenv("USER") or os.getenv("LOGNAME")
|
|
if not username:
|
|
try:
|
|
import pwd
|
|
username = pwd.getpwuid(os.getuid()).pw_name # windows-footgun: ok — POSIX loginctl helper, never invoked on Windows
|
|
except Exception:
|
|
return None, "could not determine current user"
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
["loginctl", "show-user", username, "--property=Linger", "--value"],
|
|
check=False, timeout=10, **_CAPTURE_TEXT,
|
|
)
|
|
except Exception as e:
|
|
return None, str(e)
|
|
|
|
if result.returncode != 0:
|
|
return None, _completed_process_detail(result) or "loginctl query failed"
|
|
|
|
value = (result.stdout or "").strip().lower()
|
|
if value in {"yes", "true", "1"}:
|
|
return True, ""
|
|
if value in {"no", "false", "0"}:
|
|
return False, ""
|
|
return None, f"unexpected loginctl output: {value or '<empty>'}"
|
|
|
|
|
|
def get_launchd_plist_path() -> Path:
|
|
"""``~/Library/LaunchAgents/ai.hermes.gateway[-<profile>].plist`` under the real account home."""
|
|
import pwd
|
|
suffix = _profile_suffix()
|
|
name = f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
|
|
# Real account home: profile mode may point HOME at a profile dir.
|
|
home = Path(pwd.getpwuid(os.getuid()).pw_dir) # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
|
|
return home / "Library" / "LaunchAgents" / f"{name}.plist"
|
|
|
|
|
|
def launchd_gateway_labels_for_install() -> list[str]:
|
|
"""Launchd labels for every profile of THIS install (root first, then profiles by name). Derived from
|
|
the profile layout, NOT by globbing ``~/Library/LaunchAgents``, so a sandboxed HERMES_HOME never
|
|
restarts another install's fleet. Names that can't map to a suffix are skipped."""
|
|
import re as _re
|
|
from hermes_cli.profiles import list_profiles
|
|
root_label: list[str] = []
|
|
profile_labels: list[str] = []
|
|
for profile in list_profiles():
|
|
if profile.is_default:
|
|
root_label.append("ai.hermes.gateway")
|
|
elif _re.match(r"^[a-z0-9][a-z0-9_-]{0,63}$", profile.name):
|
|
profile_labels.append(f"ai.hermes.gateway-{profile.name}")
|
|
return root_label + sorted(profile_labels)
|
|
|
|
|
|
def _detect_venv_dir() -> Path | None:
|
|
"""Active virtualenv dir: ``sys.prefix``, then ``VIRTUAL_ENV`` (uv sets it without changing
|
|
sys.prefix), then .venv/venv under PROJECT_ROOT; None if none found."""
|
|
candidates: list[Path] = []
|
|
if sys.prefix != sys.base_prefix:
|
|
candidates.append(Path(sys.prefix))
|
|
if os.environ.get("VIRTUAL_ENV"):
|
|
candidates.append(Path(os.environ["VIRTUAL_ENV"]))
|
|
candidates += [PROJECT_ROOT / ".venv", PROJECT_ROOT / "venv"]
|
|
return next((venv for venv in candidates if venv.is_dir()), None)
|
|
|
|
|
|
def get_python_path() -> str:
|
|
venv = _detect_venv_dir()
|
|
if venv is not None:
|
|
try:
|
|
from hermes_constants import venv_python_path
|
|
except ImportError:
|
|
# Update-boundary: a gateway restarted mid-update can hold a stale hermes_constants
|
|
# without this symbol; see _reload_hermes_constants() in hermes_cli/managed_uv.py.
|
|
from hermes_cli.managed_uv import _reload_hermes_constants
|
|
venv_python_path = _reload_hermes_constants().venv_python_path
|
|
|
|
venv_python = venv_python_path(venv, windows=is_windows())
|
|
if venv_python.exists():
|
|
return str(venv_python)
|
|
return sys.executable
|
|
|
|
|
|
# =============================================================================
|
|
# Systemd (Linux)
|
|
# =============================================================================
|
|
|
|
|
|
def _build_user_local_paths(home: Path, path_entries: list[str]) -> list[str]:
|
|
"""Return user-local bin dirs that exist and aren't already in *path_entries*."""
|
|
candidates = [
|
|
str(home / ".local" / "bin"), # uv, uvx, pip-installed CLIs
|
|
str(home / ".cargo" / "bin"), # Rust/cargo tools
|
|
str(home / "go" / "bin"), # Go tools
|
|
str(home / ".npm-global" / "bin"), # npm global packages
|
|
]
|
|
return [p for p in candidates if p not in path_entries and Path(p).exists()]
|
|
|
|
|
|
def _build_wsl_interop_paths(path_entries: list[str]) -> list[str]:
|
|
"""WSL Windows-interop PATH entries for generated units: systemd services don't inherit the
|
|
Windows PATH (``/mnt/c/WINDOWS/System32``…), so ``powershell.exe``/``cmd.exe`` break unless persisted."""
|
|
if not is_wsl():
|
|
return []
|
|
|
|
candidates = [entry for entry in os.environ.get("PATH", "").split(os.pathsep) if entry.startswith("/mnt/")]
|
|
for executable in ("powershell.exe", "cmd.exe", "explorer.exe", "wsl.exe"):
|
|
resolved = shutil.which(executable)
|
|
if resolved:
|
|
candidates.append(str(Path(resolved).parent))
|
|
candidates += [
|
|
entry
|
|
for entry in (
|
|
"/mnt/c/WINDOWS/system32",
|
|
"/mnt/c/WINDOWS",
|
|
"/mnt/c/WINDOWS/System32/Wbem",
|
|
"/mnt/c/WINDOWS/System32/WindowsPowerShell/v1.0/",
|
|
"/mnt/c/WINDOWS/System32/OpenSSH/",
|
|
)
|
|
if Path(entry).exists()
|
|
]
|
|
|
|
result: list[str] = []
|
|
seen = set(path_entries)
|
|
for entry in candidates:
|
|
if entry and entry not in seen:
|
|
seen.add(entry)
|
|
result.append(entry)
|
|
return result
|
|
|
|
|
|
def _remap_path_for_user(path: str, target_home_dir: str) -> str:
|
|
"""Swap the ``Path.home()`` prefix of *path* for *target_home_dir*; other paths return unchanged.
|
|
Intentionally does NOT resolve symlinks."""
|
|
current_home = Path.home()
|
|
p = Path(path).expanduser()
|
|
try:
|
|
relative = p.relative_to(current_home)
|
|
return str(Path(target_home_dir) / relative)
|
|
except ValueError:
|
|
return str(p)
|
|
|
|
|
|
def _hermes_home_for_target_user(target_home_dir: str) -> str:
|
|
"""Remap the current HERMES_HOME (root's, under sudo) to the target user's equivalent:
|
|
``/root/.hermes[/profiles/x]`` → ``/home/alice/.hermes[/profiles/x]``; custom paths kept as-is."""
|
|
current_hermes_raw = os.environ.get("HERMES_HOME", "").strip()
|
|
current_hermes = Path(current_hermes_raw).expanduser() if current_hermes_raw else get_hermes_home()
|
|
# Keep paths lexical: resolving a non-existent path can bake a different HERMES_HOME into the unit.
|
|
current_default = Path.home() / ".hermes"
|
|
target_default = Path(target_home_dir) / ".hermes"
|
|
try:
|
|
# Default ~/.hermes or a profile/subdir of it → preserve the relative structure under the target.
|
|
return str(target_default / current_hermes.relative_to(current_default))
|
|
except ValueError:
|
|
return str(current_hermes) # Completely custom path (not under ~/.hermes) — keep as-is
|
|
|
|
|
|
def _build_service_path_dirs(project_root: Path | None = None) -> list[str]:
|
|
"""Build PATH directory list for service units, excluding non-existent dirs."""
|
|
if project_root is None:
|
|
project_root = PROJECT_ROOT
|
|
|
|
def _is_dir(path: Path) -> bool:
|
|
try:
|
|
return path.is_dir()
|
|
except OSError:
|
|
return False
|
|
|
|
candidates = []
|
|
venv_bin = project_root / "venv" / "bin"
|
|
if _is_dir(venv_bin):
|
|
candidates.append(str(venv_bin))
|
|
elif sys.prefix != sys.base_prefix:
|
|
candidates.append(str(Path(sys.prefix) / "bin"))
|
|
|
|
hermes_home = get_hermes_home()
|
|
extras = (project_root / "node_modules" / ".bin", hermes_home / "node" / "bin", hermes_home / "node_modules" / ".bin")
|
|
for extra in extras:
|
|
if _is_dir(extra):
|
|
candidates.append(str(extra))
|
|
return candidates
|
|
|
|
|
|
def _stable_service_working_dir() -> str:
|
|
"""WorkingDirectory that won't disappear under systemd (HERMES_HOME, else PROJECT_ROOT). cwd is
|
|
irrelevant to ``-m`` resolution, and a pinned transient checkout rots: systemd fails at CHDIR
|
|
(status=200) before Python loads, so the unit self-heal never runs and Restart=always crash-loops."""
|
|
try:
|
|
home = get_hermes_home()
|
|
if home and Path(home).is_dir():
|
|
return str(Path(home).resolve())
|
|
except Exception:
|
|
pass
|
|
return str(PROJECT_ROOT)
|
|
|
|
|
|
def _systemd_watchdog_seconds(hermes_home: str | Path | None = None) -> int:
|
|
"""Resolve the managed-overlay-aware watchdog setting for a service home."""
|
|
override_token = reset_home_override = None
|
|
if hermes_home is not None:
|
|
from hermes_constants import (reset_hermes_home_override, set_hermes_home_override)
|
|
override_token = set_hermes_home_override(hermes_home)
|
|
reset_home_override = reset_hermes_home_override
|
|
try:
|
|
config = load_gateway_config()
|
|
return coerce_systemd_watchdog_seconds(getattr(config, "systemd_watchdog_seconds", 0))
|
|
except Exception:
|
|
logger.debug("Could not resolve effective systemd watchdog configuration", exc_info=True)
|
|
return 0
|
|
finally:
|
|
if override_token is not None and reset_home_override is not None:
|
|
reset_home_override(override_token)
|
|
|
|
|
|
def _append_node_dir_for_service(path_entries: list[str], hermes_root: Path | None = None) -> None:
|
|
"""Append the Node dir a service unit should use: managed ``<hermes_root>/node`` (profile-scoped)
|
|
first — a unit survives reboots, so baking a shell-PATH Node is permanent breakage — else PATH lookup."""
|
|
from hermes_constants import (hermes_managed_node_tree_present, iter_hermes_node_dirs)
|
|
managed_node_present = hermes_managed_node_tree_present(hermes_root)
|
|
for directory in iter_hermes_node_dirs(hermes_root) if managed_node_present else ():
|
|
entry = str(directory)
|
|
try:
|
|
present = directory.is_dir()
|
|
except OSError:
|
|
present = False
|
|
if present and entry not in path_entries:
|
|
path_entries.append(entry)
|
|
|
|
# With managed Node present, consulting the invoker's PATH would make a system unit depend on who ran sudo.
|
|
if managed_node_present:
|
|
return
|
|
|
|
resolved_node = shutil.which("node")
|
|
if not resolved_node:
|
|
return
|
|
|
|
# Use the dir where node is FOUND, not the symlink target (~/.local/bin/node often links into one profile).
|
|
resolved_node_dir = str(Path(resolved_node).parent)
|
|
if resolved_node_dir not in path_entries:
|
|
path_entries.append(resolved_node_dir)
|
|
|
|
|
|
def _service_venv_dir() -> str:
|
|
"""VIRTUAL_ENV baked into service definitions: detected venv, else ``PROJECT_ROOT/venv``."""
|
|
detected_venv = _detect_venv_dir()
|
|
return str(detected_venv) if detected_venv else str(PROJECT_ROOT / "venv")
|
|
|
|
|
|
def generate_systemd_unit(system: bool = False, run_as_user: str | None = None) -> str:
|
|
python_path = get_python_path()
|
|
working_dir = _stable_service_working_dir()
|
|
venv_dir = _service_venv_dir()
|
|
|
|
path_entries = _build_service_path_dirs()
|
|
if not system:
|
|
# System units add managed Node once the TARGET user's home is known (not the sudo caller's).
|
|
_append_node_dir_for_service(path_entries)
|
|
|
|
# TimeoutStopSec must cover the full stop budget (cron drain + cleanup) or systemd SIGKILLs mid-drain.
|
|
restart_timeout = resolve_systemd_timeout_stop_sec(_get_restart_drain_timeout(), _get_cron_drain_timeout())
|
|
|
|
if system:
|
|
username, group_name, home_dir, uid = _system_service_identity(run_as_user)
|
|
hermes_home = _hermes_home_for_target_user(home_dir)
|
|
# Profile arg relative to the TARGET user's ~/.hermes when hermes_home lives under it.
|
|
target_root = Path(home_dir) / ".hermes"
|
|
try:
|
|
Path(hermes_home).resolve().relative_to(target_root.resolve())
|
|
profile_arg = _profile_arg(hermes_home, default_root=target_root)
|
|
except ValueError:
|
|
profile_arg = _profile_arg(hermes_home)
|
|
# Remap paths under the calling user's home (/root/) to the target user's so the service can read them.
|
|
python_path = _remap_path_for_user(python_path, home_dir)
|
|
working_dir = str(hermes_home) if hermes_home else _remap_path_for_user(working_dir, home_dir)
|
|
venv_dir = _remap_path_for_user(venv_dir, home_dir)
|
|
path_entries = [_remap_path_for_user(p, home_dir) for p in path_entries]
|
|
# Managed Node for the TARGET user's tree, prepended so it outranks remapped shell-PATH entries.
|
|
_target_node_entries: list[str] = []
|
|
_append_node_dir_for_service(_target_node_entries, Path(hermes_home) if hermes_home else None)
|
|
path_entries = [e for e in _target_node_entries if e not in path_entries] + path_entries
|
|
user_home = Path(home_dir)
|
|
identity_lines = f"User={username}\nGroup={group_name}\n"
|
|
# Restart-safe cron/Kanban workers cross `systemd-run --user`, which needs this user's manager;
|
|
# without the ordering the gateway and user@<uid>.service race at boot and the one-shot bus
|
|
# adoption in run_gateway() can miss (#104893).
|
|
ordering_lines = f"After=user@{uid}.service\nWants=user@{uid}.service\n"
|
|
env_lines = (
|
|
f'Environment="HOME={home_dir}"\n'
|
|
f'Environment="USER={username}"\n'
|
|
f'Environment="LOGNAME={username}"\n'
|
|
)
|
|
wanted_by = "multi-user.target"
|
|
else:
|
|
hermes_home = str(get_hermes_home().resolve())
|
|
profile_arg = _profile_arg(hermes_home)
|
|
user_home = Path.home()
|
|
identity_lines = env_lines = ordering_lines = ""
|
|
wanted_by = "default.target"
|
|
|
|
watchdog_seconds = _systemd_watchdog_seconds(hermes_home)
|
|
systemd_type, systemd_watchdog_directives = "simple", ""
|
|
if watchdog_seconds > 0:
|
|
systemd_type, systemd_watchdog_directives = "notify", f"NotifyAccess=main\nWatchdogSec={watchdog_seconds}s\n"
|
|
path_entries.extend(_build_user_local_paths(user_home, path_entries))
|
|
path_entries.extend(_build_wsl_interop_paths(path_entries))
|
|
path_entries.extend(["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"])
|
|
sane_path = ":".join(path_entries)
|
|
return f"""[Unit]
|
|
Description={SERVICE_DESCRIPTION}
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
{ordering_lines}StartLimitIntervalSec=0
|
|
|
|
[Service]
|
|
Type={systemd_type}
|
|
{systemd_watchdog_directives}{identity_lines}ExecStart={python_path} -m hermes_cli.main{f" {profile_arg}" if profile_arg else ""} gateway run
|
|
WorkingDirectory={working_dir}
|
|
{env_lines}Environment="PATH={sane_path}"
|
|
Environment="VIRTUAL_ENV={venv_dir}"
|
|
Environment="HERMES_HOME={hermes_home}"
|
|
Environment="HERMES_SUPERVISED_CHILD=1"
|
|
Restart=always
|
|
RestartSec=5
|
|
RestartForceExitStatus={GATEWAY_SERVICE_RESTART_EXIT_CODE}
|
|
SuccessExitStatus={GATEWAY_SERVICE_RESTART_EXIT_CODE}
|
|
RestartPreventExitStatus={GATEWAY_FATAL_CONFIG_EXIT_CODE}
|
|
KillMode=mixed
|
|
KillSignal=SIGTERM
|
|
ExecReload=/bin/kill -USR1 $MAINPID
|
|
ExecStopPost=-{python_path} -m gateway.cgroup_cleanup
|
|
TimeoutStopSec={restart_timeout}
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
|
|
[Install]
|
|
WantedBy={wanted_by}
|
|
"""
|
|
|
|
|
|
def _normalize_service_definition(text: str) -> str:
|
|
return "\n".join(line.rstrip() for line in text.strip().splitlines())
|
|
|
|
|
|
# Directives older systemd silently strips; ignored in stale-checks so such units aren't flagged forever.
|
|
_SYSTEMD_OPTIONAL_DIRECTIVES = ("RestartMaxDelaySec", "RestartSteps")
|
|
|
|
|
|
def _strip_optional_systemd_directives(text: str) -> str:
|
|
"""Remove systemd directives that older hosts silently drop."""
|
|
filtered = []
|
|
for line in text.splitlines():
|
|
stripped = line.strip()
|
|
is_directive = stripped and not stripped.startswith("#")
|
|
if not (is_directive and stripped.split("=", 1)[0].strip() in _SYSTEMD_OPTIONAL_DIRECTIVES):
|
|
filtered.append(line)
|
|
return "\n".join(filtered)
|
|
|
|
|
|
def _normalize_launchd_plist_for_comparison(text: str) -> str:
|
|
"""Normalize plist text for staleness checks, ignoring the PATH payload: the generated PATH is
|
|
captured from the invoking shell and varies across shells."""
|
|
import re
|
|
return re.sub(
|
|
r"(<key>PATH</key>\s*<string>)(.*?)(</string>)", r"\1__HERMES_PATH__\3",
|
|
_normalize_service_definition(text), flags=re.S,
|
|
)
|
|
|
|
|
|
def systemd_unit_is_current(system: bool = False) -> bool:
|
|
# HERMES_HOME sync chokepoint for every compare/regenerate path: under `sudo … --system` it is often
|
|
# stripped to /root/.hermes, so refresh would rewrite a correct unit and status warn forever.
|
|
# Idempotent; the os.environ mutation persists for later runtime reads (restart's PID/drain).
|
|
_sync_hermes_home_from_systemd_unit(system=system)
|
|
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
if not unit_path.exists():
|
|
return False
|
|
|
|
installed = unit_path.read_text(encoding="utf-8")
|
|
expected_user = _read_systemd_user_from_unit(unit_path) if system else None
|
|
expected = generate_systemd_unit(system=system, run_as_user=expected_user)
|
|
# Ignore directives older systemd drops (RestartMaxDelaySec, RestartSteps) to avoid a perpetual "outdated" flag.
|
|
norm = lambda text: _normalize_service_definition(_strip_optional_systemd_directives(text)) # noqa: E731
|
|
return norm(installed) == norm(expected)
|
|
|
|
|
|
def _temp_home_in_service_definition(definition: str) -> str | None:
|
|
"""Temp-dir HERMES_HOME baked into a systemd unit / launchd plist, or None. A temp home means a
|
|
test/E2E harness generated it; installing it leaves the gateway "running" but deaf to every platform."""
|
|
import re
|
|
import tempfile
|
|
candidates = re.findall(r'HERMES_HOME=([^"\n]+)', definition)
|
|
candidates += re.findall(r"<key>HERMES_HOME</key>\s*<string>(.*?)</string>", definition, flags=re.S)
|
|
temp_roots = {
|
|
Path(tempfile.gettempdir()).resolve(),
|
|
Path("/tmp"), Path("/var/tmp"), Path("/private/tmp"), Path("/private/var/tmp"), # no-tmp: ok — detects a temp HERMES_HOME in service definitions
|
|
}
|
|
for raw in candidates:
|
|
try:
|
|
resolved = Path(raw.strip().strip('"')).resolve()
|
|
except (OSError, ValueError):
|
|
continue
|
|
if any(resolved == root or root in resolved.parents for root in temp_roots):
|
|
return raw.strip()
|
|
return None
|
|
|
|
|
|
def _refuse_temp_home_service_write(definition: str, kind: str) -> bool:
|
|
"""Refuse (with guidance) when a service definition carries a temp HERMES_HOME."""
|
|
temp_home = _temp_home_in_service_definition(definition)
|
|
if temp_home is None:
|
|
return False
|
|
print(f"✗ Refusing to write the gateway {kind}: HERMES_HOME resolves to a temporary directory ({temp_home}).")
|
|
print(
|
|
" This usually means a test/E2E environment exported HERMES_HOME. "
|
|
"Unset it (or run from a clean shell) and retry."
|
|
)
|
|
return True
|
|
|
|
|
|
def refresh_systemd_unit_if_needed(system: bool = False) -> bool:
|
|
"""Rewrite the installed systemd unit when the generated definition has changed."""
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
if not unit_path.exists():
|
|
return False
|
|
|
|
# systemd_unit_is_current is the HERMES_HOME-sync chokepoint; its env mutation persists for the regenerate below.
|
|
if systemd_unit_is_current(system=system):
|
|
return False
|
|
|
|
expected_user = _read_systemd_user_from_unit(unit_path) if system else None
|
|
new_unit = generate_systemd_unit(system=system, run_as_user=expected_user)
|
|
|
|
# Test safety belt: the user unit path is under Path.home(), which conftest does NOT sandbox, and a
|
|
# pytest-tmp HERMES_HOME baked into the developer's real unit breaks their gateway on next reboot.
|
|
if not system and any(m in new_unit for m in ("/pytest-of-", '/hermes_test"', "/hermes_test/")):
|
|
return False
|
|
|
|
# Structural variant: refuse ANY temp-dir HERMES_HOME (manual E2E homes lack the pytest markers).
|
|
if _refuse_temp_home_service_write(new_unit, "systemd unit"):
|
|
return False
|
|
|
|
unit_path.write_text(new_unit, encoding="utf-8")
|
|
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
|
|
print(f"↻ Updated gateway {_service_scope_label(system)} service definition to match the current Hermes install")
|
|
return True
|
|
|
|
|
|
def _print_linger_enable_warning(username: str, detail: str | None = None, *, system: bool = False) -> None:
|
|
print()
|
|
if system:
|
|
print(f"⚠ Linger not enabled for {username} — cron and Kanban workers cannot start (no user D-Bus).")
|
|
else:
|
|
print("⚠ Linger not enabled — gateway may stop when you close this terminal.")
|
|
if detail:
|
|
print(f" Auto-enable failed: {detail}")
|
|
print()
|
|
print(" Enable it manually:" if system else " On headless servers (VPS, cloud instances) run:")
|
|
print(f" sudo loginctl enable-linger {username}")
|
|
print()
|
|
print(" Then restart the gateway:")
|
|
sudo, _, user_flag = _systemd_cli_bits(system)
|
|
print(f" {sudo}systemctl {user_flag}restart {get_service_name()}.service")
|
|
print()
|
|
|
|
|
|
def _ensure_linger_enabled(username: str | None = None, *, system: bool = False) -> bool:
|
|
"""Enable linger for *username* (default: the current user) when possible.
|
|
|
|
A user unit needs linger so the gateway survives logout. A system unit (``system=True``) needs
|
|
it for its ``User=`` so ``user@<uid>.service`` provides the D-Bus that ``systemd-run --user
|
|
--scope`` — every restart-safe cron/Kanban worker — connects to (#104893). Returns True only
|
|
when linger was enabled by this call.
|
|
"""
|
|
if is_termux() or not is_linux():
|
|
return False
|
|
|
|
if username is None:
|
|
import getpass
|
|
username = getpass.getuser()
|
|
enabled_msg = (
|
|
f"✓ Systemd linger is enabled for {username} (worker D-Bus available)" if system
|
|
else "✓ Systemd linger is enabled (service survives logout)"
|
|
)
|
|
if Path(f"/var/lib/systemd/linger/{username}").exists():
|
|
print(enabled_msg)
|
|
return False
|
|
|
|
linger_enabled, linger_detail = get_systemd_linger_status(username)
|
|
if linger_enabled is True:
|
|
print(enabled_msg)
|
|
return False
|
|
|
|
if not shutil.which("loginctl"):
|
|
_print_linger_enable_warning(username, linger_detail or "loginctl not found", system=system)
|
|
return False
|
|
|
|
if system:
|
|
print(f"Enabling linger for {username} so cron and Kanban workers can reach systemd-run --user...")
|
|
else:
|
|
print("Enabling linger so the gateway survives SSH logout...")
|
|
try:
|
|
result = _loginctl_enable_linger(username)
|
|
except Exception as e:
|
|
_print_linger_enable_warning(username, str(e), system=system)
|
|
return False
|
|
|
|
if result.returncode != 0:
|
|
_print_linger_enable_warning(username, _completed_process_detail(result) or linger_detail, system=system)
|
|
return False
|
|
print(f"✓ Enabled linger for {username}" if system else "✓ Linger enabled — gateway will persist after logout")
|
|
return True
|
|
|
|
|
|
def _ensure_system_service_linger(username: str) -> None:
|
|
"""Enable linger for the installed unit's ``User=`` (root included: restart-safe workers always cross
|
|
``systemd-run --user``, so a root gateway needs ``user@0.service`` just the same).
|
|
|
|
After a fresh enable, wait for the TARGET user's bus: logind starts ``user@<uid>.service``
|
|
asynchronously and ``--start-now`` boots the gateway immediately. A gateway that was already running
|
|
keeps its bus-less environment and ``systemctl start`` on an active unit is a no-op — say so rather
|
|
than let the repair silently not take."""
|
|
if not _ensure_linger_enabled(username, system=True):
|
|
return
|
|
import pwd
|
|
uid = pwd.getpwnam(username).pw_uid # windows-footgun: ok — POSIX systemd helper, never invoked on Windows
|
|
if _wait_for_target_user_bus(uid):
|
|
print(f"✓ /run/user/{uid}/bus is up — cron and Kanban workers can use systemd-run --user")
|
|
else:
|
|
print(f"⚠ /run/user/{uid}/bus did not appear within 5s.")
|
|
print(f" Start the user manager: sudo systemctl start user@{uid}.service")
|
|
if _systemd_unit_is_active(system=True):
|
|
print(" The running gateway was started without a user D-Bus; restart it to pick one up:")
|
|
print(f" sudo systemctl restart {get_service_name()}.service")
|
|
|
|
|
|
def _select_systemd_scope(system: bool = False) -> bool:
|
|
return system or (get_systemd_unit_path(system=True).exists() and not get_systemd_unit_path(system=False).exists())
|
|
|
|
|
|
def _system_scope_wizard_would_need_root(system: bool = False) -> bool:
|
|
"""True when the wizard would trigger a system-scope operation as non-root — mirrors
|
|
``_select_systemd_scope`` so the dead-end is detected BEFORE prompting."""
|
|
if os.geteuid() == 0: # windows-footgun: ok — systemd scope wizard decision, never invoked on Windows
|
|
return False
|
|
return _select_systemd_scope(system=system)
|
|
|
|
|
|
def _print_system_scope_remediation(action: str) -> None:
|
|
"""Print remediation when the wizard skips a system-scope action because the user isn't root."""
|
|
print_warning(f"Gateway is installed as a system-wide service — {action} requires root.")
|
|
print_info(" Options:")
|
|
print_info(f" 1. {action.capitalize()} it this time:")
|
|
print_info(f" sudo systemctl {action} {get_service_name()}")
|
|
print_info(" 2. Switch to a per-user service (recommended for personal use):")
|
|
print_info(" sudo hermes gateway uninstall --system")
|
|
print_info(" hermes gateway install")
|
|
print_info(" hermes gateway start")
|
|
|
|
|
|
def _get_restart_drain_timeout() -> float:
|
|
"""Return the configured gateway restart drain timeout in seconds."""
|
|
raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip()
|
|
if not raw:
|
|
cfg = read_raw_config()
|
|
agent_cfg = cfg.get("agent", {}) if isinstance(cfg, dict) else {}
|
|
raw = str(agent_cfg.get("restart_drain_timeout", DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT))
|
|
return parse_restart_drain_timeout(raw)
|
|
|
|
|
|
def _agent_timeout_setting(env_var: str, key: str, parse) -> float:
|
|
"""``parse(env)`` when the env var is non-empty, else ``parse(agent.<key>)`` (None if unset)."""
|
|
env_raw = os.getenv(env_var)
|
|
if env_raw is not None and str(env_raw).strip() != "":
|
|
return parse(env_raw)
|
|
cfg = read_raw_config()
|
|
agent_cfg = cfg.get("agent", {}) if isinstance(cfg, dict) else {}
|
|
if isinstance(agent_cfg, dict) and key in agent_cfg:
|
|
return parse(agent_cfg.get(key))
|
|
return parse(None)
|
|
|
|
|
|
def _get_cron_drain_timeout() -> float:
|
|
"""Return the configured cron-only drain floor in seconds.
|
|
|
|
See #82161.
|
|
"""
|
|
return _agent_timeout_setting("HERMES_CRON_DRAIN_TIMEOUT", "cron_drain_timeout", parse_cron_drain_timeout)
|
|
|
|
|
|
def _get_restart_exit_wait_budget() -> float:
|
|
"""CLI wait for gateway exit after SIGUSR1 / self-restart (#77184)."""
|
|
return resolve_restart_exit_wait_budget(
|
|
# TimeoutStopSec must cover the full stop budget, not just restart_drain_timeout. Cron work can
|
|
# legally wait cron_drain_timeout plus cleanup reserve before interrupt/teardown, and systemd
|
|
# SIGKILLs if the unit's deadline is shorter (#94759). 30s of post-drain headroom is preserved on
|
|
# top, with a 60s floor.
|
|
_get_restart_drain_timeout(),
|
|
_agent_timeout_setting(
|
|
"HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout
|
|
),
|
|
)
|
|
|
|
|
|
def systemd_install(
|
|
force: bool = False,
|
|
system: bool = False,
|
|
run_as_user: str | None = None,
|
|
enable_on_startup: bool = True,
|
|
non_interactive: bool = False,
|
|
):
|
|
if system:
|
|
_require_root_for_system_service("install")
|
|
|
|
# Offer to remove legacy units first: alongside the new unit they flap-fight for the bot token.
|
|
if has_legacy_hermes_units():
|
|
print()
|
|
print_legacy_unit_warning()
|
|
print()
|
|
if non_interactive or prompt_yes_no("Remove the legacy unit(s) before installing?", True):
|
|
remove_legacy_hermes_units(interactive=False)
|
|
print()
|
|
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
scope_label = _service_scope_label(system)
|
|
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
|
|
|
|
# Existing system units already pin HERMES_HOME; adopt it before any regenerate.
|
|
if unit_path.exists():
|
|
_sync_hermes_home_from_systemd_unit(system=system)
|
|
|
|
if unit_path.exists() and not force:
|
|
if not systemd_unit_is_current(system=system):
|
|
print(f"↻ Repairing outdated {scope_label} systemd service at: {unit_path}")
|
|
refresh_systemd_unit_if_needed(system=system)
|
|
if enable_on_startup:
|
|
_run_systemctl(["enable", get_service_name()], system=system, check=True, timeout=30)
|
|
print(f"✓ {scope_label.capitalize()} service definition updated")
|
|
else:
|
|
print(f"Service already installed at: {unit_path}")
|
|
print("Use --force to reinstall")
|
|
# Same post-install guarantee as a fresh install: a repaired user unit must survive logout too.
|
|
configured_user = _read_systemd_user_from_unit(unit_path) if system else None
|
|
if configured_user:
|
|
_ensure_system_service_linger(configured_user)
|
|
elif not system:
|
|
_ensure_linger_enabled()
|
|
return
|
|
|
|
unit_path.parent.mkdir(parents=True, exist_ok=True)
|
|
new_unit = generate_systemd_unit(system=system, run_as_user=run_as_user)
|
|
if _refuse_temp_home_service_write(new_unit, "systemd unit"):
|
|
return
|
|
print(f"Installing {scope_label} systemd service to: {unit_path}")
|
|
unit_path.write_text(new_unit, encoding="utf-8")
|
|
|
|
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
|
|
if enable_on_startup:
|
|
_run_systemctl(["enable", get_service_name()], system=system, check=True, timeout=30)
|
|
|
|
print()
|
|
print(f"✓ {scope_label.capitalize()} service {'installed and enabled' if enable_on_startup else 'installed'}!")
|
|
print()
|
|
print("Next steps:")
|
|
print(f" {sudo}hermes gateway start{scope_flag} # Start the service")
|
|
print(f" {sudo}hermes gateway status{scope_flag} # Check status")
|
|
print(f" journalctl {user_flag}-u {get_service_name()} -f # View logs")
|
|
print()
|
|
|
|
if system:
|
|
configured_user = _read_systemd_user_from_unit(unit_path)
|
|
if configured_user:
|
|
print(f"Configured to run as: {configured_user}")
|
|
_ensure_system_service_linger(configured_user)
|
|
else:
|
|
_ensure_linger_enabled()
|
|
|
|
print_systemd_scope_conflict_warning()
|
|
print_legacy_unit_warning()
|
|
|
|
|
|
def _systemd_scope_preamble(
|
|
action: str, system: bool, *, require_installed: bool = True, preflight_user: bool = False
|
|
) -> bool:
|
|
"""Resolve the effective scope, then enforce root (system) / user D-Bus reachability (user, when
|
|
``preflight_user``) and — when ``require_installed`` — that the unit exists. Returns the scope."""
|
|
system = _select_systemd_scope(system)
|
|
if system:
|
|
_require_root_for_system_service(action)
|
|
elif preflight_user:
|
|
# Fail fast with guidance when the user D-Bus session is unreachable (raises UserSystemdUnavailableError).
|
|
_preflight_user_systemd()
|
|
if require_installed:
|
|
_require_service_installed(action, system=system)
|
|
return system
|
|
|
|
|
|
def _systemd_unit_belongs_to_current_home(system: bool = False) -> bool:
|
|
"""False (with a warning) when the installed unit pins a HERMES_HOME other than this process's: the
|
|
service name then resolved to ANOTHER install's gateway, and stop/disable/unlink would take it down."""
|
|
_sync_hermes_home_from_systemd_unit(system=system) # sudo strips HERMES_HOME; adopt the unit's first
|
|
unit_home = _hermes_home_from_systemd_unit_file(system=system)
|
|
if unit_home is None or Path(unit_home).expanduser().resolve() == get_hermes_home().resolve():
|
|
return True
|
|
print_warning(
|
|
f"Refusing to remove {get_systemd_unit_path(system=system)}: it runs HERMES_HOME={unit_home}, "
|
|
f"but this process has HERMES_HOME={get_hermes_home()}"
|
|
)
|
|
return False
|
|
|
|
|
|
def systemd_uninstall(system: bool = False):
|
|
system = _systemd_scope_preamble("uninstall", system, require_installed=False)
|
|
if not _systemd_unit_belongs_to_current_home(system):
|
|
return
|
|
_run_systemctl(["stop", get_service_name()], system=system, check=False, timeout=90)
|
|
_run_systemctl(["disable", get_service_name()], system=system, check=False, timeout=30)
|
|
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
if unit_path.exists():
|
|
unit_path.unlink()
|
|
print(f"✓ Removed {unit_path}")
|
|
|
|
_run_systemctl(["daemon-reload"], system=system, check=True, timeout=30)
|
|
print(f"✓ {_service_scope_label(system).capitalize()} service uninstalled")
|
|
|
|
|
|
def _print_service_not_installed(system: bool) -> None:
|
|
sudo, scope_flag, _ = _systemd_cli_bits(system)
|
|
print("✗ Gateway service is not installed")
|
|
print(f" Run: {sudo}hermes gateway install{scope_flag}")
|
|
|
|
|
|
def _require_service_installed(action: str, system: bool = False) -> None:
|
|
if not get_systemd_unit_path(system=system).exists():
|
|
_print_service_not_installed(system)
|
|
sys.exit(1)
|
|
|
|
|
|
def systemd_start(system: bool = False):
|
|
system = _systemd_scope_preamble("start", system, preflight_user=True)
|
|
# HERMES_HOME sync happens in refresh's systemd_unit_is_current gate; the unit is guaranteed to exist here.
|
|
refresh_systemd_unit_if_needed(system=system)
|
|
_run_systemctl(["start", get_service_name()], system=system, check=True, timeout=30)
|
|
print(f"✓ {_service_scope_label(system).capitalize()} service started")
|
|
|
|
|
|
def systemd_stop(system: bool = False):
|
|
system = _systemd_scope_preamble("stop", system)
|
|
_sync_hermes_home_from_systemd_unit(system=system)
|
|
_mark_planned_stop()
|
|
try:
|
|
_run_systemctl(["stop", get_service_name()], system=system, check=True, timeout=90)
|
|
except subprocess.TimeoutExpired:
|
|
print(
|
|
f"Gateway {_service_scope_label(system)} service is still stopping after 90s; "
|
|
"check `hermes gateway status` or logs for final shutdown state."
|
|
)
|
|
return
|
|
print(f"✓ {_service_scope_label(system).capitalize()} service stopped")
|
|
|
|
|
|
def systemd_restart(system: bool = False):
|
|
system = _systemd_scope_preamble("restart", system, preflight_user=True)
|
|
# HERMES_HOME sync happens in refresh's systemd_unit_is_current gate; its os.environ mutation
|
|
# persists for the get_running_pid / drain-timeout reads below.
|
|
refresh_systemd_unit_if_needed(system=system)
|
|
from gateway.status import get_running_pid
|
|
pid = get_running_pid() or _systemd_main_pid(system=system)
|
|
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
|
|
# Event loop provably dead: SIGUSR1 can't drain it, so bounded SIGTERM → SIGKILL and let systemd relaunch.
|
|
print(
|
|
# Health probe says the event loop is provably dead (#81642): SIGUSR1 can never drain it, so the
|
|
# graceful wait below would burn the full budget. A busy-but-alive gateway (fresh heartbeat)
|
|
# never takes this path — its in-flight work, including the #86684 cron drain floor, keeps the
|
|
# full graceful budget.
|
|
# Health probe says the event loop is provably dead (#81642): the gateway cannot process a
|
|
# graceful shutdown, so waiting the full drain budget only stalls the restart (and `hermes
|
|
# update` behind it) for 180s. Bounded escalation instead: SIGTERM grace → SIGKILL → proceed,
|
|
# ~10s worst case. Never taken for a busy-but-alive gateway — a fresh heartbeat keeps the drain
|
|
# path (and the #86684 cron drain floor) fully intact.
|
|
f"⚠ Gateway PID {pid} event loop is unresponsive — "
|
|
"skipping graceful drain and forcing a bounded stop..."
|
|
)
|
|
_escalate_wedged_gateway(pid)
|
|
svc = get_service_name()
|
|
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
|
|
_run_systemctl(["restart", svc], system=system, check=False, timeout=90)
|
|
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
|
|
return
|
|
if pid is not None:
|
|
service_action = _systemd_graceful_restart_action(system, pid)
|
|
if service_action:
|
|
_systemd_reset_and_run(service_action, system=system, previous_pid=pid)
|
|
return
|
|
|
|
if _recover_pending_systemd_restart(system=system, previous_pid=pid):
|
|
return
|
|
_systemd_reset_and_run("restart", system=system, previous_pid=pid)
|
|
|
|
|
|
def _systemd_graceful_restart_action(system: bool, pid: int) -> str | None:
|
|
"""SIGUSR1-drain the live gateway ``pid``; return the follow-up ``systemctl`` verb (``"start"`` /
|
|
``"restart"``) the caller must still issue, or None when systemd already owns the relaunch."""
|
|
scope_label = _service_scope_label(system).capitalize()
|
|
# Graceful in-band restart, mirroring the systemd branch. Previously this sent a bare SIGTERM and waited
|
|
# ``_get_restart_drain_timeout()`` — which defaults to 0, so the wait could never succeed and every
|
|
# restart fell through to ``kickstart -k``. A bare SIGTERM also leaves ``restart_requested`` False, so
|
|
# the gateway exits 1 instead of 75 and reports itself to chat as "shutting down" rather than
|
|
# "restarting", losing the resume_pending handoff. SIGUSR1 is the drain-aware path: refuse new turns,
|
|
# wait for in-flight work (``agent.restart_after_turn_timeout``), then stop() within
|
|
# ``agent.restart_drain_timeout``. The wait budget must cover BOTH phases plus headroom (#77184) — the
|
|
# raw drain timeout covers only the second. Announce the wait BEFORE it runs: it can last the full
|
|
# budget while the old gateway finishes in-flight agent runs, and it streams into surfaces with no other
|
|
# feedback — the desktop updater's live output most of all, where a silent stop here reads as "update
|
|
# stuck" (#44515).
|
|
wait_budget = _get_restart_exit_wait_budget()
|
|
print(
|
|
f"⏳ {scope_label} service restarting gracefully (PID {pid}) — "
|
|
f"waiting up to {wait_budget:.0f}s for in-flight turns + drain..."
|
|
)
|
|
from hermes_cli.update_cmd_drain_report import drain_progress_reporter
|
|
if not _graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)):
|
|
print(f"⚠ Graceful restart did not complete within {int(wait_budget)}s; forcing a service restart...")
|
|
return "restart"
|
|
|
|
# Exit 75 hands restart ownership to systemd; observe that replacement rather than restarting again.
|
|
replacement_observed: list[bool] = []
|
|
if _wait_for_systemd_service_restart(system=system, previous_pid=pid, replacement_observed=replacement_observed):
|
|
return None
|
|
if replacement_observed or _systemd_service_is_start_limited(system=system):
|
|
return None
|
|
|
|
# A replacement may have started but not reached runtime readiness in time; never stop that generation.
|
|
props = _read_systemd_unit_properties(system=system)
|
|
if not props:
|
|
return None
|
|
replacement_pid = _systemd_main_pid_from_props(props)
|
|
if (
|
|
props.get("ActiveState") in {"active", "activating", "reloading"}
|
|
or props.get("SubState") == "auto-restart"
|
|
or (replacement_pid is not None and replacement_pid != pid)
|
|
):
|
|
return None
|
|
|
|
print("⚠ Systemd did not relaunch the gateway after its graceful exit; starting the inactive service...")
|
|
# ``start`` is intentionally idempotent: a replacement appearing after the snapshot must not be stopped.
|
|
return "start"
|
|
|
|
|
|
def _systemd_reset_and_run(action: str, *, system: bool, previous_pid) -> None:
|
|
"""``reset-failed`` then ``systemctl <action>``, then wait for the relaunch. Start-limit
|
|
rejection prints the wait hint instead of raising; a 90s timeout prints where to look."""
|
|
svc = get_service_name()
|
|
_run_systemctl(["reset-failed", svc], system=system, check=False, timeout=30)
|
|
try:
|
|
_run_systemctl([action, svc], system=system, check=True, timeout=90)
|
|
except subprocess.CalledProcessError as exc:
|
|
if _systemd_error_indicates_start_limit(exc) or _systemd_service_is_start_limited(system=system):
|
|
_print_systemd_start_limit_wait(system=system)
|
|
return
|
|
raise
|
|
except subprocess.TimeoutExpired:
|
|
print(
|
|
f"Gateway {_service_scope_label(system)} service is still restarting after 90s; "
|
|
"check `hermes gateway status` or logs for final state."
|
|
)
|
|
return
|
|
_wait_for_systemd_service_restart(system=system, previous_pid=previous_pid)
|
|
|
|
|
|
def systemd_status(deep: bool = False, system: bool = False, full: bool = False):
|
|
system = _select_systemd_scope(system)
|
|
unit_path = get_systemd_unit_path(system=system)
|
|
svc = get_service_name()
|
|
scope_label = _service_scope_label(system).capitalize()
|
|
sudo, scope_flag, user_flag = _systemd_cli_bits(system)
|
|
|
|
if not unit_path.exists():
|
|
_print_service_not_installed(system)
|
|
return
|
|
|
|
if has_conflicting_systemd_units():
|
|
print_systemd_scope_conflict_warning()
|
|
print()
|
|
|
|
if has_legacy_hermes_units():
|
|
print_legacy_unit_warning()
|
|
print()
|
|
|
|
if not systemd_unit_is_current(system=system):
|
|
print("⚠ Installed gateway service definition is outdated")
|
|
print(f" Run: {sudo}hermes gateway restart{scope_flag} # auto-refreshes the unit")
|
|
print()
|
|
|
|
status_cmd = ["status", svc, "--no-pager"] + (["-l"] if full else [])
|
|
_run_systemctl(status_cmd, system=system, capture_output=False, timeout=10)
|
|
result = _run_systemctl(["is-active", svc], system=system, timeout=10, **_CAPTURE_TEXT)
|
|
if result.stdout.strip() == "active":
|
|
print(f"✓ {scope_label} gateway service is running")
|
|
else:
|
|
print(f"✗ {scope_label} gateway service is stopped")
|
|
print(f" Run: {sudo}hermes gateway start{scope_flag}")
|
|
|
|
configured_user = _read_systemd_user_from_unit(unit_path) if system else None
|
|
if configured_user:
|
|
print(f"Configured to run as: {configured_user}")
|
|
|
|
_print_runtime_health()
|
|
|
|
unit_props = _read_systemd_unit_properties(system=system)
|
|
active_state = unit_props.get("ActiveState", "")
|
|
result_code = unit_props.get("Result", "")
|
|
if active_state == "activating" and unit_props.get("SubState", "") == "auto-restart":
|
|
print(" ⏳ Restart pending: systemd is waiting to relaunch the gateway")
|
|
elif _systemd_unit_is_start_limited(unit_props):
|
|
print(" ⏳ Restart pending: systemd is temporarily rate-limiting starts")
|
|
print(f" Run after the start-limit window expires: {sudo}hermes gateway restart{scope_flag}")
|
|
print(f" Or clear it manually: systemctl {user_flag}reset-failed {svc}")
|
|
elif active_state == "failed" and unit_props.get("ExecMainStatus", "") == str(GATEWAY_SERVICE_RESTART_EXIT_CODE):
|
|
print(" ⚠ Planned restart is stuck in systemd failed state (exit 75)")
|
|
print(f" Run: systemctl {user_flag}reset-failed {svc} && {sudo}hermes gateway start{scope_flag}")
|
|
elif active_state == "failed" and result_code:
|
|
print(f" ⚠ Systemd unit result: {result_code}")
|
|
|
|
if system:
|
|
print("✓ System service starts at boot without requiring systemd linger")
|
|
else:
|
|
linger_enabled, linger_detail = get_systemd_linger_status()
|
|
if linger_enabled is True:
|
|
print("✓ Systemd linger is enabled (service survives logout)")
|
|
elif linger_enabled is False:
|
|
print("⚠ Systemd linger is disabled (gateway may stop when you log out)")
|
|
print(" Run: sudo loginctl enable-linger $USER")
|
|
elif deep:
|
|
print(f"⚠ Could not verify systemd linger ({linger_detail})")
|
|
print(" If you want the gateway user service to survive logout, run:")
|
|
print(" sudo loginctl enable-linger $USER")
|
|
|
|
if deep:
|
|
print()
|
|
print("Recent logs:")
|
|
log_cmd = ["journalctl"] + ([] if system else ["--user"]) + ["-u", svc, "-n", "20", "--no-pager"]
|
|
if full:
|
|
log_cmd.append("-l")
|
|
subprocess.run(log_cmd, timeout=10)
|
|
|
|
|
|
# =============================================================================
|
|
# Launchd (macOS)
|
|
# =============================================================================
|
|
|
|
|
|
def get_launchd_label() -> str:
|
|
"""Return the launchd service label, scoped per profile."""
|
|
suffix = _profile_suffix()
|
|
return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway"
|
|
|
|
|
|
# Cached launchd domain — probe once per process invocation.
|
|
_resolved_launchd_domain: str | None = None
|
|
|
|
|
|
def _probe_launchd_domain_for_label(label: str) -> str:
|
|
"""Launchd domain managing ``label`` (uncached): ``gui/<uid>`` (Aqua), then ``user/<uid>``
|
|
(Background/SSH), else the ``launchctl managername`` heuristic. Sibling profiles may live in
|
|
different domains, so never reuse the cached ``_launchd_domain()`` for another label."""
|
|
uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
|
|
gui_domain, user_domain = f"gui/{uid}", f"user/{uid}"
|
|
|
|
launchctl_errors = (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError)
|
|
for domain in (gui_domain, user_domain):
|
|
try:
|
|
subprocess.run(["launchctl", "print", f"{domain}/{label}"], check=True, timeout=5, capture_output=True)
|
|
return domain
|
|
except launchctl_errors:
|
|
pass
|
|
|
|
# Not loaded anywhere: Aqua → gui/<uid>; anything else (Background, loginwindow) → user/<uid>,
|
|
# the pre-probing default and the recommended domain on macOS 26+.
|
|
try:
|
|
result = subprocess.run(["launchctl", "managername"], timeout=5, **_CAPTURE_TEXT)
|
|
if "Aqua" in (result.stdout or ""):
|
|
return gui_domain
|
|
except launchctl_errors:
|
|
pass
|
|
return user_domain
|
|
|
|
|
|
def _launchd_domain() -> str:
|
|
"""Domain managing the current profile's gateway; cached per process so start/stop/restart agree.
|
|
|
|
See #40831, #23387.
|
|
"""
|
|
global _resolved_launchd_domain
|
|
if _resolved_launchd_domain is None:
|
|
_resolved_launchd_domain = _probe_launchd_domain_for_label(get_launchd_label())
|
|
return _resolved_launchd_domain
|
|
|
|
|
|
# 125 ("Domain does not support specified action") and 3/113 ("Could not find service") all mean
|
|
# the job isn't loaded in the target domain: re-bootstrap the plist and retry.
|
|
_LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125})
|
|
|
|
# 5 (EIO) / persistent 125 mean either a stale still-registered label (recoverable: bootout +
|
|
# bootstrap, which `_launchctl_bootstrap()` tries first) or a domain that genuinely can't manage
|
|
# services (macOS 26+). Only when the retry ALSO fails do callers degrade to a detached process.
|
|
# launchctl returns 5 ("Input/output error") or a persistent 125 in two very different situations, so exit 5
|
|
# is NOT on its own proof the domain is broken: 1. See #42914. 2. Here launchd cannot supervise the gateway
|
|
# at all and we degrade to a detached background process (the `nohup hermes gateway run` workaround). See
|
|
# #23387.
|
|
_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125})
|
|
|
|
|
|
def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool:
|
|
"""True when launchctl failed because the job isn't loaded (retry bootstrap)."""
|
|
return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES
|
|
|
|
|
|
def _launchctl_domain_unsupported(returncode: int) -> bool:
|
|
"""True when launchctl can't manage the domain even after a fresh bootstrap (macOS 26+) — degrade to detached."""
|
|
return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
|
|
|
|
|
|
# EIO from `launchctl bootstrap` = label *already* registered (stale load); recoverable, not an unmanageable domain.
|
|
_LAUNCHCTL_BOOTSTRAP_EIO = 5
|
|
|
|
|
|
def _launchctl_bootstrap(domain: str, plist_path, label: str, *, timeout: int = 30) -> None:
|
|
"""Bootstrap a launchd job, recovering from a stale still-registered label (EIO 5). Without the
|
|
bootout + retry that case is misread as an unmanageable domain and degrades to detached, silently
|
|
losing auto-start and crash-restart."""
|
|
bootstrap = ["launchctl", "bootstrap", domain, str(plist_path)]
|
|
try:
|
|
subprocess.run(bootstrap, check=True, timeout=timeout)
|
|
except subprocess.CalledProcessError as exc:
|
|
if exc.returncode != _LAUNCHCTL_BOOTSTRAP_EIO:
|
|
raise
|
|
# Stale registration — bootout the leftover label and bootstrap once more.
|
|
# Captured: the bootout is best-effort (a drained job may already be
|
|
# unloaded), so its expected 3/113/125 stderr must not leak to the terminal.
|
|
subprocess.run(
|
|
["launchctl", "bootout", f"{domain}/{label}"],
|
|
check=False, timeout=timeout, **_CAPTURE_TEXT)
|
|
subprocess.run(bootstrap, check=True, timeout=timeout)
|
|
|
|
|
|
def _launchd_reload_log_path() -> Path:
|
|
"""Path the launchd reload watchdog tails for persistent-orphan detection."""
|
|
return get_hermes_home() / "logs" / "launchd-reload.log"
|
|
|
|
|
|
def _append_launchd_reload_log(message: str) -> None:
|
|
"""Append a timestamped line to the launchd reload log (best-effort)."""
|
|
path = _launchd_reload_log_path()
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
from datetime import datetime as _dt
|
|
stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z")
|
|
with path.open("a", encoding="utf-8") as fh:
|
|
fh.write(f"[{stamp}] {message}\n")
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _launchd_reload_budget() -> float:
|
|
"""Bootstrap retry window for a plist reload: the failure happens while the old gateway is still
|
|
draining (default 180s), so size it to the drain timeout with a 30s floor."""
|
|
return max(30.0, _get_restart_drain_timeout())
|
|
|
|
|
|
def _launchctl_label_supervising_process(label: str) -> bool:
|
|
"""True when launchd knows ``label`` AND runs a process for it. ``launchctl list`` exits 0 for a
|
|
mere registered definition (``state = not running`` on macOS 26+), so a positive PID is required."""
|
|
try:
|
|
result = subprocess.run(["launchctl", "list", label], check=False, timeout=10, **_CAPTURE_TEXT)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return False
|
|
return result.returncode == 0 and _parse_launchd_pid_from_list_output(result.stdout) is not None
|
|
|
|
|
|
def _retry_launchctl_bootstrap_until_registered(
|
|
domain: str, plist_path, label: str, *, deadline: float
|
|
) -> bool:
|
|
"""Retry ``_launchctl_bootstrap`` until the label supervises a process or ``deadline`` passes. Under
|
|
load bootstrap can fail even after bootout, during a drain (default 180s) — ~10s is too short."""
|
|
attempt = 0
|
|
while True:
|
|
attempt += 1
|
|
try:
|
|
_launchctl_bootstrap(domain, plist_path, label, timeout=30)
|
|
if _launchctl_label_supervising_process(label):
|
|
return True
|
|
outcome = f"exited 0 but {domain}/{label} has no supervised process (launchctl list)"
|
|
except subprocess.CalledProcessError as exc:
|
|
outcome = f"failed (rc={exc.returncode}) for {domain}/{label}"
|
|
except subprocess.TimeoutExpired:
|
|
outcome = f"timed out for {domain}/{label}"
|
|
_append_launchd_reload_log(f"bootstrap attempt {attempt} {outcome} — retrying")
|
|
if time.monotonic() >= deadline:
|
|
return False
|
|
time.sleep(2)
|
|
|
|
|
|
# launchd-unsupported marker: written when the domain can't be managed (exit 5/125, macOS 26+) so
|
|
# `launchd_status()` can explain missing supervision; cleared on successful bootstrap/kickstart.
|
|
def _launchd_unsupported_marker_path() -> Path:
|
|
return get_hermes_home() / ".gateway-launchd-unsupported"
|
|
|
|
|
|
def _write_launchd_unsupported_marker() -> None:
|
|
"""Persist that launchd cannot supervise the gateway on this host."""
|
|
from datetime import datetime, timezone
|
|
payload = {
|
|
"written_at": datetime.now(timezone.utc).isoformat(),
|
|
"reason": "launchd domain unsupported (exit 5/125)",
|
|
}
|
|
with contextlib.suppress(OSError):
|
|
_launchd_unsupported_marker_path().write_text(json.dumps(payload), encoding="utf-8")
|
|
|
|
|
|
def _clear_launchd_unsupported_marker() -> None:
|
|
"""Clear the unsupported marker when launchd bootstrap succeeds."""
|
|
with contextlib.suppress(OSError):
|
|
_launchd_unsupported_marker_path().unlink(missing_ok=True)
|
|
|
|
|
|
def _launchd_unsupported_marker_exists() -> bool:
|
|
return _launchd_unsupported_marker_path().exists()
|
|
|
|
|
|
def _gateway_run_command() -> list[str]:
|
|
"""Build ``python -m hermes_cli.main [--profile X] gateway run --replace``, honoring the active profile."""
|
|
return [get_python_path(), "-m", "hermes_cli.main", *_profile_arg().split(), "gateway", "run", "--replace"]
|
|
|
|
|
|
def _timestamped_stderr_gateway_command(error_log: Path, *, external_supervisor: bool = False) -> list[str]:
|
|
"""Wrap gateway run so raw stderr lines are timestamped before file write. ``external_supervisor``
|
|
(launchd ProgramArguments only) adds ``--external-supervisor`` so ``hermes update`` hands back to
|
|
launchd, and drops ``--replace``: KeepAlive respawns would re-arm takeover, so two profiles sharing
|
|
a token would kill each other forever.
|
|
|
|
``external_supervisor=True`` is for launchd ProgramArguments only: the inner ``gateway run`` must carry
|
|
``--external-supervisor`` so ``hermes update`` sees the flag on the live grandchild argv and hands the
|
|
process back to launchd instead of starting a detached watcher (#86893 / #87005). The detached nohup
|
|
fallback stays unmarked.
|
|
Supervised starts also drop ``--replace`` (issue #79048): a launchd service is respawned by KeepAlive,
|
|
so takeover authority would be re-armed on every respawn — two profiles legitimately sharing one
|
|
platform token would each terminate the sibling, and launchd would revive the victim forever. Bounded
|
|
replacement is the lifecycle commands' job (``launchctl kickstart -k``, drain in ``launchd_restart()``,
|
|
bootout+bootstrap in install/refresh), which run before supervision resumes. Mirrors
|
|
``generate_systemd_unit``, whose ExecStart also runs ``gateway run`` without ``--replace``.
|
|
"""
|
|
inner = _gateway_run_command()
|
|
if external_supervisor:
|
|
inner = [part for part in inner if part != "--replace"]
|
|
if "--external-supervisor" not in inner:
|
|
inner.append("--external-supervisor")
|
|
return [get_python_path(), "-m", "hermes_cli.stderr_timestamp", "--error-log", str(error_log), "--", *inner]
|
|
|
|
|
|
def _spawn_detached_gateway() -> bool:
|
|
"""Launch the gateway detached (launchd fallback for macOS 26+). CLI-managed nohup equivalent:
|
|
stdout → gateway.log, timestamped stderr → gateway.error.log, PID via gateway.pid so stop/status work.
|
|
|
|
Used when launchctl can no longer bootstrap/kickstart the gateway on macOS 26+ (issue #23387). Mirrors
|
|
the `nohup hermes gateway run --replace` workaround but keeps it CLI-managed: stdout goes to
|
|
gateway.log, stderr is timestamped into gateway.error.log, and the PID is tracked via the gateway.pid
|
|
file that `run_gateway` writes, so stop/status/restart keep working.
|
|
"""
|
|
from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
|
|
log_dir = get_hermes_home() / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
with open(log_dir / "gateway.log", "ab") as out:
|
|
subprocess.Popen(
|
|
_timestamped_stderr_gateway_command(log_dir / "gateway.error.log"),
|
|
stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.DEVNULL,
|
|
**windows_detach_popen_kwargs(),
|
|
)
|
|
except OSError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool:
|
|
"""Start the gateway detached when launchd can't manage it; on failure print the manual workaround
|
|
and (by default) exit 1."""
|
|
from hermes_constants import display_hermes_home as _dhh
|
|
_write_launchd_unsupported_marker()
|
|
print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
|
|
if _spawn_detached_gateway():
|
|
print("✓ Started gateway as a background process instead")
|
|
print(" It will NOT auto-start at login or auto-restart on crash.")
|
|
print(f" Logs: {_dhh()}/logs/gateway.log")
|
|
print(" Stop it with: hermes gateway stop")
|
|
return True
|
|
print_error("Failed to start the gateway as a background process.")
|
|
print(f" Try manually: nohup hermes gateway run --replace > {_dhh()}/logs/gateway.log 2>&1 &")
|
|
if exit_on_failure:
|
|
sys.exit(1)
|
|
return False
|
|
|
|
|
|
def _launchd_degrade_or_raise(exc: subprocess.CalledProcessError, what: str) -> None:
|
|
"""Shared launchctl failure policy: domain unmanageable (5/125) → detached fallback; else re-raise.
|
|
|
|
A 5/125 exit is evidence about the *domain* only when launchd is not already supervising this job.
|
|
EIO (5) is ``launchctl bootstrap``'s answer for a label that is already loaded, so the ordinary
|
|
"reinstall/restart over the live gateway" case lands here with the service up and supervised.
|
|
Degrading there is not a graceful fallback: it writes the permanent launchd-unsupported marker and
|
|
starts a detached gateway *beside* the supervised one, and the marker makes
|
|
:func:`wait_for_launchd_gateway_supervision` answer True unconditionally — so no later
|
|
install/update can tell that nothing ties the gateway to launchd any more. A live supervised PID is
|
|
direct evidence this macOS does manage the job, so surface the failure instead of branding the host.
|
|
"""
|
|
if not _launchctl_domain_unsupported(exc.returncode):
|
|
raise exc
|
|
label = get_launchd_label()
|
|
if _launchctl_label_supervising_process(label):
|
|
print(f"⚠ {what} failed (exit {exc.returncode}), but launchd still supervises {label}")
|
|
print(" Not switching to the detached fallback — this host manages the job.")
|
|
print(" Apply the definition with: hermes gateway stop && hermes gateway install --force")
|
|
raise exc
|
|
_launchd_fallback_to_detached(f"{what} exit {exc.returncode}")
|
|
|
|
|
|
def generate_launchd_plist() -> str:
|
|
# Stable cwd anchor — never the volatile source checkout (same rot risk as systemd's WorkingDirectory).
|
|
working_dir = _stable_service_working_dir()
|
|
hermes_home = str(get_hermes_home().resolve())
|
|
log_dir = get_hermes_home() / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
label = get_launchd_label()
|
|
venv_dir = _service_venv_dir()
|
|
# launchd's default PATH misses Homebrew, nvm, cargo…; prepend venv/bin + node dirs (as in the
|
|
# systemd unit) so node stays resolvable even if the shell PATH changes, then the shell PATH.
|
|
priority_dirs = _build_service_path_dirs()
|
|
_append_node_dir_for_service(priority_dirs)
|
|
sane_path = ":".join(dict.fromkeys(priority_dirs + [p for p in os.environ.get("PATH", "").split(":") if p]))
|
|
|
|
# ProgramArguments (incl. --profile); the stderr wrapper keeps launchd restart semantics while timestamping stderr.
|
|
prog_args_xml = "\n ".join(
|
|
f"<string>{part}</string>"
|
|
for part in _timestamped_stderr_gateway_command(log_dir / "gateway.error.log", external_supervisor=True)
|
|
)
|
|
|
|
# Persist the configured RLIMIT_NOFILE floor: launchd defaults to soft 256, and every plist
|
|
# rewrite would otherwise strip a manual limit and reintroduce EMFILE crashes.
|
|
nofile_block = ""
|
|
try:
|
|
from hermes_cli.resource_limits import configured_nofile_soft_limit
|
|
nofile_target = configured_nofile_soft_limit()
|
|
except Exception:
|
|
nofile_target = None
|
|
if nofile_target:
|
|
nofile_block = f"""
|
|
<key>SoftResourceLimits</key>
|
|
<dict>
|
|
<key>NumberOfFiles</key>
|
|
<integer>{nofile_target}</integer>
|
|
</dict>
|
|
"""
|
|
|
|
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0">
|
|
<dict>
|
|
<key>Label</key>
|
|
<string>{label}</string>
|
|
|
|
<key>ProgramArguments</key>
|
|
<array>
|
|
{prog_args_xml}
|
|
</array>
|
|
|
|
<key>WorkingDirectory</key>
|
|
<string>{working_dir}</string>
|
|
|
|
<key>EnvironmentVariables</key>
|
|
<dict>
|
|
<key>PATH</key>
|
|
<string>{sane_path}</string>
|
|
<key>VIRTUAL_ENV</key>
|
|
<string>{venv_dir}</string>
|
|
<key>HERMES_HOME</key>
|
|
<string>{hermes_home}</string>
|
|
<key>HERMES_SUPERVISED_CHILD</key>
|
|
<string>1</string>
|
|
</dict>
|
|
|
|
<key>LimitLoadToSessionType</key>
|
|
<array>
|
|
<string>Aqua</string>
|
|
<string>Background</string>
|
|
</array>
|
|
|
|
<key>RunAtLoad</key>
|
|
<true/>
|
|
|
|
<key>KeepAlive</key>
|
|
<dict>
|
|
<key>SuccessfulExit</key>
|
|
<false/>
|
|
</dict>
|
|
|
|
<!-- SuccessfulExit=false parks a clean stop (exit 0), including gateway
|
|
EX_CONFIG 78 after stderr_timestamp maps it to 0 — launchd cannot
|
|
honor RestartPreventExitStatus, and KeepAlive=true respawned token
|
|
collisions forever (#89477). Exit 75 and crashes still relaunch.
|
|
ThrottleInterval raises launchd's default 10s minimum respawn interval
|
|
to 30s so a crash-looping gateway can't hammer launchd into a rapid
|
|
respawn storm; ExitTimeOut gives the gateway 25s of graceful-drain
|
|
headroom before launchd escalates from SIGTERM to SIGKILL on stop. -->
|
|
<key>ThrottleInterval</key>
|
|
<integer>30</integer>
|
|
|
|
<key>ExitTimeOut</key>
|
|
<integer>25</integer>
|
|
{nofile_block}
|
|
<key>StandardOutPath</key>
|
|
<string>{log_dir}/gateway.log</string>
|
|
|
|
<key>StandardErrorPath</key>
|
|
<string>{log_dir}/gateway.error.log</string>
|
|
</dict>
|
|
</plist>
|
|
"""
|
|
|
|
|
|
def launchd_plist_is_current() -> bool:
|
|
"""Check if the installed launchd plist matches the currently generated one."""
|
|
plist_path = get_launchd_plist_path()
|
|
if not plist_path.exists():
|
|
return False
|
|
installed = plist_path.read_text(encoding="utf-8")
|
|
norm = _normalize_launchd_plist_for_comparison
|
|
return norm(installed) == norm(generate_launchd_plist())
|
|
|
|
|
|
def _spawn_deferred_launchd_reload(
|
|
*, domain: str, label: str, target: str, plist_path: Path, gateway_pid: int
|
|
) -> bool:
|
|
"""Hand the bootout/bootstrap cycle to a transient ``launchctl submit`` job; True if spawned. The
|
|
helper waits for the OLD gateway to exit (bootstrap during drain fails EIO), then retries bootstrap
|
|
until ``launchctl list`` shows a positive PID or the drain budget elapses."""
|
|
reload_log_path = _launchd_reload_log_path()
|
|
with contextlib.suppress(OSError):
|
|
reload_log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Durable pre-bootout marker: distinguishes "helper never started" from "helper ran but failed".
|
|
_append_launchd_reload_log(f"Launchd reload helper started for {target}")
|
|
|
|
_reload_budget = int(_launchd_reload_budget())
|
|
q_target, q_label, q_log = shlex.quote(target), shlex.quote(label), shlex.quote(str(reload_log_path))
|
|
stamp = "$(date '+%Y-%m-%d %H:%M:%S %z')"
|
|
# Require a POSITIVE PID: `launchctl list` also exits 0 for a registered-but-not-running
|
|
# definition, and a crashed job reports `"PID" = -1` (mirrors _parse_launchd_pid_from_list_output).
|
|
listed = f"launchctl list {q_label} 2>/dev/null | grep -qE '\\\"PID\\\" = [0-9]+;'"
|
|
# Unique per reload so concurrent/repeated reloads never collide.
|
|
submit_label = f"{label}.reload.{os.getpid()}.{int(time.time())}"
|
|
reload_script = (
|
|
f"sleep 2; "
|
|
f"launchctl bootout {q_target} 2>/dev/null; "
|
|
# Wait for the OLD gateway to exit: bootout only SIGTERMs and every bootstrap during the drain fails EIO.
|
|
f"_wait_deadline=$(($(date +%s) + {_reload_budget})); "
|
|
f"while kill -0 {gateway_pid} 2>/dev/null; do if [ $(date +%s) -ge $_wait_deadline ]; then "
|
|
f" echo \"[{stamp}] old gateway pid {gateway_pid} still alive after {_reload_budget}s drain wait — bootstrapping anyway\" >> {q_log}; "
|
|
f" break; fi; sleep 1; done; "
|
|
# Let launchd finish unregistering the label after the process exits.
|
|
f"sleep 1; _deadline=$(($(date +%s) + {_reload_budget})); while :; do "
|
|
f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; "
|
|
f" if {listed}; then break; fi; "
|
|
f" echo \"[{stamp}] bootstrap not yet registered for {q_target} — retrying\" >> {q_log}; "
|
|
f" if [ $(date +%s) -ge $_deadline ]; then break; fi; sleep 2; done; "
|
|
f"if ! {listed}; then "
|
|
f" echo \"[{stamp}] FAILED launchd reload for {q_target} — service NOT registered after {_reload_budget}s of retries\" >> {q_log}; "
|
|
f"fi; "
|
|
# Submitted jobs stay registered after the script exits; removing our own label ends the one-shot job.
|
|
f"launchctl remove {shlex.quote(submit_label)} 2>/dev/null"
|
|
)
|
|
try:
|
|
# `launchctl submit` rather than setsid: setsid does NOT leave the launchd coalition that bootout kills.
|
|
# Spawn the reload helper via `launchctl submit` (a transient launchd one-shot job) instead of
|
|
# `start_new_session=True`. `start_new_session=True` only calls setsid(2), which creates a new POSIX
|
|
# session but does NOT move the child outside the launchd job's process coalition. When `launchctl
|
|
# bootout` fires on the gateway label, launchd terminates ALL processes in that coalition —
|
|
# including a setsid-detached child (#69098). `launchctl submit` creates a wholly independent
|
|
# transient launchd job that launchd manages separately from the gateway, so bootout of the gateway
|
|
# job cannot reach the helper.
|
|
subprocess.Popen(
|
|
[
|
|
"launchctl", "submit", "-l", submit_label, "-o", str(reload_log_path), "-e", str(reload_log_path),
|
|
"--", "/bin/bash", "-c", reload_script,
|
|
],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
except Exception as e:
|
|
# Fall through to in-process bootout/bootstrap: risky in the coalition, but better than a never-reloaded plist.
|
|
logger.warning("Deferred launchd reload could not be spawned: %s", e)
|
|
_append_launchd_reload_log(
|
|
f"FAILED to spawn launchd reload helper for {target}: {e} — falling back to in-process bootout/bootstrap"
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def refresh_launchd_plist_if_needed() -> bool:
|
|
"""Rewrite the installed plist when the generated one differs, then bootout/bootstrap so launchd
|
|
re-reads it immediately."""
|
|
plist_path = get_launchd_plist_path()
|
|
if not plist_path.exists() or launchd_plist_is_current():
|
|
return False
|
|
|
|
new_plist = generate_launchd_plist()
|
|
if _refuse_temp_home_service_write(new_plist, "launchd plist"):
|
|
return False
|
|
|
|
plist_path.write_text(new_plist, encoding="utf-8")
|
|
label = get_launchd_label()
|
|
domain = _launchd_domain()
|
|
target = f"{domain}/{label}"
|
|
|
|
# Inside the gateway's launchd process tree (agent self-update) a direct bootout kills THIS CLI
|
|
# before bootstrap runs, leaving the job unloaded with no KeepAlive.
|
|
try:
|
|
from gateway.status import get_running_pid
|
|
gateway_pid = get_running_pid()
|
|
except Exception:
|
|
gateway_pid = None
|
|
|
|
# POSIX ancestry is NOT a reliable "bootout will kill us" test (coalition membership survives
|
|
# reparenting), so always prefer the detached helper; in-process is only the spawn-failure fallback.
|
|
if (
|
|
gateway_pid is not None
|
|
and hasattr(os, "setsid") # POSIX-only; launchd is macOS so always true here
|
|
) and _spawn_deferred_launchd_reload(
|
|
domain=domain, label=label, target=target, plist_path=plist_path, gateway_pid=gateway_pid
|
|
):
|
|
print(
|
|
"↻ Updated gateway launchd service definition; reload deferred to "
|
|
"a transient launchd job (survives the bootout of this process)"
|
|
)
|
|
return True
|
|
|
|
# Bootout/bootstrap so launchd reads the new definition; bootstrap can fail silently under load
|
|
# during a drain, and KeepAlive can't revive an unregistered job.
|
|
# Captured: best-effort (the job may already be unloaded), keep expected noise off the terminal.
|
|
subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_CAPTURE_TEXT)
|
|
_reload_budget = _launchd_reload_budget()
|
|
# Wait out the old gateway's drain first so the budget isn't burned on guaranteed EIO ("already loaded").
|
|
if gateway_pid is not None and not _wait_for_pid_exit(gateway_pid, _reload_budget):
|
|
_append_launchd_reload_log(
|
|
f"old gateway pid {gateway_pid} still alive after "
|
|
f"{int(_reload_budget)}s drain wait — bootstrapping {target} anyway"
|
|
)
|
|
_deadline = time.monotonic() + _reload_budget
|
|
if not _retry_launchctl_bootstrap_until_registered(domain, plist_path, label, deadline=_deadline):
|
|
_append_launchd_reload_log(
|
|
f"FAILED launchd reload of {target} — service NOT registered after "
|
|
f"retrying for {int(_reload_budget)}s (in-process fallback path)"
|
|
)
|
|
logger.error(
|
|
"launchd reload of %s failed — service not registered after %ds of retries; see %s",
|
|
target, int(_reload_budget), _launchd_reload_log_path(),
|
|
)
|
|
return False
|
|
print("↻ Updated gateway launchd service definition to match the current Hermes install")
|
|
return True
|
|
|
|
|
|
def launchd_install(force: bool = False):
|
|
plist_path = get_launchd_plist_path()
|
|
|
|
if plist_path.exists() and not force:
|
|
if not launchd_plist_is_current():
|
|
print(f"↻ Repairing outdated launchd service at: {plist_path}")
|
|
if refresh_launchd_plist_if_needed():
|
|
print("✓ Service definition updated")
|
|
else:
|
|
# The plist was rewritten but launchd never registered it (or the write was refused):
|
|
# a success line here would hide an unloaded service with no KeepAlive.
|
|
from hermes_constants import display_hermes_home
|
|
print(
|
|
"⚠ Service definition could not be reloaded with launchd. "
|
|
"Run 'hermes gateway install --force' or check "
|
|
f"{display_hermes_home()}/logs/launchd-reload.log for details."
|
|
)
|
|
return
|
|
print(f"Service already installed at: {plist_path}")
|
|
print("Use --force to reinstall")
|
|
return
|
|
|
|
plist_path.parent.mkdir(parents=True, exist_ok=True)
|
|
new_plist = generate_launchd_plist()
|
|
if _refuse_temp_home_service_write(new_plist, "launchd plist"):
|
|
return
|
|
print(f"Installing launchd service to: {plist_path}")
|
|
plist_path.write_text(new_plist, encoding="utf-8")
|
|
|
|
try:
|
|
_launchctl_bootstrap(_launchd_domain(), plist_path, get_launchd_label(), timeout=30)
|
|
except subprocess.CalledProcessError as e:
|
|
_launchd_degrade_or_raise(e, "launchctl bootstrap")
|
|
return
|
|
|
|
print()
|
|
print("✓ Service installed and loaded!")
|
|
_clear_launchd_unsupported_marker()
|
|
print()
|
|
print("Next steps:")
|
|
print(" hermes gateway status # Check status")
|
|
from hermes_constants import display_hermes_home as _dhh
|
|
print(f" tail -f {_dhh()}/logs/gateway.log # View logs")
|
|
|
|
|
|
def launchd_uninstall():
|
|
plist_path = get_launchd_plist_path()
|
|
# Captured: uninstalling an already-unloaded job is fine — don't print Boot-out failed: 3.
|
|
subprocess.run(
|
|
["launchctl", "bootout", f"{_launchd_domain()}/{get_launchd_label()}"],
|
|
check=False, timeout=90, **_CAPTURE_TEXT)
|
|
if plist_path.exists():
|
|
plist_path.unlink()
|
|
print(f"✓ Removed {plist_path}")
|
|
print("✓ Service uninstalled")
|
|
|
|
|
|
def launchd_start():
|
|
plist_path = get_launchd_plist_path()
|
|
label = get_launchd_label()
|
|
|
|
# Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade)
|
|
if not plist_path.exists():
|
|
new_plist = generate_launchd_plist()
|
|
if _refuse_temp_home_service_write(new_plist, "launchd plist"):
|
|
sys.exit(1)
|
|
print("↻ launchd plist missing; regenerating service definition")
|
|
plist_path.parent.mkdir(parents=True, exist_ok=True)
|
|
plist_path.write_text(new_plist, encoding="utf-8")
|
|
if _launchd_bootstrap_and_kickstart(plist_path, label):
|
|
_launchd_ok("✓ Service started")
|
|
return
|
|
|
|
refresh_launchd_plist_if_needed()
|
|
try:
|
|
_launchctl_kickstart_current(label)
|
|
except subprocess.CalledProcessError as e:
|
|
if not _launchd_error_indicates_unloaded(e):
|
|
raise
|
|
# Job not loaded in this domain — re-bootstrap the plist and retry.
|
|
print("↻ launchd job was unloaded; reloading service definition")
|
|
if not _launchd_bootstrap_and_kickstart(plist_path, label):
|
|
return
|
|
_launchd_ok("✓ Service started")
|
|
|
|
|
|
def _launchctl_kickstart_current(label: str) -> None:
|
|
subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30)
|
|
|
|
|
|
def _launchd_bootstrap_and_kickstart(plist_path: Path, label: str) -> bool:
|
|
"""Bootstrap then kickstart; False after degrading to detached (domain unsupported). Other errors propagate."""
|
|
try:
|
|
_launchctl_bootstrap(_launchd_domain(), plist_path, label, timeout=30)
|
|
_launchctl_kickstart_current(label)
|
|
except subprocess.CalledProcessError as e:
|
|
_launchd_degrade_or_raise(e, "launchctl")
|
|
return False
|
|
return True
|
|
|
|
|
|
def _launchd_ok(message: str) -> None:
|
|
"""Print a launchd success line and clear the unsupported marker (an OS fix recovers automatically)."""
|
|
print(message)
|
|
_clear_launchd_unsupported_marker()
|
|
|
|
|
|
def launchd_stop():
|
|
target = f"{_launchd_domain()}/{get_launchd_label()}"
|
|
_mark_planned_stop()
|
|
# bootout unloads the definition so KeepAlive doesn't respawn; `hermes gateway start` re-bootstraps.
|
|
try:
|
|
# Captured: an already-unloaded job (3/113/125) is handled below, so launchctl's own
|
|
# "Boot-out failed: 3" must not print around the ✓ line; e.stderr stays on the raised error.
|
|
subprocess.run(["launchctl", "bootout", target], check=True, timeout=90, **_CAPTURE_TEXT)
|
|
except subprocess.CalledProcessError as e:
|
|
# Job already unloaded (3/113/125), or the domain can't be managed at all (5/125, macOS 26+
|
|
# detached-fallback process, issue #23387) — in both cases just fall through to the PID-based kill
|
|
# below.
|
|
if not (_launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(e.returncode)):
|
|
raise
|
|
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
|
|
print("✓ Service stopped")
|
|
|
|
|
|
def _wait_for_gateway_exit(timeout: float = 10.0, force_after: float | None = 5.0) -> bool:
|
|
"""Wait up to ``timeout`` s for the gateway (by gateway.pid, not launchd labels, so multiple
|
|
HERMES_HOMEs work) to exit; SIGKILL it after ``force_after`` s of graceful waiting."""
|
|
from gateway.status import get_process_start_time, get_running_pid
|
|
deadline = time.monotonic() + timeout
|
|
force_deadline = (time.monotonic() + force_after) if force_after is not None else None
|
|
force_sent = False
|
|
|
|
while time.monotonic() < deadline:
|
|
pid = get_running_pid()
|
|
if pid is None:
|
|
return True # Process exited cleanly.
|
|
|
|
if force_after is not None and not force_sent and time.monotonic() >= force_deadline:
|
|
# Grace period expired — force-kill the specific PID.
|
|
try:
|
|
terminate_pid(pid, force=True, expected_start_time=get_process_start_time(pid))
|
|
print(f"⚠ Gateway PID {pid} did not exit gracefully; sent SIGKILL")
|
|
except (ProcessLookupError, PermissionError, OSError):
|
|
return True # Already gone or we can't touch it.
|
|
force_sent = True
|
|
|
|
time.sleep(0.3)
|
|
|
|
# Timed out even after force-kill.
|
|
remaining_pid = get_running_pid()
|
|
if remaining_pid is not None:
|
|
print(f"⚠ Gateway PID {remaining_pid} still running after {timeout}s — restart may fail")
|
|
return False
|
|
return True
|
|
|
|
|
|
def _wait_for_tcp_port_free(host: str, port: int, *, timeout: float = 10.0) -> bool:
|
|
"""Wait until nothing accepts TCP connections on host:port.
|
|
|
|
PID exit is not enough on macOS: api_server disables SO_REUSEADDR, so a restart that wins
|
|
the race logs EADDRINUSE and keeps running with no API. Connection-refused means the
|
|
listener is gone; a timed-out connect is a live listener with a slow accept queue.
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=0.2):
|
|
pass
|
|
except ConnectionRefusedError:
|
|
return True
|
|
except TimeoutError:
|
|
pass # a slow accept queue is still a live listener
|
|
except OSError:
|
|
return True # unresolvable/unreachable address: nothing to wait for; the bind retry covers it
|
|
time.sleep(0.1)
|
|
return False
|
|
|
|
|
|
def _wait_for_api_server_port_free(*, timeout: float = 10.0) -> bool:
|
|
"""Wait for the configured api_server listen address to stop accepting.
|
|
|
|
Only when api_server is enabled: with the platform off, a foreign listener on the default
|
|
port is nobody's race and must not delay the restart."""
|
|
from gateway.config import Platform
|
|
from gateway.platforms.api_server import listen_address
|
|
pconfig = load_gateway_config().platforms.get(Platform.API_SERVER)
|
|
if pconfig is None or not pconfig.enabled:
|
|
return True
|
|
host, port = listen_address(pconfig.extra or {})
|
|
freed = _wait_for_tcp_port_free(host, port, timeout=timeout)
|
|
if not freed:
|
|
print(
|
|
f"⚠ {host}:{port} still accepting connections — "
|
|
"new api_server may fail to bind"
|
|
)
|
|
return freed
|
|
|
|
|
|
def _launchd_kickstart(label: str, domain: str) -> None:
|
|
"""``launchctl kickstart -k domain/label``; raises so callers own per-label failure accounting."""
|
|
subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True, timeout=90, **_CAPTURE_TEXT)
|
|
|
|
|
|
def _wait_for_launchd_service_pid(
|
|
label: str, old_pid: int | None, timeout: float = 10.0, *, domain: str
|
|
) -> bool:
|
|
"""Poll ``domain/label`` (0.5s) until it runs on a fresh PID or ``timeout`` passes — KeepAlive respawn
|
|
isn't instantaneous. launchctl ``TimeoutExpired`` propagates; callers own failure accounting."""
|
|
deadline = time.monotonic() + max(timeout, 0.5)
|
|
while True:
|
|
_loaded, pid = _launchd_print_service_pid(domain, label)
|
|
if pid is not None and pid > 0 and pid != old_pid:
|
|
return True
|
|
if time.monotonic() >= deadline:
|
|
return False
|
|
time.sleep(0.5)
|
|
|
|
|
|
def launchd_restart():
|
|
label = get_launchd_label()
|
|
domain = _launchd_domain()
|
|
target = f"{domain}/{label}"
|
|
from gateway.status import get_running_pid
|
|
try:
|
|
pid = get_running_pid()
|
|
if pid is not None and _request_gateway_self_restart(pid):
|
|
_launchd_ok("✓ Service restart requested")
|
|
return
|
|
if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
|
|
# Event loop provably dead: it can't process a graceful shutdown, so a full drain wait
|
|
# only stalls the restart (and `hermes update`). Bounded SIGTERM → SIGKILL, ~10s.
|
|
print(f"⚠ Gateway PID {pid} event loop is unresponsive — " "skipping drain and forcing a bounded stop...")
|
|
_escalate_wedged_gateway(pid)
|
|
pid = None
|
|
if pid is not None:
|
|
# Graceful in-band restart via SIGUSR1 (mirrors systemd); the budget covers both the idle wait
|
|
# and the drain. A bare SIGTERM would lose the resume_pending handoff. Announce BEFORE waiting:
|
|
# surfaces with no other feedback (desktop updater) read silence as "update stuck".
|
|
wait_budget = _get_restart_exit_wait_budget()
|
|
print(f"→ Stopping gateway (PID {pid}) — draining in-flight runs (up to {wait_budget:.0f}s)...")
|
|
from hermes_cli.update_cmd_drain_report import drain_progress_reporter
|
|
if _graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)):
|
|
# KeepAlive revives a planned exit, so do NOT kickstart (-k would kill the replacement) —
|
|
# but a clean exit doesn't prove supervision, so verify a replacement PID appears first.
|
|
if _wait_for_launchd_service_pid(label, pid, timeout=15.0, domain=domain):
|
|
_launchd_ok("✓ Service restart requested")
|
|
return
|
|
print("⚠ launchd did not revive the gateway after its graceful exit — forcing restart")
|
|
else:
|
|
print(f"⚠ Gateway drain timed out after {wait_budget:.0f}s — forcing launchd restart")
|
|
# Captured: an unloaded job (3/113/125) is the expected case below, which
|
|
# prints its own ↻ line — and e.stderr feeds the update_cmd failure diagnostic.
|
|
_wait_for_api_server_port_free()
|
|
subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90, **_CAPTURE_TEXT)
|
|
_launchd_ok("✓ Service restarted")
|
|
except subprocess.CalledProcessError as e:
|
|
if not _launchd_error_indicates_unloaded(e):
|
|
_launchd_degrade_or_raise(e, "launchctl kickstart")
|
|
return
|
|
# Job not loaded — bootstrap and start fresh
|
|
print("↻ launchd job was unloaded; reloading")
|
|
try:
|
|
# After a drain the job is usually still registered (bootstrap would hit EIO): boot it out first.
|
|
# Captured: best-effort (the job may already be unloaded after the drain),
|
|
# so an expected Boot-out failed: 3 must not leak past the ↻ line below.
|
|
subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_CAPTURE_TEXT)
|
|
plist_path = str(get_launchd_plist_path())
|
|
subprocess.run(["launchctl", "bootstrap", _launchd_domain(), plist_path], check=True, timeout=30)
|
|
subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
|
|
except subprocess.CalledProcessError as e2:
|
|
_launchd_degrade_or_raise(e2, "launchctl")
|
|
return
|
|
_launchd_ok("✓ Service restarted")
|
|
|
|
|
|
# KeepAlive relaunches at most ~once per 10s, so a self-restart leaves the label pid-less that long.
|
|
LAUNCHD_SUPERVISION_VERIFY_TIMEOUT = 20.0
|
|
|
|
|
|
def wait_for_launchd_gateway_supervision(
|
|
*,
|
|
timeout: float = LAUNCHD_SUPERVISION_VERIFY_TIMEOUT,
|
|
label: str | None = None,
|
|
poll_interval: float = 0.5,
|
|
) -> bool:
|
|
"""Poll launchd until it supervises a live gateway; True at once if the detached fallback is active.
|
|
``launchd_restart`` returns once the restart is *requested* (asynchronous), so it can't see a helper
|
|
dying before bootstrap or a ``launchctl bootstrap`` that exits 0 without registering.
|
|
|
|
The ``_request_gateway_self_restart`` branch hands the work to the running gateway and returns
|
|
immediately, and a plist reload is handed to a detached helper. Both are asynchronous, so a caller that
|
|
reads "returned without raising" as "the service is up" cannot see a helper that dies before its first
|
|
bootstrap (#88848) — nor a ``launchctl bootstrap`` that exits 0 without registering, which the reporter
|
|
measured on macOS 26.6.1.
|
|
Judge the outcome the way #80491 taught the helper to judge it: by a live supervised pid, never by an
|
|
exit code. :func:`_launchctl_label_supervising_process` is already that predicate, so this only adds
|
|
the wait.
|
|
"""
|
|
if _launchd_unsupported_marker_exists():
|
|
return True
|
|
|
|
label = label or get_launchd_label()
|
|
deadline = time.monotonic() + max(timeout, 0.0)
|
|
while True:
|
|
if _launchctl_label_supervising_process(label):
|
|
return True
|
|
if time.monotonic() >= deadline:
|
|
return False
|
|
time.sleep(max(poll_interval, 0.01))
|
|
|
|
|
|
def launchd_status(deep: bool = False):
|
|
plist_path = get_launchd_plist_path()
|
|
label = get_launchd_label()
|
|
try:
|
|
result = subprocess.run(["launchctl", "list", label], timeout=10, **_CAPTURE_TEXT)
|
|
service_listed = result.returncode == 0
|
|
list_output = result.stdout
|
|
except subprocess.TimeoutExpired:
|
|
service_listed = False
|
|
list_output = ""
|
|
|
|
# `launchctl list` exits 0 for any registered definition (even `state = not running`); only a PID proves a process.
|
|
launchd_pid = _parse_launchd_pid_from_list_output(list_output) if service_listed else None
|
|
|
|
# Hermes PID may be a detached fallback process; when launchd IS supervising both PIDs match — don't double-count.
|
|
from gateway.status import get_running_pid
|
|
fallback_pid = get_running_pid(cleanup_stale=False)
|
|
if launchd_pid is not None and fallback_pid == launchd_pid:
|
|
fallback_pid = None
|
|
|
|
# Marker from a 5/125 bootstrap/kickstart failure explains *why* launchd can't supervise.
|
|
launchd_unsupported = _launchd_unsupported_marker_exists()
|
|
|
|
print(f"Launchd plist: {plist_path}")
|
|
if launchd_plist_is_current():
|
|
print("✓ Service definition matches the current Hermes install")
|
|
else:
|
|
print("⚠ Service definition is stale relative to the current Hermes install")
|
|
print(" Run: hermes gateway start")
|
|
|
|
if not service_listed:
|
|
print("✗ Gateway service is not loaded")
|
|
print(" Service definition exists locally but launchd has not loaded it.")
|
|
print(" Run: hermes gateway start")
|
|
if fallback_pid:
|
|
print(f" Note: a detached gateway process is running (PID {fallback_pid})")
|
|
elif launchd_pid is not None:
|
|
print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})")
|
|
print(" Auto-start at login and auto-restart on crash are available.")
|
|
if launchd_unsupported:
|
|
print(" (launchd domain was previously unavailable but is now working)")
|
|
elif launchd_unsupported:
|
|
print("⚠ Gateway service is registered but launchd is not supervising it")
|
|
print(" launchd cannot manage the gateway on this macOS version.")
|
|
if fallback_pid:
|
|
print(f"✓ Detached fallback process is running (PID {fallback_pid})")
|
|
print(" Cron jobs will fire. Stop with: hermes gateway stop")
|
|
else:
|
|
print("✗ No fallback process is running")
|
|
print(" Run: hermes gateway start")
|
|
print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.")
|
|
else:
|
|
print("✓ Gateway service is registered with launchd")
|
|
print(list_output)
|
|
if fallback_pid:
|
|
print(f" Detached gateway process is running (PID {fallback_pid})")
|
|
|
|
if deep:
|
|
log_file = get_hermes_home() / "logs" / "gateway.log"
|
|
if log_file.exists():
|
|
print()
|
|
print("Recent logs:")
|
|
subprocess.run(["tail", "-20", str(log_file)], timeout=10)
|
|
|
|
|
|
# =============================================================================
|
|
# Gateway Runner
|
|
# =============================================================================
|
|
|
|
|
|
def _truthy_env(value: str | None) -> bool:
|
|
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _is_official_docker_checkout() -> bool:
|
|
return str(PROJECT_ROOT) == "/opt/hermes" and (PROJECT_ROOT / "docker" / "entrypoint.sh").is_file()
|
|
|
|
|
|
def _running_under_gateway_supervisor() -> bool:
|
|
"""True when this process IS the supervisor-launched gateway, so the conflict guard never wedges
|
|
the service into a respawn/refuse loop. Markers: systemd INVOCATION_ID, launchd XPC_SERVICE_NAME
|
|
(shells inherit "0"), s6 HERMES_S6_SUPERVISED_CHILD, or ``--external-supervisor``."""
|
|
return is_gateway_supervisor_process()
|
|
|
|
|
|
def named_profile_served_by_running_multiplexer(profile_name: str | None = None) -> bool:
|
|
"""True when a live default multiplexer already ticks this named profile (a satellite profile has no
|
|
gateway.pid; the multiplexer fires its jobs and serves its platforms). Defaults to the current profile.
|
|
|
|
See #97120.
|
|
"""
|
|
try:
|
|
suffix = profile_name if profile_name is not None else _current_profile_name()
|
|
except Exception:
|
|
return False
|
|
if not suffix or suffix == "default":
|
|
return False
|
|
|
|
try:
|
|
from hermes_constants import get_default_hermes_root
|
|
default_root = get_default_hermes_root()
|
|
except Exception:
|
|
return False
|
|
|
|
try:
|
|
from hermes_cli.gateway_multiplex_served import live_default_gateway_pid, recorded_served_profiles
|
|
if live_default_gateway_pid() is None:
|
|
return False
|
|
from hermes_cli.profiles import normalize_profile_name
|
|
# The live gateway's own record wins: the CLI process cannot see an env-only opt-in on the
|
|
# default profile (`hermes -p X` loads X's .env) and a config edit after start is not live yet.
|
|
# Only a record without the key (pre-multiplex writer) falls through to config derivation.
|
|
recorded = recorded_served_profiles(default_root)
|
|
if recorded is not None:
|
|
return normalize_profile_name(suffix) in {normalize_profile_name(p) for p in recorded}
|
|
|
|
# No record (older gateway): only an EXPLICIT opt-in counts. The unset default is settled by
|
|
# the gateway at boot (it may have stayed standalone); a CLI process must not guess it on.
|
|
from hermes_cli.gateway_multiplex_mode import explicit_multiplex_flag
|
|
return explicit_multiplex_flag(default_root) is True # a multiplexer serves every named profile
|
|
except Exception:
|
|
logger.debug("Multiplexer-serving probe failed", exc_info=True)
|
|
return False
|
|
|
|
|
|
def _served_profile_needs_no_service() -> bool:
|
|
"""Print the "already served" note and return True when a setup flow must not install a standalone
|
|
service: a live multiplexing default gateway already serves this named profile, so the unit/plist it
|
|
would register can only sit dead (the start guard refuses it) or double-bind its platforms.
|
|
Shared by ``hermes setup gateway`` / ``hermes setup`` / ``hermes import`` (``ensure_gateway_service``)
|
|
and the ``hermes gateway setup`` wizard. See #111958."""
|
|
if not named_profile_served_by_running_multiplexer():
|
|
return False
|
|
print_success(
|
|
f"Profile '{_current_profile_name()}' is already served by the default multiplexer."
|
|
)
|
|
print_info(" (served now by the running multiplexed gateway — add its bot token and it connects)")
|
|
print_info(" No standalone gateway service was installed or started.")
|
|
return True
|
|
|
|
|
|
def _named_profile_refused_under_multiplexer(force: bool = False) -> bool:
|
|
"""Print the served-profile refusal and return True when a named-profile gateway must not start:
|
|
a multiplexing default gateway already serves it (a second one would double-bind its platforms: two
|
|
pollers on one token, port fights). ``--force`` overrides. Shared by ``run`` and the service verbs
|
|
(``start``/``install``/``restart``): a refusal only inside ``gateway run`` leaves the service manager
|
|
to discover it — systemd parks the unit on exit 78 while the CLI prints "started"; launchd
|
|
(KeepAlive, no exit-status gating) respawns it every ThrottleInterval forever."""
|
|
if force:
|
|
return False
|
|
try:
|
|
suffix = _current_profile_name()
|
|
except Exception:
|
|
return False
|
|
if not named_profile_served_by_running_multiplexer():
|
|
return False
|
|
|
|
print_error(
|
|
f"The default gateway is running as a profile multiplexer and already "
|
|
f"serves profile '{suffix}'."
|
|
)
|
|
print(
|
|
" When gateway.multiplex_profiles is on, the default gateway is the\n"
|
|
" single inbound process for every profile. Starting a separate\n"
|
|
" gateway for this profile would double-bind its platforms (two\n"
|
|
" pollers on one bot token, port conflicts).\n"
|
|
)
|
|
print(" Manage the multiplexer instead (from the default profile):")
|
|
print()
|
|
print(" hermes gateway restart")
|
|
print()
|
|
print(" Pass --force to start a separate profile gateway anyway (not")
|
|
print(" recommended while the multiplexer is running).")
|
|
return True
|
|
|
|
|
|
def _guard_named_profile_under_multiplexer(force: bool = False) -> None:
|
|
"""Exit-78 form of ``_named_profile_refused_under_multiplexer`` for the CLI entry points."""
|
|
if not _named_profile_refused_under_multiplexer(force=force):
|
|
return
|
|
# EX_CONFIG, not 1: the refusal is decided purely by config, so it is permanent. The systemd unit
|
|
# (Restart=always, StartLimitIntervalSec=0) relies on RestartPreventExitStatus=78 as its only
|
|
# backstop — exit 1 turned a correct refusal into an unbounded restart loop; s6 maps 78 to
|
|
# "permanent failure" too.
|
|
# This refusal is decided entirely by configuration (multiplex_profiles plus the allowlist), so it is
|
|
# permanent: no number of retries can change the answer. Exiting 1 made it look transient to a service
|
|
# manager -- and the systemd unit this module generates pairs Restart=always/RestartSec=5 with
|
|
# StartLimitIntervalSec=0, deliberately trading systemd's generic start-rate limiter for the specific
|
|
# RestartPreventExitStatus=GATEWAY_FATAL_CONFIG_EXIT_CODE backstop declared beside it. Returning 1 left
|
|
# that backstop unarmed with the limiter already off, so a correct refusal became an unbounded restart
|
|
# loop. 78 also reaches the s6 finish script's 125 "permanent failure" translation (see #51228), the
|
|
# same path the other fatal-config exits take.
|
|
sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
|
|
|
|
|
|
def _guard_supervised_gateway_conflict(force: bool = False) -> None:
|
|
"""Refuse a foreground gateway when a service manager already supervises one: a shell-launched run
|
|
becomes a second dispatcher that escapes the cgroup, survives ``systemctl restart``, and writes the
|
|
shared kanban DB concurrently (multi-writer SQLite WAL corruption). ``--force`` starts anyway.
|
|
|
|
See #35240.
|
|
"""
|
|
if force or _running_under_gateway_supervisor():
|
|
return
|
|
try:
|
|
snapshot = get_gateway_runtime_snapshot()
|
|
except Exception:
|
|
logger.debug("Supervised-gateway conflict probe failed", exc_info=True)
|
|
return
|
|
if not (snapshot.service_installed and snapshot.service_running):
|
|
return
|
|
|
|
print_error(f"A gateway is already running under {snapshot.manager} for this profile.")
|
|
print(
|
|
" Starting another one from a shell leaves an orphan dispatcher that\n"
|
|
" escapes the service, survives restarts, and writes to the same kanban\n"
|
|
" DB concurrently — which can corrupt it. Restart the supervised gateway\n"
|
|
" instead:"
|
|
)
|
|
print()
|
|
print(" hermes gateway restart")
|
|
print()
|
|
print(
|
|
" Pass --force to start a foreground gateway anyway (not recommended\n"
|
|
" while the service is running)."
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
def _guard_existing_gateway_process_conflict(replace: bool = False) -> None:
|
|
"""Cheap PID-file preflight before the expensive ``gateway.run`` import (the authoritative lock check):
|
|
supervisor loops re-running bare ``gateway run`` burned memory on plugin discovery just to fail
|
|
"already running". Same user-facing contract; never scans other HERMES_HOME roots."""
|
|
if replace or _running_under_gateway_supervisor():
|
|
return
|
|
try:
|
|
from gateway.status import get_running_pid
|
|
pid = get_running_pid()
|
|
except Exception:
|
|
logger.debug("Existing-gateway process probe failed", exc_info=True)
|
|
return
|
|
if pid is None:
|
|
# get_running_pid() filters by the current profile's HERMES_HOME; warn if the PID file
|
|
# belongs to another profile (user switched profiles while the old gateway still runs).
|
|
try:
|
|
from gateway.status import _read_pid_record, _pid_record_belongs_to_current_profile
|
|
stale = _read_pid_record()
|
|
if stale is not None and not _pid_record_belongs_to_current_profile(stale):
|
|
logger.warning(
|
|
"PID file belongs to another profile (hermes_home=%s). "
|
|
"The old gateway may still be running under that profile.",
|
|
stale.get("hermes_home", "<unknown>"),
|
|
)
|
|
except Exception:
|
|
pass
|
|
return
|
|
|
|
print_error(f"A gateway is already running (PID {pid}), so your bots are most likely online already.")
|
|
print(" Check with `hermes gateway status`.")
|
|
print(" To restart it: `hermes gateway restart`. To stop it: `hermes gateway stop`.")
|
|
print(" To replace it from here: `hermes gateway run --replace`.")
|
|
sys.exit(1)
|
|
|
|
|
|
def _guard_official_docker_root_gateway() -> None:
|
|
"""Refuse gateway startup when the official Docker privilege drop was bypassed."""
|
|
if not hasattr(os, "geteuid") or os.geteuid() != 0 or _truthy_env(os.getenv("HERMES_ALLOW_ROOT_GATEWAY")):
|
|
return
|
|
if not _is_official_docker_checkout():
|
|
return
|
|
|
|
print_error("Refusing to run the Hermes gateway as root inside the official Docker image.")
|
|
print(
|
|
" The image entrypoint normally drops privileges to the 'hermes' user. "
|
|
"If you override entrypoint in Docker Compose, include "
|
|
"/opt/hermes/docker/entrypoint.sh before the Hermes command."
|
|
)
|
|
print(
|
|
" Running the gateway as root can leave root-owned files in "
|
|
"$HERMES_HOME and break later non-root dashboard/gateway runs."
|
|
)
|
|
print(" Set HERMES_ALLOW_ROOT_GATEWAY=1 only if you intentionally accept this risk.")
|
|
sys.exit(1)
|
|
|
|
|
|
def _apply_startup_watchdog_config() -> None:
|
|
"""Idempotent backstop arming of the startup-liveness watchdog. Must run AFTER the conflict guards (a
|
|
--replace loser must not arm one). config.yaml gateway.startup_watchdog* is the user surface; env
|
|
vars bridge it because the argv fast-path arms before config loads, and explicit env wins. arm() is
|
|
idempotent, so a config timeout needs disarm+re-arm. GatewayRunner disarms once the loop is live."""
|
|
try:
|
|
from hermes_startup_watchdog import (
|
|
ENV_STARTUP_WATCHDOG, ENV_STARTUP_WATCHDOG_TIMEOUT_S, arm_startup_watchdog,
|
|
disarm_startup_watchdog, startup_watchdog_disabled,
|
|
)
|
|
_sw_timeout_bridged = False
|
|
try:
|
|
from hermes_cli.config import load_config as _sw_load_config
|
|
_gw_cfg = (_sw_load_config() or {}).get("gateway", {}) or {}
|
|
if ENV_STARTUP_WATCHDOG not in os.environ and not _gw_cfg.get("startup_watchdog", True):
|
|
os.environ[ENV_STARTUP_WATCHDOG] = "0"
|
|
_sw_timeout = _gw_cfg.get("startup_watchdog_timeout_seconds")
|
|
if ENV_STARTUP_WATCHDOG_TIMEOUT_S not in os.environ and _sw_timeout is not None:
|
|
os.environ[ENV_STARTUP_WATCHDOG_TIMEOUT_S] = str(_sw_timeout)
|
|
_sw_timeout_bridged = True
|
|
except Exception:
|
|
pass
|
|
if startup_watchdog_disabled():
|
|
disarm_startup_watchdog()
|
|
else:
|
|
if _sw_timeout_bridged:
|
|
disarm_startup_watchdog()
|
|
arm_startup_watchdog()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def _absorb_windows_console_controls() -> None:
|
|
"""Make a detached Windows gateway ignore console-control broadcasts from sibling CLIs."""
|
|
try:
|
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
if hasattr(signal, "SIGBREAK"):
|
|
signal.signal(signal.SIGBREAK, signal.SIG_IGN)
|
|
except (OSError, ValueError):
|
|
pass # SetConsoleCtrlHandler unavailable (rare) — best-effort
|
|
# signal only hooks SIGINT/SIGBREAK; SetConsoleCtrlHandler(NULL, TRUE) ignores ALL console
|
|
# control events (CTRL_CLOSE/CTRL_LOGOFF included), as background services should.
|
|
try:
|
|
import ctypes
|
|
ctypes.windll.kernel32.SetConsoleCtrlHandler(None, 1) # type: ignore[attr-defined]
|
|
except (OSError, AttributeError):
|
|
pass
|
|
|
|
|
|
def _make_exit_diag():
|
|
"""``_exit_diag(tag, **extra)`` recorder writing ``logs/gateway-exit-diag.log`` — captures every way
|
|
``asyncio.run()`` can return, for chasing silent Windows gateway deaths. HERMES_GATEWAY_EXIT_DIAG=0 opts out."""
|
|
from datetime import datetime as _dt, timezone as _tz
|
|
|
|
def _exit_diag(tag: str, **extra: object) -> None:
|
|
if os.environ.get("HERMES_GATEWAY_EXIT_DIAG", "1") != "1":
|
|
return
|
|
try:
|
|
from hermes_constants import get_hermes_home as _ghh
|
|
log_dir = _ghh() / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
line = {
|
|
"ts": _dt.now(_tz.utc).isoformat(), "tag": tag, "pid": os.getpid(),
|
|
"python": sys.version.split()[0], "platform": sys.platform, **extra,
|
|
}
|
|
with open(log_dir / "gateway-exit-diag.log", "a", encoding="utf-8") as f:
|
|
f.write(json.dumps(line, default=str) + "\n")
|
|
except Exception:
|
|
pass # never let the diagnostic itself crash the gateway
|
|
|
|
return _exit_diag
|
|
|
|
|
|
def _respawn_storm_backoff() -> None:
|
|
"""Portable app-level respawn-storm breaker (for supervisors without a floor). Defaults mirror
|
|
DEFAULT_CONFIG ``gateway.respawn_storm``; HERMES_GATEWAY_MAX_STARTS / HERMES_GATEWAY_START_WINDOW_S
|
|
override; max_starts <= 0 disables. Never blocks startup."""
|
|
try:
|
|
from gateway.status import record_start_and_check_storm
|
|
_max_starts = 5
|
|
_win = 120.0
|
|
try:
|
|
from hermes_cli.config import load_config
|
|
_cfg = load_config()
|
|
_gw = _cfg.get("gateway") if isinstance(_cfg, dict) else None
|
|
_rs = _gw.get("respawn_storm") if isinstance(_gw, dict) else None
|
|
if isinstance(_rs, dict):
|
|
if isinstance(_rs.get("max_starts"), int):
|
|
_max_starts = _rs["max_starts"]
|
|
if isinstance(_rs.get("window_seconds"), (int, float)):
|
|
_win = float(_rs["window_seconds"])
|
|
except Exception:
|
|
pass
|
|
try:
|
|
_max_starts = int(os.environ["HERMES_GATEWAY_MAX_STARTS"])
|
|
except (KeyError, ValueError):
|
|
pass
|
|
try:
|
|
_win = float(os.environ["HERMES_GATEWAY_START_WINDOW_S"])
|
|
except (KeyError, ValueError):
|
|
pass
|
|
_storm = record_start_and_check_storm(max_starts=_max_starts, window_s=_win) if _max_starts > 0 else None
|
|
if _storm is not None:
|
|
logger.warning(
|
|
"Gateway (re)started %d times in %.0fs — backing off %.0fs to break a respawn storm.",
|
|
_storm.count, _storm.window_s, _storm.backoff_s,
|
|
)
|
|
# Tell the startup watchdog the backoff sleep is intentional, not a parked deadlock.
|
|
try:
|
|
from hermes_startup_watchdog import kick_startup_watchdog
|
|
kick_startup_watchdog(extra_s=_storm.backoff_s)
|
|
except Exception:
|
|
pass
|
|
time.sleep(_storm.backoff_s)
|
|
except Exception as _be:
|
|
logger.debug("respawn-storm breaker check failed (non-fatal): %s", _be)
|
|
|
|
|
|
def run_gateway(verbose: int = 0, quiet: bool = False, replace: bool = False, force: bool = False):
|
|
"""Run the gateway in foreground. verbose 1=INFO/2+=DEBUG on stderr; quiet: no stderr logs; replace:
|
|
kill an existing instance first (avoids systemd restart loops); force: skip the supervised guard."""
|
|
_guard_official_docker_root_gateway()
|
|
_guard_named_profile_under_multiplexer(force=force)
|
|
_guard_supervised_gateway_conflict(force=force)
|
|
_guard_existing_gateway_process_conflict(replace=replace)
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
_apply_startup_watchdog_config()
|
|
|
|
# Detached Windows runs (HERMES_GATEWAY_DETACHED=1, or non-TTY for older wrappers) ignore
|
|
# console-control broadcasts from sibling CLIs; foreground runs keep Ctrl+C-to-stop.
|
|
stdin_is_tty = bool(_stdin_is_tty())
|
|
_console_window_attached = _windows_console_window_attached()
|
|
_breakaway = _windows_gateway_breakaway_state()
|
|
_absorb = _windows_gateway_should_absorb_console_controls()
|
|
if _absorb:
|
|
_absorb_windows_console_controls()
|
|
|
|
# A system-level unit execs us without XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS; adopt our own
|
|
# user bus before any worker env snapshot so `systemd-run --user --scope` works (#104893).
|
|
if is_linux() and os.environ.get("INVOCATION_ID"):
|
|
_ensure_user_systemd_env()
|
|
|
|
# Refresh the systemd unit on every boot so restart settings stay current even after an
|
|
# exit-code-75 respawn (stale-code or /restart), which bypasses `hermes gateway restart`.
|
|
if supports_systemd_services():
|
|
try:
|
|
refresh_systemd_unit_if_needed(system=False)
|
|
except Exception:
|
|
pass # best-effort; don't block gateway startup
|
|
|
|
from gateway.run import start_gateway
|
|
print("┌─────────────────────────────────────────────────────────┐")
|
|
print("│ ☤ Hermes Gateway Starting... │")
|
|
print("├─────────────────────────────────────────────────────────┤")
|
|
print("│ Messaging platforms + cron scheduler │")
|
|
print("│ Press Ctrl+C to stop │")
|
|
print("└─────────────────────────────────────────────────────────┘")
|
|
print()
|
|
|
|
# Exit 1 if no platform connects so systemd Restart=always retries transient errors.
|
|
verbosity = None if quiet else verbose
|
|
|
|
import atexit as _atexit
|
|
import traceback as _traceback
|
|
_exit_diag = _make_exit_diag()
|
|
_exit_diag(
|
|
"gateway.start", replace=replace, argv=sys.argv, stdin_is_tty=stdin_is_tty,
|
|
console_window_attached=_console_window_attached, detached=_gateway_detached_env(),
|
|
breakaway=_breakaway, absorb_windows_console_controls=_absorb,
|
|
)
|
|
_atexit.register(lambda: _exit_diag("atexit.hook", sys_exc=repr(sys.exc_info())))
|
|
|
|
_respawn_storm_backoff()
|
|
|
|
def _hard_exit_after_gateway_teardown(code: int) -> None:
|
|
# Mirror gateway.run.main()'s wedge-proof exit: bypass Python finalization so non-daemon
|
|
# threads (in-flight cron jobs) can't delay a /restart by minutes.
|
|
from gateway.run import _exit_after_graceful_shutdown
|
|
_exit_after_graceful_shutdown(code)
|
|
|
|
success = False
|
|
try:
|
|
success = asyncio.run(start_gateway(replace=replace, verbosity=verbosity))
|
|
_exit_diag("asyncio.run.returned", success=success)
|
|
except KeyboardInterrupt:
|
|
# Detached Windows runs absorb SIGINT above; keep the handler for console runs.
|
|
_exit_diag("asyncio.run.KeyboardInterrupt", traceback=_traceback.format_exc())
|
|
print("\nGateway stopped.")
|
|
_hard_exit_after_gateway_teardown(0)
|
|
return # unreachable in production (os._exit); guard for test stubs
|
|
except SystemExit as e:
|
|
_exit_diag("asyncio.run.SystemExit", code=e.code, traceback=_traceback.format_exc())
|
|
_hard_exit_after_gateway_teardown(0 if e.code is None else e.code if isinstance(e.code, int) else 1)
|
|
except BaseException as e:
|
|
# Everything else (CancelledError, exotic BaseExceptions): log the cause, then re-raise.
|
|
_exit_diag("asyncio.run.exception", exc_type=type(e).__name__, exc_repr=repr(e), traceback=_traceback.format_exc())
|
|
raise
|
|
if not success:
|
|
_exit_diag("gateway.exit_nonzero")
|
|
_hard_exit_after_gateway_teardown(1)
|
|
_exit_diag("gateway.exit_clean")
|
|
_hard_exit_after_gateway_teardown(0)
|
|
|
|
|
|
# =============================================================================
|
|
# Gateway Setup (Interactive Messaging Platform Configuration)
|
|
# =============================================================================
|
|
|
|
# Built-in per-platform setup config (env vars, instructions, prompts). Telegram, WhatsApp, Email,
|
|
# SMS, etc. live in plugins/platforms/<name>/ and are discovered via the platform registry.
|
|
_PLATFORMS = [
|
|
{
|
|
"key": "mattermost", "label": "Mattermost", "emoji": "💬", "token_var": "MATTERMOST_TOKEN",
|
|
"setup_instructions": [
|
|
"1. In Mattermost: Integrations → Bot Accounts → Add Bot Account",
|
|
" (System Console → Integrations → Bot Accounts must be enabled)",
|
|
"2. Give it a username (e.g. hermes) and copy the bot token",
|
|
"3. Works with any self-hosted Mattermost instance — enter your server URL",
|
|
"4. To find your user ID: click your avatar (top-left) → Profile",
|
|
" Your user ID is displayed there — click it to copy.",
|
|
" ⚠ This is NOT your username — it's a 26-character alphanumeric ID.",
|
|
"5. To get a channel ID: click the channel name → View Info → copy the ID",
|
|
],
|
|
"vars": [
|
|
{"name": "MATTERMOST_URL", "prompt": "Server URL (e.g. https://mm.example.com)",
|
|
"password": False, "help": "Your Mattermost server URL. Works with any self-hosted instance."},
|
|
{"name": "MATTERMOST_TOKEN", "prompt": "Bot token", "password": True,
|
|
"help": "Paste the bot token from step 2 above."},
|
|
{"name": "MATTERMOST_ALLOWED_USERS", "prompt": "Allowed user IDs (comma-separated)",
|
|
"password": False, "is_allowlist": True, "help": "Your Mattermost user ID from step 4 above."},
|
|
{"name": "MATTERMOST_HOME_CHANNEL",
|
|
"prompt": "Home channel ID (for cron/notification delivery, or empty to set later with /set-home)",
|
|
"password": False, "help": "Channel ID where Hermes delivers cron results and notifications."},
|
|
{"name": "MATTERMOST_REPLY_MODE",
|
|
"prompt": "Reply mode — 'off' for flat messages, 'thread' for threaded replies (default: off)",
|
|
"password": False,
|
|
"help": "off = flat channel messages, thread = replies nest under your message."},
|
|
],
|
|
},
|
|
{"key": "signal", "label": "Signal", "emoji": "📡", "token_var": "SIGNAL_HTTP_URL"},
|
|
{"key": "weixin", "label": "Weixin / WeChat", "emoji": "💬", "token_var": "WEIXIN_ACCOUNT_ID"},
|
|
{
|
|
"key": "bluebubbles", "label": "BlueBubbles (iMessage)",
|
|
"emoji": "💬", "token_var": "BLUEBUBBLES_SERVER_URL",
|
|
"setup_instructions": [
|
|
"1. Install BlueBubbles on a Mac that will act as your iMessage server:",
|
|
" https://bluebubbles.app/",
|
|
"2. Complete the BlueBubbles setup wizard — sign in with your Apple ID",
|
|
"3. In BlueBubbles Settings → API, note the Server URL and password",
|
|
"4. The server URL is typically http://<your-mac-ip>:1234",
|
|
"5. Hermes connects via the BlueBubbles REST API and receives",
|
|
" incoming messages via a local webhook",
|
|
"6. To authorize users, use DM pairing: hermes pairing generate bluebubbles",
|
|
" Share the code — the user sends it via iMessage to get approved",
|
|
],
|
|
"vars": [
|
|
{"name": "BLUEBUBBLES_SERVER_URL",
|
|
"prompt": "BlueBubbles server URL (e.g. http://192.168.1.10:1234)", "password": False,
|
|
"help": "The URL shown in BlueBubbles Settings → API."},
|
|
{"name": "BLUEBUBBLES_PASSWORD", "prompt": "BlueBubbles server password", "password": True,
|
|
"help": "The password shown in BlueBubbles Settings → API."},
|
|
{"name": "BLUEBUBBLES_ALLOWED_USERS",
|
|
"prompt": "Pre-authorized phone numbers or iMessage IDs (comma-separated, or leave empty for DM pairing)",
|
|
"password": False, "is_allowlist": True,
|
|
"help": "Optional — pre-authorize specific users. Leave empty to use DM pairing instead (recommended)."},
|
|
{"name": "BLUEBUBBLES_HOME_CHANNEL",
|
|
"prompt": "Home channel (phone number or iMessage ID for cron/notifications, or empty)",
|
|
"password": False,
|
|
"help": "Phone number or Apple ID to deliver cron results and notifications to."},
|
|
],
|
|
},
|
|
{
|
|
"key": "qqbot", "label": "QQ Bot", "emoji": "🐧", "token_var": "QQ_APP_ID",
|
|
"setup_instructions": [
|
|
"1. Register a QQ Bot application at q.qq.com",
|
|
"2. Note your App ID and App Secret from the application page",
|
|
"3. Enable the required intents (C2C, Group, Guild messages)",
|
|
"4. Configure sandbox or publish the bot",
|
|
],
|
|
"vars": [
|
|
{"name": "QQ_APP_ID", "prompt": "QQ Bot App ID", "password": False,
|
|
"help": "Your QQ Bot App ID from q.qq.com."},
|
|
{"name": "QQ_CLIENT_SECRET", "prompt": "QQ Bot App Secret", "password": True,
|
|
"help": "Your QQ Bot App Secret from q.qq.com."},
|
|
{"name": "QQ_ALLOWED_USERS",
|
|
"prompt": "Allowed user OpenIDs (comma-separated, leave empty for open access)",
|
|
"password": False, "is_allowlist": True,
|
|
"help": "Optional — restrict DM access to specific user OpenIDs."},
|
|
{"name": "QQBOT_HOME_CHANNEL",
|
|
"prompt": "Home channel (user/group OpenID for cron delivery, or empty)", "password": False,
|
|
"help": "OpenID to deliver cron results and notifications to."},
|
|
],
|
|
},
|
|
{
|
|
"key": "yuanbao", "label": "Yuanbao", "emoji": "💎", "token_var": "YUANBAO_APP_ID",
|
|
"setup_instructions": [
|
|
"1. Download the Yuanbao app from https://yuanbao.tencent.com/",
|
|
"2. In the app, go to PAI → My Bot and create a new bot",
|
|
"3. After the bot is created, copy the App ID and App Secret",
|
|
"4. Enter them below and Hermes will connect automatically over WebSocket",
|
|
],
|
|
"vars": [
|
|
{"name": "YUANBAO_APP_ID", "prompt": "App ID", "password": False,
|
|
"help": "The App ID from your Yuanbao IM Bot credentials."},
|
|
{"name": "YUANBAO_APP_SECRET", "prompt": "App Secret", "password": True,
|
|
"help": "The App Secret (used for HMAC signing) from your Yuanbao IM Bot."},
|
|
],
|
|
},
|
|
]
|
|
|
|
|
|
def _all_platforms() -> list[dict]:
|
|
"""Built-in ``_PLATFORMS`` plus registry plugin platforms (same dict shape, source in
|
|
``_registry_entry``). Plugins are discovered here (idempotent) so the setup menu works without a
|
|
running gateway; user-installed ones still need ``plugins.enabled`` (untrusted code). Matrix is
|
|
hidden on Windows: python-olm has no wheel or native build (use WSL)."""
|
|
try:
|
|
from hermes_cli.plugins import discover_plugins
|
|
discover_plugins()
|
|
except Exception as e:
|
|
logger.debug("plugin discovery failed during platform enumeration: %s", e)
|
|
|
|
hide_matrix = sys.platform == "win32"
|
|
platforms = [dict(p) for p in _PLATFORMS if not (hide_matrix and p.get("key") == "matrix")]
|
|
by_key = {p["key"]: p for p in platforms}
|
|
|
|
try:
|
|
from gateway.platform_registry import platform_registry
|
|
except Exception:
|
|
return platforms
|
|
|
|
for entry in platform_registry.all_entries():
|
|
if entry.name in by_key or (hide_matrix and entry.name == "matrix"):
|
|
continue
|
|
platforms.append({
|
|
"key": entry.name, "label": entry.label, "emoji": entry.emoji,
|
|
"token_var": entry.required_env[0] if entry.required_env else "",
|
|
"install_hint": entry.install_hint, "_registry_entry": entry,
|
|
})
|
|
return platforms
|
|
|
|
|
|
def _platform_status(platform: dict) -> str:
|
|
"""Plain-text status string; uncolored because ANSI codes break curses menu width math."""
|
|
entry = platform.get("_registry_entry")
|
|
if entry is not None:
|
|
# Prefer is_connected (env + config.yaml) over check_fn (a coarse deps gate). Never fall back
|
|
# to check_fn when is_connected returned False, or "SDK installed" would override "no token".
|
|
try:
|
|
if entry.is_connected is not None:
|
|
from gateway.config import PlatformConfig
|
|
configured = bool(entry.is_connected(PlatformConfig(enabled=True)))
|
|
else:
|
|
configured = bool(entry.check_fn())
|
|
except Exception:
|
|
configured = False
|
|
return "configured" if configured else "not configured"
|
|
|
|
token_var = platform.get("token_var", "")
|
|
if not token_var:
|
|
return "not configured"
|
|
# Built-ins needing a second credential to count as fully configured.
|
|
second_var = {"signal": "SIGNAL_ACCOUNT", "weixin": "WEIXIN_TOKEN"}.get(platform.get("key"))
|
|
present = [bool(get_env_value(v)) for v in (token_var, second_var) if v]
|
|
if all(present):
|
|
return "configured"
|
|
return "partially configured" if any(present) else "not configured"
|
|
|
|
|
|
# Operator wording for the out-of-loop watchdog exit reasons stamped by gateway/shutdown_watchdog.py.
|
|
_WATCHDOG_EXIT_REASONS = {
|
|
"loop_liveness_watchdog": (
|
|
"event loop stopped dispatching (housekeeping, cron and the kanban dispatcher froze); "
|
|
f"the liveness watchdog exited with code {GATEWAY_SERVICE_RESTART_EXIT_CODE} for the supervisor to restart it"
|
|
),
|
|
"shutdown_watchdog": "shutdown drain wedged; the shutdown watchdog forced the exit (see logs/gateway-shutdown-watchdog.log)",
|
|
}
|
|
|
|
|
|
def _runtime_health_lines() -> list[str]:
|
|
"""Summarize the latest persisted gateway runtime health state."""
|
|
try:
|
|
from gateway.status import (
|
|
read_runtime_status, runtime_status_heartbeat_age_s, runtime_status_is_stale, runtime_status_pid_is_live)
|
|
except Exception:
|
|
return []
|
|
|
|
state = read_runtime_status()
|
|
if not state:
|
|
return []
|
|
|
|
gateway_state = state.get("gateway_state")
|
|
exit_reason = state.get("exit_reason")
|
|
lines = [
|
|
f"⚠ {platform}: {pdata.get('error_message') or 'unknown error'}"
|
|
for platform, pdata in (state.get("platforms", {}) or {}).items()
|
|
if pdata.get("state") == "fatal"
|
|
]
|
|
|
|
# A live-claiming snapshot can outlive an ungracefully killed gateway (taskkill /F, OOM). Past
|
|
# the freshness TTL with the recorded PID gone, say so instead of rendering stale live state.
|
|
if gateway_state in ("running", "degraded", "starting", "draining") and runtime_status_is_stale(state):
|
|
if not runtime_status_pid_is_live(state):
|
|
lines.append(
|
|
f"⚠ Stale gateway_state.json: recorded state '{gateway_state}' but the "
|
|
"recorded process is gone (likely an ungraceful shutdown)"
|
|
)
|
|
return lines
|
|
# PID alive but housekeeping stopped re-stamping the file: the reporter's "not a crash" case
|
|
# (#113372) — the process looks 'running' while housekeeping/cron/kanban dispatch are frozen.
|
|
age = runtime_status_heartbeat_age_s(state)
|
|
if gateway_state != "draining" and age is not None:
|
|
lines.append(
|
|
f"⚠ Gateway heartbeat stale: housekeeping has not refreshed gateway_state.json for {age} s "
|
|
f"(event loop or housekeeping wedged; pid {state.get('pid')} alive) — restart the gateway"
|
|
)
|
|
|
|
if gateway_state == "startup_failed" and exit_reason:
|
|
lines.append(f"⚠ Last startup issue: {exit_reason}")
|
|
elif gateway_state == "degraded" and exit_reason:
|
|
# An out-of-loop watchdog hard-exited the process (#113372): the loop stopped dispatching, so
|
|
# housekeeping/cron/kanban froze together. Without this arm the file would read 'running'.
|
|
lines.append(f"⚠ Gateway exited degraded: {_WATCHDOG_EXIT_REASONS.get(exit_reason, exit_reason)}")
|
|
elif gateway_state == "draining":
|
|
action = "restart" if state.get("restart_requested") else "shutdown"
|
|
from gateway.status import parse_active_agents
|
|
count = parse_active_agents(state.get("active_agents"))
|
|
lines.append(f"⏳ Gateway draining for {action} ({count} active agent(s))")
|
|
work = state.get("active_work")
|
|
if isinstance(work, list) and work:
|
|
from hermes_cli.update_cmd_drain_report import describe_active_work_unit
|
|
lines.extend(f" • {describe_active_work_unit(u)}" for u in work if isinstance(u, dict))
|
|
elif gateway_state == "stopped" and exit_reason:
|
|
lines.append(f"⚠ Last shutdown reason: {exit_reason}")
|
|
|
|
return lines
|
|
|
|
|
|
def _set_platform_unauthorized_dm_behavior(platform_key: str, behavior: str) -> None:
|
|
"""Persist a platform-specific unauthorized-DM policy in config.yaml."""
|
|
write_platform_config_field(platform_key, "unauthorized_dm_behavior", behavior, raw=True)
|
|
|
|
|
|
def _print_setup_header(title: str) -> None:
|
|
print()
|
|
print(color(f" ─── {title} Setup ───", Colors.CYAN))
|
|
|
|
|
|
def _print_info_lines(*lines: str) -> None:
|
|
for line in lines:
|
|
print_info(line)
|
|
|
|
|
|
def _confirm_reconfigure(label: str, *env_vars: str) -> bool:
|
|
"""False when ``label`` is already configured (all ``env_vars`` set) and the user declines."""
|
|
if all(get_env_value(v) for v in env_vars):
|
|
print()
|
|
print_success(f"{label} is already configured.")
|
|
return prompt_yes_no(f" Reconfigure {label}?", False)
|
|
return True
|
|
|
|
|
|
def _offer_home_channel(home_var: str, user_id: str, what: str) -> None:
|
|
"""Offer to persist ``user_id`` as ``home_var`` (e.g. "your Telegram user ID")."""
|
|
if prompt_yes_no(f" Use {what} ({user_id}) as the home channel?", True):
|
|
save_env_value(home_var, user_id)
|
|
print_success(f" Home channel set to {user_id}")
|
|
|
|
|
|
def _save_env_values(**values: str) -> None:
|
|
for name, value in values.items():
|
|
save_env_value(name, value)
|
|
|
|
|
|
def _prompt_csv(prompt_text: str, default: str) -> str:
|
|
"""Comma-separated ID prompt with whitespace stripped."""
|
|
return prompt(prompt_text, default, password=False).replace(" ", "")
|
|
|
|
|
|
# (default index, *choices) for the no-allowlist access prompt, keyed by is_email.
|
|
_UNAUTHORIZED_ACCESS_CHOICES = {
|
|
True: (3,
|
|
"Enable open access (any email sender can message the bot)",
|
|
"Use DM pairing (unknown email senders receive a pairing code)",
|
|
"Politely decline unknown senders (one-time message, then silence)",
|
|
"Keep unknown senders silent"),
|
|
False: (1,
|
|
"Enable open access (anyone can message the bot)",
|
|
"Use DM pairing (unknown users request access, you approve with 'hermes pairing approve')",
|
|
"Politely decline unknown senders (one-time message, then silence)",
|
|
"Skip for now (bot will deny all users until configured)"),
|
|
}
|
|
|
|
|
|
def _prompt_unauthorized_access(platform_key: str) -> None:
|
|
"""No allowlist was given — ask open access vs DM pairing vs decline vs skip/silent, and persist."""
|
|
is_email = platform_key == "email"
|
|
print()
|
|
default_idx, *access_choices = _UNAUTHORIZED_ACCESS_CHOICES[is_email]
|
|
access_idx = prompt_choice(" How should unauthorized users be handled?", access_choices, default_idx)
|
|
if access_idx == 0:
|
|
save_env_value("EMAIL_ALLOW_ALL_USERS" if is_email else "GATEWAY_ALLOW_ALL_USERS", "true")
|
|
print_warning(" Open access enabled — anyone can use your bot!")
|
|
elif access_idx == 1:
|
|
if is_email:
|
|
_set_platform_unauthorized_dm_behavior("email", "pair")
|
|
print_success(" DM pairing mode — users will receive a code to request access.")
|
|
print_info(" Approve with: hermes pairing approve <platform> <code>")
|
|
elif access_idx == 2:
|
|
_set_platform_unauthorized_dm_behavior(platform_key, "decline")
|
|
print_success(" Unknown senders get one polite decline, then silence (unauthorized_dm_behavior: decline).")
|
|
elif is_email:
|
|
print_success(" Unknown email senders will be ignored.")
|
|
else:
|
|
print_info(" Skipped — configure later with 'hermes gateway setup'")
|
|
|
|
|
|
def _telegram_auto_setup(token_var: str) -> tuple[bool, object]:
|
|
"""Offer the managed-bot QR flow. Returns (token_saved, owner_user_id)."""
|
|
print()
|
|
_print_info_lines(
|
|
" Telegram can be configured automatically with a managed bot:",
|
|
" [1] Automatic (scan QR → confirm in Telegram → done)", " [2] Manual BotFather token",
|
|
)
|
|
if prompt(" Choice [1/2]", default="1").strip() != "1":
|
|
return False, None
|
|
try:
|
|
from hermes_cli.telegram_managed_bot import (
|
|
auto_setup_telegram_bot_result, is_valid_telegram_bot_token,
|
|
)
|
|
except ImportError:
|
|
print_warning(" Automatic setup is unavailable in this install.")
|
|
return False, None
|
|
result = auto_setup_telegram_bot_result()
|
|
if result and is_valid_telegram_bot_token(result.token):
|
|
save_env_value(token_var, result.token)
|
|
print_success(" Saved TELEGRAM_BOT_TOKEN")
|
|
return True, result.owner_user_id
|
|
if result:
|
|
print_warning(" Automatic setup returned an invalid Telegram token.")
|
|
print()
|
|
print_info(" Falling back to manual setup...")
|
|
return False, None
|
|
|
|
|
|
def _clean_discord_ids(cleaned: str) -> str:
|
|
"""Strip common Discord prefixes (user:123, <@123>, <@!123>) from a comma-separated list."""
|
|
parts = []
|
|
for uid in cleaned.split(","):
|
|
uid = uid.strip()
|
|
if uid.startswith("<@") and uid.endswith(">"):
|
|
uid = uid.lstrip("<@!").rstrip(">")
|
|
if uid.lower().startswith("user:"):
|
|
uid = uid[5:]
|
|
if uid:
|
|
parts.append(uid)
|
|
return ",".join(parts)
|
|
|
|
|
|
def _prompt_allowlist_var(var: dict, platform_key: str, auto_owner_user_id) -> str | None:
|
|
"""Allowlist prompt for one var; returns the saved value or None (open-access prompt shown)."""
|
|
if "TELEGRAM" in var["name"] and auto_owner_user_id:
|
|
detected_id = str(auto_owner_user_id)
|
|
print_success(f" Detected your Telegram user ID: {detected_id}")
|
|
if prompt_yes_no(" Allow this Telegram account to use the bot?", True):
|
|
extra = prompt(" Additional allowed user IDs (comma-separated, optional)", password=False)
|
|
ids = [detected_id]
|
|
for uid in extra.replace(" ", "").split(","):
|
|
if uid and uid not in ids:
|
|
ids.append(uid)
|
|
cleaned = ",".join(ids)
|
|
save_env_value(var["name"], cleaned)
|
|
print_success(" Saved — only these users can interact with the bot.")
|
|
return cleaned
|
|
|
|
_print_info_lines(
|
|
" The gateway DENIES all users by default for security.",
|
|
" Enter user IDs to create an allowlist, or leave empty",
|
|
" and you'll be asked about open access next.",
|
|
)
|
|
value = prompt(f" {var['prompt']}", password=False)
|
|
if not value:
|
|
_prompt_unauthorized_access(platform_key)
|
|
return None
|
|
cleaned = value.replace(" ", "")
|
|
if "DISCORD" in var["name"]:
|
|
cleaned = _clean_discord_ids(cleaned)
|
|
save_env_value(var["name"], cleaned)
|
|
print_success(" Saved — only these users can interact with the bot.")
|
|
return cleaned
|
|
|
|
|
|
def _setup_standard_platform(platform: dict):
|
|
"""Interactive setup for Telegram, Discord, or Slack."""
|
|
from hermes_cli.setup_hidden_env import is_setup_hidden_env as _is_setup_hidden_env
|
|
emoji, label, token_var = platform["emoji"], platform["label"], platform["token_var"]
|
|
_print_setup_header(f"{emoji} {label}")
|
|
|
|
instructions = platform.get("setup_instructions")
|
|
if instructions:
|
|
print()
|
|
_print_info_lines(*(f" {line}" for line in instructions))
|
|
|
|
if not _confirm_reconfigure(label, token_var):
|
|
return
|
|
|
|
auto_token_saved, auto_owner_user_id = False, None
|
|
if platform.get("key") == "telegram":
|
|
auto_token_saved, auto_owner_user_id = _telegram_auto_setup(token_var)
|
|
|
|
allowed_val_set = None # Track if user set an allowlist (for home channel offer)
|
|
|
|
# Skip knobs the setup forms hide (home channel, reply mode, proxy...): they're self-configuring.
|
|
setup_vars = [
|
|
v for v in platform["vars"]
|
|
if v["name"] == token_var or v.get("is_allowlist") or not _is_setup_hidden_env(v["name"])
|
|
]
|
|
|
|
for var in setup_vars:
|
|
print()
|
|
print_info(f" {var['help']}")
|
|
existing = get_env_value(var["name"])
|
|
if existing and var["name"] != token_var:
|
|
print_info(f" Current: {existing}")
|
|
|
|
if auto_token_saved and var["name"] == token_var:
|
|
print_info(" Token saved by automatic setup.")
|
|
continue
|
|
|
|
if var.get("is_allowlist"):
|
|
saved = _prompt_allowlist_var(var, platform.get("key"), auto_owner_user_id)
|
|
if saved is not None:
|
|
allowed_val_set = saved
|
|
continue
|
|
|
|
value = prompt(f" {var['prompt']}", password=var.get("password", False))
|
|
if value:
|
|
save_env_value(var["name"], value)
|
|
print_success(f" Saved {var['name']}")
|
|
elif var["name"] == token_var:
|
|
print_warning(f" Skipped — {label} won't work without this.")
|
|
return
|
|
else:
|
|
print_info(" Skipped (can configure later)")
|
|
|
|
# Offer the first allowlisted user ID as home channel when none is set (Telegram DMs).
|
|
home_var = f"{label.upper()}_HOME_CHANNEL"
|
|
home_val = get_env_value(home_var)
|
|
if allowed_val_set and not home_val and label == "Telegram":
|
|
first_id = allowed_val_set.split(",")[0].strip()
|
|
if first_id:
|
|
_offer_home_channel(home_var, first_id, "your user ID")
|
|
|
|
print()
|
|
print_success(f"{emoji} {label} configured!")
|
|
|
|
|
|
# WhatsApp/DingTalk/WeCom/Feishu setup flows live in their plugins' adapter.py::interactive_setup.
|
|
|
|
|
|
def _running_under_s6() -> bool:
|
|
from hermes_cli.service_manager import detect_service_manager
|
|
return detect_service_manager() == "s6"
|
|
|
|
|
|
def _systemd_unit_installed() -> bool:
|
|
return supports_systemd_services() and (
|
|
get_systemd_unit_path(system=False).exists() or get_systemd_unit_path(system=True).exists()
|
|
)
|
|
|
|
|
|
def _is_service_installed() -> bool:
|
|
return _installed_service_kind() is not None
|
|
|
|
|
|
def _is_service_running() -> bool:
|
|
"""Check if the gateway service is currently running."""
|
|
if supports_systemd_services():
|
|
return _systemd_unit_is_active(False) or _systemd_unit_is_active(True)
|
|
if is_macos() and get_launchd_plist_path().exists():
|
|
try:
|
|
return _launchd_service_registered(get_launchd_label(), timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
return False
|
|
# Windows "installed" doesn't mean "running"; like manual runs, a live gateway process decides.
|
|
return len(find_gateway_pids()) > 0
|
|
|
|
|
|
# Weixin DM policy by menu index (index 2 = allowlist is prompted separately).
|
|
_WEIXIN_DM_POLICIES = {
|
|
0: ("pairing", "false", print_success, " DM pairing enabled."),
|
|
1: ("open", "true", print_warning, " Open DM access enabled for Weixin."),
|
|
3: ("disabled", "false", print_warning, " Direct messages disabled."),
|
|
}
|
|
_WEIXIN_GROUP_NOTE = (
|
|
" Note: QR login connects an iLink bot identity (e.g. ...@im.bot), not a",
|
|
" scriptable personal WeChat account. Ordinary WeChat groups typically cannot",
|
|
" invite an @im.bot identity, and iLink does not deliver ordinary-group events",
|
|
" to most bot accounts. The settings below only apply when iLink actually",
|
|
" delivers group events for your account type — otherwise DM remains the only",
|
|
" working channel regardless of this choice.",
|
|
)
|
|
|
|
|
|
def _setup_weixin():
|
|
"""Interactive setup for Weixin / WeChat personal accounts."""
|
|
_print_setup_header("💬 Weixin / WeChat")
|
|
print()
|
|
_print_info_lines(
|
|
" 1. Hermes will open Tencent iLink QR login in this terminal.",
|
|
" 2. Use WeChat to scan and confirm the QR code.",
|
|
" 3. Hermes will store the returned account_id/token in ~/.hermes/.env.",
|
|
" 4. This adapter supports native text, image, video, and document delivery.",
|
|
)
|
|
|
|
if not _confirm_reconfigure("Weixin", "WEIXIN_ACCOUNT_ID", "WEIXIN_TOKEN"):
|
|
return
|
|
|
|
try:
|
|
from gateway.platforms.weixin import check_weixin_requirements, qr_login
|
|
except Exception as exc:
|
|
print_error(f" Weixin adapter import failed: {exc}")
|
|
print_info(" Install gateway dependencies first, then retry.")
|
|
return
|
|
|
|
if not check_weixin_requirements():
|
|
print_error(" Missing dependencies: Weixin needs aiohttp and cryptography.")
|
|
print_info(" Install them, then rerun `hermes gateway setup`.")
|
|
return
|
|
|
|
print()
|
|
if not prompt_yes_no(" Start QR login now?", True):
|
|
print_info(" Cancelled.")
|
|
return
|
|
|
|
try:
|
|
credentials = asyncio.run(qr_login(str(get_hermes_home())))
|
|
except KeyboardInterrupt:
|
|
print()
|
|
print_warning(" Weixin setup cancelled.")
|
|
return
|
|
except Exception as exc:
|
|
print_error(f" QR login failed: {exc}")
|
|
return
|
|
|
|
if not credentials:
|
|
print_warning(" QR login did not complete.")
|
|
return
|
|
|
|
account_id = credentials.get("account_id", "")
|
|
user_id = credentials.get("user_id", "")
|
|
save_env_value("WEIXIN_ACCOUNT_ID", account_id)
|
|
save_env_value("WEIXIN_TOKEN", credentials.get("token", ""))
|
|
if credentials.get("base_url", ""):
|
|
save_env_value("WEIXIN_BASE_URL", credentials.get("base_url", ""))
|
|
save_env_value(
|
|
"WEIXIN_CDN_BASE_URL", get_env_value("WEIXIN_CDN_BASE_URL") or "https://novac2c.cdn.weixin.qq.com/c2c"
|
|
)
|
|
|
|
print()
|
|
access_choices = [
|
|
"Use DM pairing approval (recommended)", "Allow all direct messages", "Only allow listed user IDs",
|
|
"Disable direct messages",
|
|
]
|
|
access_idx = prompt_choice(" How should direct messages be authorized?", access_choices, 0)
|
|
if access_idx == 2:
|
|
allowlist = _prompt_csv(" Allowed Weixin user IDs (comma-separated)", user_id or "")
|
|
_save_env_values(
|
|
WEIXIN_DM_POLICY="allowlist", WEIXIN_ALLOW_ALL_USERS="false", WEIXIN_ALLOWED_USERS=allowlist
|
|
)
|
|
print_success(" Weixin allowlist saved.")
|
|
else:
|
|
policy, allow_all, emit, message = _WEIXIN_DM_POLICIES.get(access_idx, _WEIXIN_DM_POLICIES[3])
|
|
_save_env_values(WEIXIN_DM_POLICY=policy, WEIXIN_ALLOW_ALL_USERS=allow_all, WEIXIN_ALLOWED_USERS="")
|
|
emit(message)
|
|
if access_idx == 0:
|
|
print_info(
|
|
" Unknown DM users can request access and you approve them with `hermes pairing approve`."
|
|
)
|
|
|
|
print()
|
|
_print_info_lines(*_WEIXIN_GROUP_NOTE)
|
|
group_choices = [
|
|
"Disable group chats (recommended)", "Allow all group chats", "Only allow listed group chat IDs",
|
|
]
|
|
group_idx = prompt_choice(" How should group chats be handled?", group_choices, 0)
|
|
if group_idx == 0:
|
|
_save_env_values(WEIXIN_GROUP_POLICY="disabled", WEIXIN_GROUP_ALLOWED_USERS="")
|
|
print_info(" Group chats disabled.")
|
|
elif group_idx == 1:
|
|
_save_env_values(WEIXIN_GROUP_POLICY="open", WEIXIN_GROUP_ALLOWED_USERS="")
|
|
print_warning(" All group chats enabled (only takes effect if iLink delivers group events).")
|
|
else:
|
|
allow_groups = _prompt_csv(" Allowed group chat IDs (comma-separated, not member user IDs)", "")
|
|
_save_env_values(WEIXIN_GROUP_POLICY="allowlist", WEIXIN_GROUP_ALLOWED_USERS=allow_groups)
|
|
print_success(" Group allowlist saved (only takes effect if iLink delivers group events).")
|
|
|
|
if user_id:
|
|
print()
|
|
_offer_home_channel("WEIXIN_HOME_CHANNEL", user_id, "your Weixin user ID")
|
|
|
|
print()
|
|
print_success("Weixin configured!")
|
|
print_info(f" Account ID: {account_id}")
|
|
if user_id:
|
|
print_info(f" User ID: {user_id}")
|
|
|
|
|
|
def _setup_qqbot():
|
|
"""Interactive setup for QQ Bot — scan-to-configure or manual credentials."""
|
|
_print_setup_header("🐧 QQ Bot")
|
|
|
|
if not _confirm_reconfigure("QQ Bot", "QQ_APP_ID", "QQ_CLIENT_SECRET"):
|
|
return
|
|
|
|
print()
|
|
method_choices = ["Scan QR code to add bot automatically (recommended)", "Enter existing App ID and App Secret manually"]
|
|
credentials = None
|
|
if prompt_choice(" How would you like to set up QQ Bot?", method_choices, 0) == 0:
|
|
try:
|
|
from gateway.platforms.qqbot import qr_register
|
|
credentials = qr_register()
|
|
except KeyboardInterrupt:
|
|
print()
|
|
print_warning(" QQ Bot setup cancelled.")
|
|
return
|
|
if not credentials:
|
|
print_info(" QR setup did not complete. Continuing with manual input.")
|
|
|
|
if not credentials:
|
|
print()
|
|
_print_info_lines(
|
|
" Go to https://q.qq.com to register a QQ Bot application.",
|
|
" Note your App ID and App Secret from the application page.",
|
|
)
|
|
print()
|
|
app_id = prompt(" App ID", password=False)
|
|
if not app_id:
|
|
print_warning(" Skipped — QQ Bot won't work without an App ID.")
|
|
return
|
|
app_secret = prompt(" App Secret", password=True)
|
|
if not app_secret:
|
|
print_warning(" Skipped — QQ Bot won't work without an App Secret.")
|
|
return
|
|
credentials = {"app_id": app_id.strip(), "client_secret": app_secret.strip(), "user_openid": ""}
|
|
|
|
save_env_value("QQ_APP_ID", credentials["app_id"])
|
|
save_env_value("QQ_CLIENT_SECRET", credentials["client_secret"])
|
|
|
|
user_openid = credentials.get("user_openid", "")
|
|
|
|
print()
|
|
access_choices = ["Use DM pairing approval (recommended)", "Allow all direct messages", "Only allow listed user OpenIDs"]
|
|
access_idx = prompt_choice(" How should direct messages be authorized?", access_choices, 0)
|
|
if access_idx == 0:
|
|
save_env_value("QQ_ALLOW_ALL_USERS", "false")
|
|
allowed = ""
|
|
if user_openid:
|
|
print()
|
|
if prompt_yes_no(f" Add yourself ({user_openid}) to the allow list?", True):
|
|
allowed = user_openid
|
|
print_success(f" Allow list set to {user_openid}")
|
|
save_env_value("QQ_ALLOWED_USERS", allowed)
|
|
print_success(" DM pairing enabled.")
|
|
print_info(" Unknown users can request access; approve with `hermes pairing approve`.")
|
|
elif access_idx == 1:
|
|
_save_env_values(QQ_ALLOW_ALL_USERS="true", QQ_ALLOWED_USERS="")
|
|
print_warning(" Open DM access enabled for QQ Bot.")
|
|
else:
|
|
allowlist = _prompt_csv(" Allowed user OpenIDs (comma-separated)", user_openid or "")
|
|
_save_env_values(QQ_ALLOW_ALL_USERS="false", QQ_ALLOWED_USERS=allowlist)
|
|
print_success(" Allowlist saved.")
|
|
|
|
print()
|
|
if user_openid:
|
|
_offer_home_channel("QQBOT_HOME_CHANNEL", user_openid, "your QQ user ID")
|
|
else:
|
|
home_channel = prompt(" Home channel OpenID (for cron/notifications, or empty)", password=False)
|
|
if home_channel:
|
|
save_env_value("QQBOT_HOME_CHANNEL", home_channel.strip())
|
|
print_success(f" Home channel set to {home_channel.strip()}")
|
|
|
|
print()
|
|
print_success("🐧 QQ Bot configured!")
|
|
print_info(f" App ID: {credentials['app_id']}")
|
|
|
|
|
|
def _signal_line_input(prompt_text: str) -> str | None:
|
|
"""``line_input`` for the Signal wizard; None (after printing the cancel line) on EOF/Ctrl+C."""
|
|
try:
|
|
return line_input(prompt_text).strip()
|
|
except (EOFError, KeyboardInterrupt):
|
|
print("\n Setup cancelled.")
|
|
return None
|
|
|
|
|
|
def _setup_signal():
|
|
"""Interactive setup for Signal messenger."""
|
|
_print_setup_header("📡 Signal")
|
|
|
|
existing_url = get_env_value("SIGNAL_HTTP_URL")
|
|
existing_account = get_env_value("SIGNAL_ACCOUNT")
|
|
if not _confirm_reconfigure("Signal", "SIGNAL_HTTP_URL", "SIGNAL_ACCOUNT"):
|
|
return
|
|
|
|
print()
|
|
if shutil.which("signal-cli"):
|
|
print_success("signal-cli found on PATH.")
|
|
else:
|
|
print_warning("signal-cli not found on PATH.")
|
|
_print_info_lines(
|
|
" Signal requires signal-cli running as an HTTP daemon.", " Install options:",
|
|
" Linux: download from https://github.com/AsamK/signal-cli/releases",
|
|
" macOS: brew install signal-cli", " Docker: bbernhard/signal-cli-rest-api",
|
|
)
|
|
print()
|
|
_print_info_lines(
|
|
" After installing, link your account and start the daemon:",
|
|
' signal-cli link -n "HermesAgent"',
|
|
" signal-cli --account +YOURNUMBER daemon --http 127.0.0.1:8080",
|
|
)
|
|
print()
|
|
|
|
print()
|
|
print_info(" Enter the URL where signal-cli HTTP daemon is running.")
|
|
default_url = existing_url or "http://127.0.0.1:8080"
|
|
url = _signal_line_input(f" HTTP URL [{default_url}]: ")
|
|
if url is None:
|
|
return
|
|
url = url or default_url
|
|
|
|
print_info(" Testing connection...")
|
|
try:
|
|
import httpx
|
|
resp = httpx.get(f"{url.rstrip('/')}/api/v1/check", timeout=10.0)
|
|
if resp.status_code == 200:
|
|
print_success(" signal-cli daemon is reachable!")
|
|
else:
|
|
print_warning(f" signal-cli responded with status {resp.status_code}.")
|
|
if not prompt_yes_no(" Continue anyway?", False):
|
|
return
|
|
except Exception as e:
|
|
print_warning(f" Could not reach signal-cli at {url}: {e}")
|
|
if not prompt_yes_no(" Save this URL anyway? (you can start signal-cli later)", True):
|
|
return
|
|
|
|
save_env_value("SIGNAL_HTTP_URL", url)
|
|
|
|
print()
|
|
_print_info_lines(" Enter your Signal account phone number in E.164 format.", " Example: +15551234567")
|
|
default_account = existing_account or ""
|
|
account = _signal_line_input(f" Account number{f' [{default_account}]' if default_account else ''}: ")
|
|
if account is None:
|
|
return
|
|
account = account or default_account
|
|
if not account:
|
|
print_error(" Account number is required.")
|
|
return
|
|
|
|
save_env_value("SIGNAL_ACCOUNT", account)
|
|
|
|
print()
|
|
_print_info_lines(
|
|
" The gateway DENIES all users by default for security.",
|
|
" Enter phone numbers or UUIDs of allowed users (comma-separated).",
|
|
)
|
|
default_allowed = get_env_value("SIGNAL_ALLOWED_USERS") or account
|
|
allowed = _signal_line_input(f" Allowed users [{default_allowed}]: ")
|
|
if allowed is None:
|
|
return
|
|
save_env_value("SIGNAL_ALLOWED_USERS", allowed or default_allowed)
|
|
|
|
print()
|
|
if prompt_yes_no(" Enable group messaging? (disabled by default for security)", False):
|
|
print()
|
|
print_info(" Enter group IDs to allow, or * for all groups.")
|
|
existing_groups = get_env_value("SIGNAL_GROUP_ALLOWED_USERS") or ""
|
|
groups = _signal_line_input(f" Group IDs [{existing_groups or '*'}]: ")
|
|
if groups is None:
|
|
return
|
|
save_env_value("SIGNAL_GROUP_ALLOWED_USERS", groups or existing_groups or "*")
|
|
|
|
print()
|
|
print_success("Signal configured!")
|
|
_print_info_lines(
|
|
f" URL: {url}", f" Account: {account}", " DM auth: via SIGNAL_ALLOWED_USERS + DM pairing",
|
|
f" Groups: {'enabled' if get_env_value('SIGNAL_GROUP_ALLOWED_USERS') else 'disabled'}",
|
|
)
|
|
|
|
|
|
def _builtin_setup_fn(key: str):
|
|
"""Resolve a built-in platform's setup function; late-bound to dodge the hermes_cli.setup cycle."""
|
|
from hermes_cli import setup as _s
|
|
return {
|
|
# telegram/discord/slack/whatsapp/dingtalk/feishu/wecom setup_fns come from their plugins.
|
|
"bluebubbles": setup_platforms._setup_bluebubbles,
|
|
"webhooks": setup_platforms._setup_webhooks,
|
|
"signal": _setup_signal,
|
|
"weixin": _setup_weixin,
|
|
"qqbot": _setup_qqbot,
|
|
}.get(key)
|
|
|
|
|
|
def _configure_platform(platform: dict) -> None:
|
|
"""Plugin ``setup_fn`` -> built-in by key -> ``_setup_standard_platform`` (``vars``) -> env-var hint.
|
|
Bundled plugins auto-load; user plugins must already be in ``plugins.enabled``."""
|
|
entry = platform.get("_registry_entry")
|
|
fn = entry.setup_fn if entry is not None else None
|
|
if fn is None:
|
|
fn = _builtin_setup_fn(platform["key"])
|
|
if fn is not None:
|
|
fn()
|
|
return
|
|
if platform.get("vars"):
|
|
_setup_standard_platform(platform)
|
|
return
|
|
|
|
label = platform.get("label", platform["key"])
|
|
_print_setup_header(f"{platform.get('emoji', '🔌')} {label}")
|
|
required = entry.required_env if entry else []
|
|
if required:
|
|
print_info(f" Set these env vars in ~/.hermes/.env: {', '.join(required)}")
|
|
else:
|
|
print_info(f" Configure {label} in config.yaml under gateway.platforms.{platform['key']}")
|
|
if platform.get("install_hint"):
|
|
print_info(f" {platform['install_hint']}")
|
|
|
|
|
|
def _print_indented(text: str, emit=print) -> None:
|
|
for line in text.splitlines():
|
|
emit(f" {line}")
|
|
|
|
|
|
def _service_backend(*, windows: bool = True) -> str | None:
|
|
"""Host service manager: ``"systemd"`` / ``"launchd"`` / ``"windows"`` / None, in the canonical
|
|
predicate order every subcommand routes on. ``windows=False`` never probes ``is_windows()``."""
|
|
if supports_systemd_services():
|
|
return "systemd"
|
|
if is_macos():
|
|
return "launchd"
|
|
if windows and is_windows():
|
|
return "windows"
|
|
return None
|
|
|
|
|
|
def _service_call(backend: str, verb: str, system: bool | None = False) -> None:
|
|
"""Run ``verb`` (start/stop/restart/uninstall) on ``backend``. Names resolve at call time so tests
|
|
can monkeypatch them; only systemd takes a scope, and ``system=None`` omits it (wizard restart)."""
|
|
if backend == "windows":
|
|
return getattr(_gw_windows(), verb)()
|
|
if backend == "launchd":
|
|
return globals()[f"launchd_{verb}"]()
|
|
fn = globals()[f"systemd_{verb}"]
|
|
return fn() if system is None else fn(system=system)
|
|
|
|
|
|
def _wizard_offer_service_action(action: str, question: str, failed_label: str, **kwargs) -> None:
|
|
"""Wizard start/restart prompt; prints remediation instead when system scope would need root."""
|
|
if supports_systemd_services() and _system_scope_wizard_would_need_root():
|
|
_print_system_scope_remediation(action)
|
|
elif prompt_yes_no(question, True):
|
|
_setup_service_action(action, failed_label=failed_label, **kwargs)
|
|
|
|
|
|
def _setup_service_action(
|
|
action: str, *, failed_label: str, windows: bool = True, system: bool = False
|
|
) -> None:
|
|
"""Run a wizard service start/restart, printing remediation instead of raising. ``windows=False``
|
|
skips Windows (pre-platform status block never offers it); ``system`` is a fresh install's scope."""
|
|
try:
|
|
backend = _service_backend(windows=windows)
|
|
if backend is not None:
|
|
_service_call(backend, action, None if action == "restart" else system)
|
|
elif action == "restart" and windows:
|
|
stop_profile_gateway()
|
|
print_info("Start manually: hermes gateway")
|
|
except UserSystemdUnavailableError as e:
|
|
print_error(f" {failed_label} — user systemd not reachable:")
|
|
_print_indented(str(e))
|
|
except SystemScopeRequiresRootError as e:
|
|
# Defense in depth: the wizard's root pre-check should have caught this.
|
|
print_error(f" {failed_label}: {e}")
|
|
_print_system_scope_remediation(action)
|
|
except subprocess.CalledProcessError as e:
|
|
print_error(f" {failed_label}: {e}")
|
|
|
|
|
|
_WIZARD_BANNER = (
|
|
"┌─────────────────────────────────────────────────────────┐",
|
|
"│ ☤ Gateway Setup │",
|
|
"├─────────────────────────────────────────────────────────┤",
|
|
"│ Configure messaging platforms and the gateway service. │",
|
|
"│ Press Ctrl+C at any time to exit. │",
|
|
"└─────────────────────────────────────────────────────────┘",
|
|
)
|
|
_WIZARD_BACKEND_LABELS = {"systemd": "systemd", "launchd": "launchd", "windows": "Scheduled Task"}
|
|
# Post-setup guidance when no service backend applies, keyed by the fallthrough reason.
|
|
_WIZARD_NO_SERVICE_LINES = {
|
|
"wsl": (
|
|
" WSL detected but systemd is not running.", " Run in foreground: hermes gateway run",
|
|
" For persistence: tmux new -s hermes 'hermes gateway run'",
|
|
" To enable systemd: add systemd=true to /etc/wsl.conf, then 'wsl --shutdown'",
|
|
),
|
|
"termux": (
|
|
" Termux does not use systemd/launchd services.", " Run in foreground: hermes gateway run",
|
|
" Or start it manually in the background (best effort): nohup hermes gateway run >{home}/logs/gateway.log 2>&1 &",
|
|
),
|
|
"unsupported": (
|
|
" Service install not supported on this platform.", " Run in foreground: hermes gateway run",
|
|
),
|
|
}
|
|
|
|
|
|
def _wizard_service_status_block() -> None:
|
|
"""Pre-platform service status: warnings, then offer to start an installed-but-stopped service."""
|
|
print()
|
|
service_installed = _is_service_installed()
|
|
service_running = _is_service_running()
|
|
|
|
if supports_systemd_services() and has_conflicting_systemd_units():
|
|
print_systemd_scope_conflict_warning()
|
|
print()
|
|
|
|
if supports_systemd_services() and has_legacy_hermes_units():
|
|
print_legacy_unit_warning()
|
|
print()
|
|
|
|
if service_installed and service_running:
|
|
print_success("Gateway service is installed and running.")
|
|
elif service_installed:
|
|
print_warning("Gateway service is installed but not running.")
|
|
_wizard_offer_service_action("start", " Start it now?", "Failed to start", windows=False)
|
|
else:
|
|
print_info("Gateway service is not installed yet.")
|
|
print_info("You'll be offered to install it after configuring platforms.")
|
|
|
|
|
|
def _wizard_platform_loop() -> None:
|
|
while True:
|
|
print()
|
|
print_header("Messaging Platforms")
|
|
|
|
platforms = _all_platforms()
|
|
menu_items = [f"{p['emoji']} {p['label']} ({_platform_status(p)})" for p in platforms] + ["Done"]
|
|
choice = prompt_choice("Select a platform to configure:", menu_items, len(menu_items) - 1)
|
|
if choice == len(platforms):
|
|
break
|
|
_configure_platform(platforms[choice])
|
|
|
|
|
|
def _wizard_install_service(backend: str) -> None:
|
|
"""Fresh install from the wizard: ask start-now / start-on-login, install, then start."""
|
|
wsl_note = " (note: services may not survive WSL restarts)" if is_wsl() else ""
|
|
start_now = prompt_yes_no(" Start the gateway now?", True)
|
|
start_on_login = prompt_yes_no(
|
|
f" Start the gateway automatically on login/boot as a {_WIZARD_BACKEND_LABELS[backend]} service?"
|
|
f"{wsl_note}",
|
|
True,
|
|
)
|
|
if not (start_now or start_on_login):
|
|
print_info(" Skipped start and auto-start setup.")
|
|
print_info(" You can install later: hermes gateway install")
|
|
if supports_systemd_services():
|
|
print_info(" Or as a boot-time service: sudo hermes gateway install --system")
|
|
print_info(" Or run in foreground: hermes gateway run")
|
|
return
|
|
try:
|
|
installed_scope, did_install = None, True
|
|
if backend == "systemd":
|
|
installed_scope, did_install = install_linux_gateway_from_setup(
|
|
force=False, enable_on_startup=start_on_login
|
|
)
|
|
elif backend == "launchd":
|
|
launchd_install(force=False)
|
|
else:
|
|
_gw_windows().install(force=False)
|
|
print()
|
|
if did_install and start_now:
|
|
_setup_service_action("start", failed_label="Start failed", system=installed_scope == "system")
|
|
except subprocess.CalledProcessError as e:
|
|
print_error(f" Install failed: {e}")
|
|
print_info(" You can try manually: hermes gateway install")
|
|
|
|
|
|
def _wizard_post_setup() -> None:
|
|
"""Offer to install/start/restart the gateway once at least one platform has progress."""
|
|
print()
|
|
print(color("─" * 58, Colors.DIM))
|
|
if _served_profile_needs_no_service():
|
|
return
|
|
service_installed = _is_service_installed()
|
|
service_running = _is_service_running()
|
|
|
|
if service_running:
|
|
_wizard_offer_service_action("restart", " Restart the gateway to pick up changes?", "Restart failed")
|
|
elif service_installed:
|
|
_wizard_offer_service_action("start", " Start the gateway service?", "Start failed")
|
|
else:
|
|
print()
|
|
backend = _service_backend()
|
|
if backend is not None:
|
|
_wizard_install_service(backend)
|
|
return
|
|
if is_wsl():
|
|
reason, home = "wsl", ""
|
|
elif is_termux():
|
|
from hermes_constants import display_hermes_home as _dhh
|
|
reason, home = "termux", _dhh()
|
|
else:
|
|
reason, home = "unsupported", ""
|
|
_print_info_lines(*(line.format(home=home) for line in _WIZARD_NO_SERVICE_LINES[reason]))
|
|
|
|
|
|
def gateway_setup():
|
|
"""Interactive setup for messaging platforms + gateway service."""
|
|
if is_managed():
|
|
managed_error("run gateway setup")
|
|
return
|
|
|
|
print()
|
|
for banner_line in _WIZARD_BANNER:
|
|
print(color(banner_line, Colors.MAGENTA))
|
|
|
|
_wizard_service_status_block()
|
|
_wizard_platform_loop()
|
|
|
|
# Meaningful progress on any platform; ``_platform_status`` already handles plugin dual states.
|
|
def _is_progress(status: str) -> bool:
|
|
s = status.lower()
|
|
return not (s == "not configured" or s.startswith("partially") or s.startswith("plugin disabled"))
|
|
|
|
if any(_is_progress(_platform_status(p)) for p in _all_platforms()):
|
|
_wizard_post_setup()
|
|
else:
|
|
print()
|
|
print_info("No platforms configured. Run 'hermes gateway setup' when ready.")
|
|
|
|
print()
|
|
|
|
|
|
# =============================================================================
|
|
# Main Command Handler
|
|
# =============================================================================
|
|
|
|
def _dispatch_via_service_manager_if_s6(action: str, profile: str | None = None) -> bool:
|
|
"""Dispatch start/stop/restart via s6 inside an s6 container; True iff dispatched (caller returns).
|
|
Profile defaults to the current one; missing slot / s6 errors become actionable CLI messages."""
|
|
from hermes_cli.service_manager import (
|
|
GatewayNotRegisteredError, detect_service_manager, get_service_manager,
|
|
register_unregistered_profile_gateway,
|
|
)
|
|
|
|
if detect_service_manager() != "s6":
|
|
return False
|
|
if profile is None:
|
|
profile = _current_profile_name() # root home (Docker /opt/data included) is gateway-default
|
|
mgr = get_service_manager()
|
|
if action not in ("start", "stop", "restart"):
|
|
return False
|
|
service = f"gateway-{profile}"
|
|
try:
|
|
try:
|
|
getattr(mgr, action)(service)
|
|
except GatewayNotRegisteredError:
|
|
# A profile created from the HOST against a bind-mounted home has a directory but no
|
|
# slot (`profile create` cannot reach the container's /run/service). Only `start`
|
|
# repairs that; stop/restart on a missing slot stay an error.
|
|
if action != "start" or not register_unregistered_profile_gateway(mgr, profile):
|
|
raise
|
|
print(f"✓ registered the s6 gateway slot for profile {profile!r}")
|
|
mgr.start(service)
|
|
except (RuntimeError, ValueError, OSError) as exc: # S6Error is a RuntimeError
|
|
print(f"✗ {exc}")
|
|
sys.exit(1)
|
|
return True
|
|
|
|
|
|
def _dispatch_all_via_service_manager_if_s6(action: str) -> bool:
|
|
"""Dispatch ``--all`` stop/restart to every registered profile gateway under s6; True iff dispatched.
|
|
A bare pkill is seen by s6-supervise as a crash and restarted ~1s later; the service manager flips
|
|
``want up``/``want down`` correctly. ``start --all`` is not a CLI surface."""
|
|
from hermes_cli.service_manager import (detect_service_manager, get_service_manager)
|
|
if detect_service_manager() != "s6" or action not in ("stop", "restart"):
|
|
return False
|
|
mgr = get_service_manager()
|
|
profiles = mgr.list_profile_gateways()
|
|
if not profiles:
|
|
print("✗ No profile gateways registered under s6")
|
|
return True
|
|
fn = mgr.stop if action == "stop" else mgr.restart
|
|
errors: list[tuple[str, Exception]] = []
|
|
for profile in profiles:
|
|
try:
|
|
fn(f"gateway-{profile}")
|
|
except Exception as exc: # noqa: BLE001 — report and continue
|
|
errors.append((profile, exc))
|
|
succeeded = len(profiles) - len(errors)
|
|
verb = "stopped" if action == "stop" else "restarted"
|
|
if succeeded:
|
|
print(f"✓ {verb.capitalize()} {succeeded} profile gateway(s) under s6")
|
|
for profile, exc in errors:
|
|
print(f"✗ Could not {action} gateway-{profile}: {exc}")
|
|
return True
|
|
|
|
|
|
def gateway_command(args):
|
|
"""Handle gateway subcommands."""
|
|
try:
|
|
return _gateway_command_inner(args)
|
|
except UserSystemdUnavailableError as e:
|
|
# Actionable message, not a traceback, when the user D-Bus session is unreachable.
|
|
print_error("User systemd not reachable:")
|
|
_print_indented(str(e))
|
|
sys.exit(1)
|
|
except SystemScopeRequiresRootError as e:
|
|
# System-scope action typed without sudo; the wizard intercepts this earlier with guidance.
|
|
print(str(e))
|
|
sys.exit(1)
|
|
except (subprocess.CalledProcessError, RuntimeError) as e:
|
|
# systemctl exited non-zero or is missing entirely: guidance, not a traceback.
|
|
from hermes_cli.gateway_command_errors import explain_service_failure
|
|
lines = explain_service_failure(e)
|
|
if lines is None:
|
|
raise
|
|
print_error(lines[0])
|
|
_print_indented("\n".join(lines[1:]))
|
|
sys.exit(1)
|
|
|
|
|
|
def _maybe_redirect_run_to_s6_supervision(args) -> bool:
|
|
"""Inside an s6 container, upgrade bare ``gateway run`` to the supervised s6 longrun; True iff dispatched.
|
|
``HERMES_S6_SUPERVISED_CHILD`` (set by ``S6ServiceManager._render_run_script``) marks the supervised
|
|
child, which must run in foreground or we'd recurse run → start → run; ``--no-supervise`` /
|
|
HERMES_GATEWAY_NO_SUPERVISE=1 opts out (CI smoke, debugging)."""
|
|
no_supervise = getattr(args, "no_supervise", False) or \
|
|
os.environ.get("HERMES_GATEWAY_NO_SUPERVISE", "").lower() in ("1", "true", "yes")
|
|
# HERMES_S6_SUPERVISED_CHILD: we ARE the supervised child; fall through so the gateway starts.
|
|
if no_supervise or os.environ.get("HERMES_S6_SUPERVISED_CHILD"):
|
|
return False
|
|
if not _dispatch_via_service_manager_if_s6("start"):
|
|
return False
|
|
# This process never reaches a GatewayRunner, so the watchdog armed by hermes_cli.main's argv
|
|
# fast-path has no other disarm site: the in-process heartbeat below parks with zero CPU and no
|
|
# progress lease, which the watchdog reads as a startup deadlock and os._exit(75)s the CMD process.
|
|
from hermes_startup_watchdog import disarm_startup_watchdog
|
|
|
|
disarm_startup_watchdog()
|
|
# Breadcrumb on stderr (keep stdout clean for scripts); gateway logs follow via s6-log.
|
|
print(
|
|
"→ gateway is now running under s6 supervision (auto-restart on crash,\n"
|
|
" dashboard supervised alongside if HERMES_DASHBOARD is set).\n"
|
|
" This is the recommended setup for the s6 container image — the\n"
|
|
" gateway will keep running even if it crashes.\n"
|
|
" Use `--no-supervise` (or HERMES_GATEWAY_NO_SUPERVISE=1) to opt out\n"
|
|
" and get the pre-s6 foreground behavior instead.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
# Keep the CMD process alive as a heartbeat so the container survives gateway flaps (`docker stop`
|
|
# SIGTERMs it). Prefer `sleep infinity` (frees the interpreter); execvp only returns by raising
|
|
# (ENOENT with a clobbered PATH / no `sleep`), which used to crash containers.
|
|
try:
|
|
# The supervised gateway's lifetime is independent of this process — s6-supervise restarts it on
|
|
# crash, and we don't want the container to exit when the gateway flaps. The CMD process keeps /init
|
|
# alive until `docker stop` sends SIGTERM, at which point /init runs stage 3 shutdown (which tears
|
|
# down the supervised gateway cleanly). Prefer `sleep infinity` (matches the static main-hermes
|
|
# service's pattern in docker/s6-rc.d/main-hermes/run, and frees the Python interpreter — the
|
|
# heartbeat is a tiny `sleep` process, not a resident interpreter). But `os.execvp` does a PATH
|
|
# lookup for the `sleep` binary and historically crashed the whole container with FileNotFoundError
|
|
# when PATH was empty/truncated/clobbered at this point — e.g. after user customizations rewrote
|
|
# PATH, or on minimal images without `sleep` on PATH (issue #36208). Fall back to an in-process
|
|
# block (no external binary, can't fail on PATH) so the container keeps running instead of dying
|
|
# during boot.
|
|
os.execvp("sleep", ["sleep", "infinity"])
|
|
except OSError:
|
|
print(
|
|
"→ `sleep` is unavailable; keeping the s6 CMD process alive "
|
|
"in-process until the container is stopped.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
_block_until_terminated()
|
|
return True # unreachable on the execvp success path
|
|
|
|
|
|
def _block_until_terminated() -> None:
|
|
"""Heartbeat when ``execvp("sleep")`` fails. SIGTERM exits 128+signum so ``docker stop`` is clean;
|
|
``Event().wait()`` covers platforms without ``signal.pause()``.
|
|
|
|
Fallback heartbeat for when ``os.execvp("sleep", ...)`` can't run (``sleep`` missing from PATH — issue
|
|
#36208). Installs a SIGTERM handler that exits with the conventional 128+signum code so ``docker stop``
|
|
produces a clean, expected exit, then blocks on ``signal.pause()``. Windows) — although this path only
|
|
runs inside the s6 Linux container image, the fallback keeps the helper safe to import and unit-test
|
|
anywhere.
|
|
"""
|
|
signal.signal(signal.SIGTERM, lambda signum, _frame: sys.exit(128 + signum))
|
|
pause = getattr(signal, "pause", None)
|
|
if pause is not None:
|
|
while True:
|
|
pause()
|
|
else: # pragma: no cover - non-Unix fallback, not exercised in the s6 image
|
|
import threading
|
|
threading.Event().wait()
|
|
|
|
|
|
def _installed_service_kind_for(windows) -> str | None:
|
|
"""``"systemd"`` / ``"launchd"`` when the unit/plist exists, else ``"windows"`` iff ``windows()``
|
|
(a thunk so it runs last, like every caller's original ladder), else None."""
|
|
if _systemd_unit_installed():
|
|
return "systemd"
|
|
if is_macos() and get_launchd_plist_path().exists():
|
|
return "launchd"
|
|
return "windows" if windows() else None
|
|
|
|
|
|
def _installed_service_kind() -> str | None:
|
|
"""Installed service kind; stricter than ``_service_backend`` (unit/plist/task must exist)."""
|
|
return _installed_service_kind_for(lambda: is_windows() and _gw_windows().is_installed())
|
|
|
|
|
|
def _stop_installed_service(system: bool) -> bool:
|
|
"""Stop the installed systemd/launchd/Windows service. Returns True if one was stopped."""
|
|
kind = _installed_service_kind()
|
|
if kind is None:
|
|
return False
|
|
# SystemScopeRequiresRootError is a RuntimeError and must propagate from systemd_stop.
|
|
try:
|
|
_service_call(kind, "stop", system)
|
|
return True
|
|
except (subprocess.CalledProcessError, *((RuntimeError,) if kind == "windows" else ())):
|
|
return False
|
|
|
|
|
|
def _refuse_from_inside_gateway(verb: str, reason: str) -> None:
|
|
"""Refuse self-targeting stop/restart/uninstall from inside the gateway process (#92560)."""
|
|
from tools.process_registry import _is_supervised_gateway_process
|
|
if _is_supervised_gateway_process():
|
|
print_error(
|
|
f"Refusing to {verb} the gateway from inside the gateway process.\n"
|
|
f"This command was blocked to prevent {reason}.\n"
|
|
f"Use `hermes gateway {verb}` from a shell outside the running gateway."
|
|
)
|
|
sys.exit(1)
|
|
|
|
|
|
def _print_lines(*lines: str) -> None:
|
|
for line in lines:
|
|
print(line)
|
|
|
|
|
|
def _print_runtime_health() -> None:
|
|
runtime_lines = _runtime_health_lines()
|
|
if runtime_lines:
|
|
print()
|
|
print("Recent gateway health:")
|
|
for line in runtime_lines:
|
|
print(f" {line}")
|
|
|
|
|
|
def _cmd_run(args):
|
|
if _maybe_redirect_run_to_s6_supervision(args):
|
|
return # unreachable; execvp doesn't return
|
|
if getattr(args, "external_supervisor", False):
|
|
os.environ[EXTERNAL_GATEWAY_SUPERVISOR_ENV] = "1"
|
|
run_gateway(
|
|
getattr(args, "verbose", 0), quiet=getattr(args, "quiet", False),
|
|
replace=getattr(args, "replace", False), force=getattr(args, "force", False),
|
|
)
|
|
|
|
|
|
def _cmd_setup(args):
|
|
gateway_setup()
|
|
|
|
|
|
_WSL_FOREGROUND_HINT = (
|
|
"", " hermes gateway run # direct foreground",
|
|
" tmux new -s hermes 'hermes gateway run' # persistent via tmux",
|
|
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # background",
|
|
)
|
|
# ``(exit_code, *lines)`` when a subcommand has no service backend, keyed by (subcommand, reason).
|
|
# Reasons in check order: "termux", "wsl" (no operational systemd), "s6" / "container", "unsupported".
|
|
# ``None`` exit code means plain return.
|
|
_NO_BACKEND_MESSAGES = {
|
|
("install", "termux"): (1,
|
|
"Gateway service installation is not supported on Termux.", "Run manually: hermes gateway"),
|
|
("install", "wsl"): (1,
|
|
"WSL detected but systemd is not running.",
|
|
"Either enable systemd (add systemd=true to /etc/wsl.conf and restart WSL)",
|
|
"or run the gateway in foreground mode:", *_WSL_FOREGROUND_HINT),
|
|
("install", "s6"): (None,
|
|
"Per-profile gateways are auto-registered when you create a profile.", "",
|
|
" hermes profile create <name> # creates the s6 service slot",
|
|
" hermes -p <name> gateway start # bring it up via s6",
|
|
" hermes status # see currently-supervised gateways"),
|
|
("install", "container"): (0,
|
|
"Service installation is not needed inside a Docker container.",
|
|
"The container runtime is your service manager — use Docker restart policies instead:", "",
|
|
" docker run --restart unless-stopped ... # auto-restart on crash/reboot",
|
|
" docker restart <container> # manual restart", "",
|
|
"To run the gateway: hermes gateway run"),
|
|
("install", "unsupported"): (1,
|
|
"Service installation not supported on this platform.", "Run manually: hermes gateway run"),
|
|
("uninstall", "termux"): (1,
|
|
"Gateway service uninstall is not supported on Termux because there is no managed service to remove.",
|
|
"Stop manual runs with: hermes gateway stop"),
|
|
("uninstall", "s6"): (None,
|
|
"Per-profile gateways are auto-unregistered when you delete the profile.", "",
|
|
" hermes profile delete <name> # tears down the s6 service slot",
|
|
" hermes -p <name> gateway stop # stop without deleting the profile"),
|
|
("uninstall", "container"): (0,
|
|
"Service uninstall is not applicable inside a Docker container.",
|
|
"To stop the gateway, stop or remove the container:", "",
|
|
" docker stop <container>", " docker rm <container>"),
|
|
("uninstall", "unsupported"): (1,
|
|
"Running the gateway as a background service is not available on this platform "
|
|
"(no systemd, launchd or Scheduled Tasks), so there is nothing to uninstall.",
|
|
"Stop a manually started gateway with: hermes gateway stop"),
|
|
("start", "termux"): (1,
|
|
"Gateway service start is not supported on Termux because there is no system service manager.",
|
|
"Run manually: hermes gateway"),
|
|
("start", "wsl"): (1,
|
|
"WSL detected but systemd is not available.",
|
|
"Run the gateway in foreground mode instead:", *_WSL_FOREGROUND_HINT, "",
|
|
"To enable systemd: add systemd=true to /etc/wsl.conf and run 'wsl --shutdown' from PowerShell."),
|
|
("start", "container"): (0,
|
|
"Service start is not applicable inside a Docker container.",
|
|
"The gateway runs as the container's main process.", "",
|
|
" docker start <container> # start a stopped container",
|
|
" docker restart <container> # restart a running container", "",
|
|
"Or run the gateway directly: hermes gateway run"),
|
|
("start", "unsupported"): (1,
|
|
"Running the gateway as a background service is not available on this platform "
|
|
"(no systemd, launchd or Scheduled Tasks).",
|
|
"Run it directly with: hermes gateway run"),
|
|
}
|
|
|
|
|
|
def _no_backend_exit(subcommand: str, reason: str) -> None:
|
|
code, *lines = _NO_BACKEND_MESSAGES[(subcommand, reason)]
|
|
_print_lines(*lines)
|
|
if code is not None:
|
|
sys.exit(code)
|
|
|
|
|
|
def _handle_no_backend(subcommand: str, *, wsl: bool, s6: bool) -> None:
|
|
"""Fallthrough when no service backend matched. Predicate order: WSL (only when ``wsl``) ->
|
|
container (s6 slot hint only when ``s6``; ``start`` reaches here only when s6 isn't running) ->
|
|
unsupported."""
|
|
if wsl and is_wsl():
|
|
reason = "wsl"
|
|
elif is_container():
|
|
reason = "s6" if s6 and _running_under_s6() else "container"
|
|
else:
|
|
reason = "unsupported"
|
|
_no_backend_exit(subcommand, reason)
|
|
|
|
|
|
def _install_systemd_from_cli(args, *, force: bool, system: bool, run_as_user) -> None:
|
|
if is_wsl():
|
|
print_warning("WSL detected — systemd services may not survive WSL restarts.")
|
|
_print_info_lines(
|
|
" Consider running in foreground instead: hermes gateway run",
|
|
" Or use tmux/screen for persistence: tmux new -s hermes 'hermes gateway run'",
|
|
)
|
|
print()
|
|
# Honor --start-now/--start-on-login; else prompt on a TTY, default True headless.
|
|
non_interactive = not (hasattr(sys.stdin, "isatty") and sys.stdin.isatty())
|
|
|
|
def _flag(name: str, question: str) -> bool:
|
|
value = getattr(args, name, None)
|
|
if value is not None:
|
|
return value
|
|
return prompt_yes_no(question, True) if not non_interactive else True
|
|
|
|
start_now = _flag("start_now", "Start the gateway now after installing the service?")
|
|
start_on_login = _flag("start_on_login", "Start the gateway automatically on login/boot with systemd?")
|
|
systemd_install(
|
|
force=force, system=system, run_as_user=run_as_user,
|
|
enable_on_startup=start_on_login, non_interactive=non_interactive,
|
|
)
|
|
if start_now:
|
|
systemd_start(system=system)
|
|
|
|
|
|
def _cmd_install(args):
|
|
if is_managed():
|
|
managed_error("install gateway service")
|
|
return
|
|
force = getattr(args, "force", False)
|
|
# `--force` doubles as the reinstall flag here; a served profile's unit would only ever exit 78.
|
|
_guard_named_profile_under_multiplexer(force=force)
|
|
system = getattr(args, "system", False)
|
|
run_as_user = getattr(args, "run_as_user", None)
|
|
if is_termux():
|
|
_no_backend_exit("install", "termux")
|
|
backend = _service_backend()
|
|
if backend == "systemd":
|
|
if refuses_container_user_scope_install(system):
|
|
sys.exit(1)
|
|
_install_systemd_from_cli(args, force=force, system=system, run_as_user=run_as_user)
|
|
elif backend == "launchd":
|
|
launchd_install(force)
|
|
elif backend == "windows":
|
|
_gw_windows().install(
|
|
force=force,
|
|
start_now=getattr(args, 'start_now', None),
|
|
start_on_login=getattr(args, 'start_on_login', None),
|
|
elevated_handoff=getattr(args, 'elevated_handoff', False),
|
|
)
|
|
else:
|
|
_handle_no_backend("install", wsl=True, s6=True)
|
|
|
|
|
|
def _cmd_uninstall(args):
|
|
_refuse_from_inside_gateway("uninstall", "the gateway from terminating itself")
|
|
if is_managed():
|
|
managed_error("uninstall gateway service")
|
|
return
|
|
system = getattr(args, "system", False)
|
|
if is_termux():
|
|
_no_backend_exit("uninstall", "termux")
|
|
backend = _service_backend()
|
|
if backend is not None:
|
|
_service_call(backend, "uninstall", system)
|
|
else:
|
|
_handle_no_backend("uninstall", wsl=False, s6=True)
|
|
|
|
|
|
def _cmd_start(args):
|
|
system = getattr(args, "system", False)
|
|
start_all = getattr(args, "all", False)
|
|
_guard_named_profile_under_multiplexer(force=getattr(args, "force", False))
|
|
if not start_all and _dispatch_via_service_manager_if_s6("start"):
|
|
return
|
|
if start_all:
|
|
killed = kill_gateway_processes(all_profiles=True)
|
|
if killed:
|
|
print(f"✓ Killed {killed} stale gateway process(es) across all profiles")
|
|
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
|
|
_wait_for_api_server_port_free()
|
|
|
|
if is_termux():
|
|
_no_backend_exit("start", "termux")
|
|
backend = _service_backend()
|
|
if backend is not None:
|
|
_service_call(backend, "start", system)
|
|
else:
|
|
_handle_no_backend("start", wsl=True, s6=False)
|
|
|
|
|
|
def _cmd_stop(args):
|
|
_refuse_from_inside_gateway("stop", "restart loops")
|
|
stop_all = getattr(args, "all", False)
|
|
system = getattr(args, "system", False)
|
|
if not stop_all and not find_gateway_pids() and named_profile_served_by_running_multiplexer():
|
|
# A served profile owns no gateway to stop; "No gateway running for this profile" (exit 0) would
|
|
# contradict `gateway status` ("running via the default-profile multiplexer") on the same profile.
|
|
# A `--force`-started separate gateway HAS a pid of its own and is stopped normally.
|
|
print_error(
|
|
f"The default gateway is running as a profile multiplexer and serves profile "
|
|
f"'{_current_profile_name()}' — there is no separate gateway for this profile to stop."
|
|
)
|
|
print(" Stop or restart the multiplexer from the default profile instead:")
|
|
print()
|
|
print(" hermes gateway stop # takes every served profile offline")
|
|
print(" hermes gateway restart")
|
|
sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)
|
|
# Under s6 a bare pkill is seen as a crash and restarted; go through the supervisor.
|
|
if stop_all and _dispatch_all_via_service_manager_if_s6("stop"):
|
|
return
|
|
if not stop_all and _dispatch_via_service_manager_if_s6("stop"):
|
|
return
|
|
|
|
service_available = _stop_installed_service(system)
|
|
if stop_all:
|
|
total = kill_gateway_processes(all_profiles=True) + (1 if service_available else 0)
|
|
if total:
|
|
print(f"✓ Stopped {total} gateway process(es) across all profiles")
|
|
else:
|
|
print("✗ No gateway processes found")
|
|
elif not service_available:
|
|
if stop_profile_gateway():
|
|
print("✓ Stopped gateway for this profile")
|
|
else:
|
|
print("✗ No gateway running for this profile")
|
|
else:
|
|
print(f"✓ Stopped {get_service_name()} service")
|
|
|
|
|
|
def _restart_all(system: bool) -> None:
|
|
service_stopped = _stop_installed_service(system)
|
|
total = kill_gateway_processes(all_profiles=True) + (1 if service_stopped else 0)
|
|
if total:
|
|
print(f"✓ Stopped {total} gateway process(es) across all profiles")
|
|
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
|
|
_wait_for_api_server_port_free()
|
|
|
|
print("Starting gateway...")
|
|
# Even without a registered task, gateway_windows.start() uses the detached launcher.
|
|
kind = _installed_service_kind_for(is_windows)
|
|
if kind is None:
|
|
run_gateway(verbose=0)
|
|
else:
|
|
_service_call(kind, "start", system)
|
|
|
|
|
|
def _cmd_restart(args):
|
|
_refuse_from_inside_gateway("restart", "restart loops")
|
|
system = getattr(args, "system", False)
|
|
restart_all = getattr(args, "all", False)
|
|
force = getattr(args, "force", False)
|
|
_guard_named_profile_under_multiplexer(force=force)
|
|
if restart_all and _dispatch_all_via_service_manager_if_s6("restart"):
|
|
return
|
|
if not restart_all and _dispatch_via_service_manager_if_s6("restart"):
|
|
return
|
|
if restart_all:
|
|
_restart_all(system)
|
|
return
|
|
|
|
# The Windows restart path handles both registered installs and detached restarts.
|
|
kind = _installed_service_kind_for(is_windows)
|
|
service_configured = kind is not None and (kind != "windows" or _gw_windows().is_installed())
|
|
if kind is not None:
|
|
swallow = (RuntimeError, OSError) if kind == "windows" else ()
|
|
try:
|
|
_service_call(kind, "restart", system)
|
|
return
|
|
except (subprocess.CalledProcessError, *swallow):
|
|
pass
|
|
|
|
# Linger only explains a FAILED systemd unit restart. Without an installed unit the
|
|
# detached run below is the restart; bailing here left `hermes gateway restart` a
|
|
# silent exit-0 no-op on any Linux login session (Desktop read it as success).
|
|
if kind == "systemd" and supports_systemd_services():
|
|
linger_ok, _detail = get_systemd_linger_status()
|
|
if linger_ok is not True:
|
|
import getpass
|
|
_print_lines(
|
|
"", "⚠ Cannot restart gateway as a service — linger is not enabled.",
|
|
" The gateway user service requires linger to function on headless servers.", "",
|
|
f" Run: sudo loginctl enable-linger {getpass.getuser()}", "",
|
|
" Then restart the gateway:", " hermes gateway restart",
|
|
)
|
|
return
|
|
|
|
if service_configured:
|
|
_print_lines(
|
|
"", "✗ Gateway service restart failed.",
|
|
" The service definition exists, but the service manager did not recover it.",
|
|
" Fix the service, then retry: hermes gateway start",
|
|
)
|
|
sys.exit(1)
|
|
|
|
# A gateway that declares an external supervisor (custom launchd agent / unit running
|
|
# `gateway run --external-supervisor`) restarts by exiting back to it: the stop + foreground
|
|
# run below would stamp this CLI's PID as the gateway and wedge every respawn (#110637).
|
|
from gateway.status import get_running_pid
|
|
from hermes_cli.gateway_supervised_restart import (
|
|
gateway_declares_external_supervisor, restart_externally_supervised_gateway,
|
|
)
|
|
supervised_pid = get_running_pid()
|
|
if supervised_pid and gateway_declares_external_supervisor(supervised_pid):
|
|
restart_externally_supervised_gateway(supervised_pid)
|
|
return
|
|
|
|
if stop_profile_gateway():
|
|
print("✓ Stopped gateway for this profile")
|
|
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
|
|
_wait_for_api_server_port_free()
|
|
print("Starting gateway...")
|
|
run_gateway(verbose=0, force=force)
|
|
|
|
|
|
# ``hermes gateway status`` hints for a manually-run / stopped gateway, keyed by host kind.
|
|
_STATUS_RUNNING_HINTS = {
|
|
"termux": ("Termux note:", " Android may stop background jobs when Termux is suspended"),
|
|
"wsl": (
|
|
"WSL note:", " The gateway is running in foreground/manual mode (recommended for WSL).",
|
|
" Use tmux or screen for persistence across terminal closes.",
|
|
),
|
|
"windows": ("To install as a Windows Scheduled Task (auto-start on login):", " hermes gateway install"),
|
|
"other": (
|
|
"To install as a service:", " hermes gateway install", " sudo hermes gateway install --system",
|
|
),
|
|
}
|
|
_STATUS_STOPPED_HINTS = {
|
|
"termux": (
|
|
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # Best-effort background start",
|
|
),
|
|
"wsl": (
|
|
" tmux new -s hermes 'hermes gateway run' # persistent via tmux",
|
|
" nohup hermes gateway run > ~/.hermes/logs/gateway.log 2>&1 & # background",
|
|
),
|
|
"windows": (" hermes gateway install # Install as Windows Scheduled Task (auto-start on login)",),
|
|
"other": (
|
|
" hermes gateway install # Install as user service",
|
|
" sudo hermes gateway install --system # Install as boot-time system service",
|
|
),
|
|
}
|
|
|
|
|
|
def _status_host_kind() -> str:
|
|
if is_termux():
|
|
return "termux"
|
|
if is_wsl():
|
|
return "wsl"
|
|
return "windows" if is_windows() else "other"
|
|
|
|
|
|
def _cmd_status(args):
|
|
deep = getattr(args, "deep", False)
|
|
full = getattr(args, "full", False)
|
|
system = getattr(args, "system", False)
|
|
snapshot = get_gateway_runtime_snapshot(system=system)
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
|
|
_windows_service_installed = is_windows() and _gw_windows().is_installed()
|
|
if not snapshot.running and named_profile_served_by_running_multiplexer():
|
|
# Satellite profile: the default multiplexer is the live inbound process for it.
|
|
print("✓ Gateway is running via the default-profile multiplexer")
|
|
print(" Manage it from the default profile: hermes gateway status")
|
|
_print_served_ingress_urls(get_active_profile_name())
|
|
_print_unserved_shared_ingress(get_active_profile_name())
|
|
elif (kind := _installed_service_kind_for(lambda: _windows_service_installed)) is not None:
|
|
if kind == "systemd":
|
|
systemd_status(deep, system=system, full=full)
|
|
elif kind == "launchd":
|
|
launchd_status(deep)
|
|
else:
|
|
_gw_windows().status(deep=deep)
|
|
_print_gateway_process_mismatch(snapshot)
|
|
_print_multiplex_standalone_reason()
|
|
_print_served_ingress_urls()
|
|
else:
|
|
pids = list(snapshot.gateway_pids)
|
|
if pids:
|
|
print(f"✓ Gateway is running (PID: {', '.join(map(str, pids))})")
|
|
print(" (Running manually, not as a system service)")
|
|
_print_runtime_health()
|
|
_print_multiplex_standalone_reason()
|
|
_print_served_ingress_urls()
|
|
print()
|
|
_print_lines(*_STATUS_RUNNING_HINTS[_status_host_kind()])
|
|
else:
|
|
print("✗ Gateway is not running")
|
|
_print_runtime_health()
|
|
print()
|
|
print("To start:")
|
|
print(" hermes gateway run # Run in foreground")
|
|
_print_lines(*_STATUS_STOPPED_HINTS[_status_host_kind()])
|
|
|
|
_print_other_profiles_gateway_status()
|
|
|
|
|
|
def _cmd_list(args):
|
|
_gateway_list()
|
|
|
|
|
|
def _cmd_migrate_legacy(args):
|
|
"""Stop, disable, and remove legacy Hermes gateway unit files (e.g. hermes.service)."""
|
|
dry_run = getattr(args, "dry_run", False)
|
|
yes = getattr(args, "yes", False)
|
|
if not supports_systemd_services() and not is_macos():
|
|
print("Legacy unit migration only applies to systemd-based Linux hosts.")
|
|
return
|
|
remove_legacy_hermes_units(interactive=not yes, dry_run=dry_run)
|
|
|
|
|
|
def _cmd_migrate(args):
|
|
from hermes_cli.gateway_migrate import cmd_migrate
|
|
cmd_migrate(args)
|
|
|
|
|
|
_GATEWAY_SUBCOMMANDS = {
|
|
None: _cmd_run, "run": _cmd_run, "setup": _cmd_setup, "install": _cmd_install,
|
|
"uninstall": _cmd_uninstall, "start": _cmd_start, "stop": _cmd_stop, "restart": _cmd_restart,
|
|
"status": _cmd_status, "list": _cmd_list, "migrate-legacy": _cmd_migrate_legacy, "migrate": _cmd_migrate,
|
|
}
|
|
|
|
|
|
def _gateway_command_inner(args):
|
|
handler = _GATEWAY_SUBCOMMANDS.get(getattr(args, "gateway_command", None))
|
|
if handler is not None:
|
|
handler(args)
|
|
|
|
|
|
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----
|
|
# Names external plugins imported from this module before the Sep 2026 decomposition.
|
|
# Internal code MUST NOT use these (scripts/check_compat_pointers.py fails CI if it does).
|
|
# The whole block is removed by reverting the commit that added it.
|
|
|
|
def print_systemd_linger_guidance() -> None:
|
|
"""Print the current linger status and the fix when it is disabled."""
|
|
linger_enabled, linger_detail = get_systemd_linger_status()
|
|
if linger_enabled is True:
|
|
print("✓ Systemd linger is enabled (service survives logout)")
|
|
elif linger_enabled is False:
|
|
print("⚠ Systemd linger is disabled (gateway may stop when you log out)")
|
|
print(" Run: sudo loginctl enable-linger $USER")
|
|
else:
|
|
print(f"⚠ Could not verify systemd linger ({linger_detail})")
|
|
print(" If you want the gateway user service to survive logout, run:")
|
|
print(" sudo loginctl enable-linger $USER")
|
|
|
|
|
|
_PLUGIN_COMPAT_LAZY = {
|
|
'DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT': ('gateway.restart', 'DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT'),
|
|
}
|
|
|
|
|
|
def __getattr__(name): # PEP 562 — lazy so no import cycles
|
|
target = _PLUGIN_COMPAT_LAZY.get(name)
|
|
if target is None:
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
import importlib
|
|
from hermes_cli.plugin_compat import warn_once
|
|
warn_once(__name__, name, *target)
|
|
return getattr(importlib.import_module(target[0]), target[1])
|
|
# ---- END PLUGIN-COMPAT ----
|