From ae85aaa366a8add2c05cb18bb63e280cb4d2b9f7 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 21 Sep 2026 00:16:46 -0700 Subject: [PATCH] refactor(gateway): move the launchd backend out of hermes_cli/gateway.py hermes_cli/gateway.py (6,511 lines) keeps the facade; the launchd (macOS LaunchAgent) cluster - plist generation/refresh, launchctl bootstrap/kickstart, start/stop/restart/status and the detached-process degrade path - moves to hermes_cli/gateway_launchd.py (41 names). Moved bodies reach facade helpers through _gw() (late binding on hermes_cli.gateway) so monkeypatches on the facade keep intercepting; the launchd-domain cache stays a facade global and is rebound through the facade (a moved global would have rebound the sibling module and left the facade reader None forever). Part of #116913 --- hermes_cli/gateway.py | 883 ++-------------------------------- hermes_cli/gateway_launchd.py | 860 +++++++++++++++++++++++++++++++++ 2 files changed, 906 insertions(+), 837 deletions(-) create mode 100644 hermes_cli/gateway_launchd.py diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index d3074cd8be..b5aa227b1c 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -9,7 +9,6 @@ from hermes_cli.cli_output import line_input import json import logging import os -import shlex import shutil import signal import socket @@ -31,9 +30,9 @@ if os.name == "posix": PROJECT_ROOT = Path(__file__).parent.parent.resolve() -from gateway.config import coerce_systemd_watchdog_seconds, load_gateway_config +from gateway.config import coerce_systemd_watchdog_seconds, load_gateway_config # noqa: F401 — resolved lazily by siblings through the facade from gateway.status import terminate_pid -from gateway.restart import ( +from gateway.restart import ( # noqa: F401 — resolved lazily by siblings through the facade DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT, EXTERNAL_GATEWAY_SUPERVISOR_ENV, GATEWAY_FATAL_CONFIG_EXIT_CODE, @@ -67,7 +66,7 @@ from hermes_cli.setup import ( prompt_yes_no, ) from hermes_cli.colors import Colors, color -from hermes_cli.gateway_service_unit import ( +from hermes_cli.gateway_service_unit import ( # noqa: F401 — resolved lazily by siblings through the facade _systemd_env_line, _installed_unit_ld_library_path, _ld_library_path_line, @@ -3447,662 +3446,55 @@ def systemd_status(deep: bool = False, system: bool = False, full: bool = False) # ============================================================================= -def get_launchd_label() -> str: - """Return the launchd service label, scoped per profile.""" - suffix = _profile_suffix() - return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway" +from hermes_cli.gateway_launchd import ( # noqa: E402,F401 — facade re-exports; tests patch here + get_launchd_label, + _probe_launchd_domain_for_label, + _launchd_domain, + _LAUNCHD_JOB_UNLOADED_EXIT_CODES, + _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES, + _launchd_error_indicates_unloaded, + _launchctl_domain_unsupported, + _LAUNCHCTL_BOOTSTRAP_EIO, + _launchctl_bootstrap, + _launchd_reload_log_path, + _append_launchd_reload_log, + _launchd_reload_budget, + _launchctl_supervised_pid, + _launchctl_label_supervising_process, + _retry_launchctl_bootstrap_until_registered, + _launchd_unsupported_marker_path, + _write_launchd_unsupported_marker, + _clear_launchd_unsupported_marker, + _launchd_unsupported_marker_exists, + _gateway_run_command, + _timestamped_stderr_gateway_command, + _spawn_detached_gateway, + _launchd_fallback_to_detached, + _launchd_degrade_or_raise, + generate_launchd_plist, + launchd_plist_is_current, + _spawn_deferred_launchd_reload, + refresh_launchd_plist_if_needed, + launchd_install, + launchd_uninstall, + launchd_start, + _launchctl_kickstart_current, + _launchd_bootstrap_and_kickstart, + _launchd_ok, + launchd_stop, + _launchd_kickstart, + _wait_for_launchd_service_pid, + launchd_restart, + LAUNCHD_SUPERVISION_VERIFY_TIMEOUT, + wait_for_launchd_gateway_supervision, + launchd_status, +) # Cached launchd domain — probe once per process invocation. _resolved_launchd_domain: str | None = None -def _probe_launchd_domain_for_label(label: str) -> str: - """Launchd domain managing ``label`` (uncached): ``gui/`` (Aqua), then ``user/`` - (Background/SSH), else the ``launchctl managername`` heuristic. Sibling profiles may live in - different domains, so never reuse the cached ``_launchd_domain()`` for another label.""" - uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows - gui_domain, user_domain = f"gui/{uid}", f"user/{uid}" - - launchctl_errors = (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) - for domain in (gui_domain, user_domain): - try: - subprocess.run(["launchctl", "print", f"{domain}/{label}"], check=True, timeout=5, capture_output=True) - return domain - except launchctl_errors: - pass - - # Not loaded anywhere: Aqua → gui/; anything else (Background, loginwindow) → user/, - # the pre-probing default and the recommended domain on macOS 26+. - try: - result = subprocess.run(["launchctl", "managername"], timeout=5, **_CAPTURE_TEXT) - if "Aqua" in (result.stdout or ""): - return gui_domain - except launchctl_errors: - pass - return user_domain - - -def _launchd_domain() -> str: - """Domain managing the current profile's gateway; cached per process so start/stop/restart agree. - - See #40831, #23387. - """ - global _resolved_launchd_domain - if _resolved_launchd_domain is None: - _resolved_launchd_domain = _probe_launchd_domain_for_label(get_launchd_label()) - return _resolved_launchd_domain - - -# 125 ("Domain does not support specified action") and 3/113 ("Could not find service") all mean -# the job isn't loaded in the target domain: re-bootstrap the plist and retry. -_LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125}) - -# 5 (EIO) / persistent 125 mean either a stale still-registered label (recoverable: bootout + -# bootstrap, which `_launchctl_bootstrap()` tries first) or a domain that genuinely can't manage -# services (macOS 26+). Only when the retry ALSO fails do callers degrade to a detached process. -# launchctl returns 5 ("Input/output error") or a persistent 125 in two very different situations, so exit 5 -# is NOT on its own proof the domain is broken: 1. See #42914. 2. Here launchd cannot supervise the gateway -# at all and we degrade to a detached background process (the `nohup hermes gateway run` workaround). See -# #23387. -_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125}) - - -def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool: - """True when launchctl failed because the job isn't loaded (retry bootstrap).""" - return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES - - -def _launchctl_domain_unsupported(returncode: int) -> bool: - """True when launchctl can't manage the domain even after a fresh bootstrap (macOS 26+) — degrade to detached.""" - return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES - - -# EIO from `launchctl bootstrap` = label *already* registered (stale load); recoverable, not an unmanageable domain. -_LAUNCHCTL_BOOTSTRAP_EIO = 5 - - -def _launchctl_bootstrap(domain: str, plist_path, label: str, *, timeout: int = 30) -> None: - """Bootstrap a launchd job, recovering from a stale still-registered label (EIO 5). Without the - bootout + retry that case is misread as an unmanageable domain and degrades to detached, silently - losing auto-start and crash-restart.""" - bootstrap = ["launchctl", "bootstrap", domain, str(plist_path)] - try: - subprocess.run(bootstrap, check=True, timeout=timeout) - except subprocess.CalledProcessError as exc: - if exc.returncode != _LAUNCHCTL_BOOTSTRAP_EIO: - raise - # Stale registration — bootout the leftover label and bootstrap once more. - # Captured: the bootout is best-effort (a drained job may already be - # unloaded), so its expected 3/113/125 stderr must not leak to the terminal. - subprocess.run( - ["launchctl", "bootout", f"{domain}/{label}"], - check=False, timeout=timeout, **_CAPTURE_TEXT) - subprocess.run(bootstrap, check=True, timeout=timeout) - - -def _launchd_reload_log_path() -> Path: - """Path the launchd reload watchdog tails for persistent-orphan detection.""" - return get_hermes_home() / "logs" / "launchd-reload.log" - - -def _append_launchd_reload_log(message: str) -> None: - """Append a timestamped line to the launchd reload log (best-effort).""" - path = _launchd_reload_log_path() - try: - path.parent.mkdir(parents=True, exist_ok=True) - from datetime import datetime as _dt - stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z") - with path.open("a", encoding="utf-8") as fh: - fh.write(f"[{stamp}] {message}\n") - except OSError: - pass - - -def _launchd_reload_budget() -> float: - """Bootstrap retry window for a plist reload: the failure happens while the old gateway is still - draining (default 180s), so size it to the drain timeout with a 30s floor.""" - return max(30.0, _get_restart_drain_timeout()) - - -def _launchctl_supervised_pid(label: str) -> int | None: - """PID launchd currently runs for ``label``, or None when it runs none. ``launchctl list`` exits 0 for - a mere registered definition (``state = not running`` on macOS 26+), so a PID — not the exit code — is - the answer. Domain-agnostic on purpose: ``launchctl print`` domain probes fail on macOS-26 per-user - domains, which is why the invoking profile verifies through this and not ``_launchd_print_service_pid``.""" - try: - result = subprocess.run(["launchctl", "list", label], check=False, timeout=10, **_CAPTURE_TEXT) - except (subprocess.TimeoutExpired, OSError): - return None - if result.returncode != 0: - return None - return _parse_launchd_pid_from_list_output(result.stdout) - - -def _launchctl_label_supervising_process(label: str) -> bool: - """True when launchd knows ``label`` AND runs a process for it.""" - return _launchctl_supervised_pid(label) is not None - - -def _retry_launchctl_bootstrap_until_registered( - domain: str, plist_path, label: str, *, deadline: float -) -> bool: - """Retry ``_launchctl_bootstrap`` until the label supervises a process or ``deadline`` passes. Under - load bootstrap can fail even after bootout, during a drain (default 180s) — ~10s is too short.""" - attempt = 0 - while True: - attempt += 1 - try: - _launchctl_bootstrap(domain, plist_path, label, timeout=30) - if _launchctl_label_supervising_process(label): - return True - outcome = f"exited 0 but {domain}/{label} has no supervised process (launchctl list)" - except subprocess.CalledProcessError as exc: - outcome = f"failed (rc={exc.returncode}) for {domain}/{label}" - except subprocess.TimeoutExpired: - outcome = f"timed out for {domain}/{label}" - _append_launchd_reload_log(f"bootstrap attempt {attempt} {outcome} — retrying") - if time.monotonic() >= deadline: - return False - time.sleep(2) - - -# launchd-unsupported marker: written when the domain can't be managed (exit 5/125, macOS 26+) so -# `launchd_status()` can explain missing supervision; cleared on successful bootstrap/kickstart. -def _launchd_unsupported_marker_path() -> Path: - return get_hermes_home() / ".gateway-launchd-unsupported" - - -def _write_launchd_unsupported_marker() -> None: - """Persist that launchd cannot supervise the gateway on this host.""" - from datetime import datetime, timezone - payload = { - "written_at": datetime.now(timezone.utc).isoformat(), - "reason": "launchd domain unsupported (exit 5/125)", - } - with contextlib.suppress(OSError): - _launchd_unsupported_marker_path().write_text(json.dumps(payload), encoding="utf-8") - - -def _clear_launchd_unsupported_marker() -> None: - """Clear the unsupported marker when launchd bootstrap succeeds.""" - with contextlib.suppress(OSError): - _launchd_unsupported_marker_path().unlink(missing_ok=True) - - -def _launchd_unsupported_marker_exists() -> bool: - return _launchd_unsupported_marker_path().exists() - - -def _gateway_run_command() -> list[str]: - """Build ``python -m hermes_cli.main [--profile X] gateway run --replace``, honoring the active profile.""" - return [get_python_path(), "-m", "hermes_cli.main", *_profile_arg().split(), "gateway", "run", "--replace"] - - -def _timestamped_stderr_gateway_command(error_log: Path, *, external_supervisor: bool = False) -> list[str]: - """Wrap gateway run so raw stderr lines are timestamped before file write. ``external_supervisor`` - (launchd ProgramArguments only) adds ``--external-supervisor`` so ``hermes update`` hands back to - launchd, and drops ``--replace``: KeepAlive respawns would re-arm takeover, so two profiles sharing - a token would kill each other forever. - - ``external_supervisor=True`` is for launchd ProgramArguments only: the inner ``gateway run`` must carry - ``--external-supervisor`` so ``hermes update`` sees the flag on the live grandchild argv and hands the - process back to launchd instead of starting a detached watcher (#86893 / #87005). The detached nohup - fallback stays unmarked. - Supervised starts also drop ``--replace`` (issue #79048): a launchd service is respawned by KeepAlive, - so takeover authority would be re-armed on every respawn — two profiles legitimately sharing one - platform token would each terminate the sibling, and launchd would revive the victim forever. Bounded - replacement is the lifecycle commands' job (``launchctl kickstart -k``, drain in ``launchd_restart()``, - bootout+bootstrap in install/refresh), which run before supervision resumes. Mirrors - ``generate_systemd_unit``, whose ExecStart also runs ``gateway run`` without ``--replace``. - """ - inner = _gateway_run_command() - if external_supervisor: - inner = [part for part in inner if part != "--replace"] - if "--external-supervisor" not in inner: - inner.append("--external-supervisor") - return [get_python_path(), "-m", "hermes_cli.stderr_timestamp", "--error-log", str(error_log), "--", *inner] - - -def _spawn_detached_gateway() -> bool: - """Launch the gateway detached (launchd fallback for macOS 26+). CLI-managed nohup equivalent: - stdout → gateway.log, timestamped stderr → gateway.error.log, PID via gateway.pid so stop/status work. - - Used when launchctl can no longer bootstrap/kickstart the gateway on macOS 26+ (issue #23387). Mirrors - the `nohup hermes gateway run --replace` workaround but keeps it CLI-managed: stdout goes to - gateway.log, stderr is timestamped into gateway.error.log, and the PID is tracked via the gateway.pid - file that `run_gateway` writes, so stop/status/restart keep working. - """ - from hermes_cli._subprocess_compat import windows_detach_popen_kwargs - log_dir = get_hermes_home() / "logs" - log_dir.mkdir(parents=True, exist_ok=True) - try: - with open(log_dir / "gateway.log", "ab") as out: - subprocess.Popen( - _timestamped_stderr_gateway_command(log_dir / "gateway.error.log"), - stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.DEVNULL, - **windows_detach_popen_kwargs(), - ) - except OSError: - return False - return True - - -def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool: - """Start the gateway detached when launchd can't manage it; on failure print the manual workaround - and (by default) exit 1.""" - from hermes_constants import display_hermes_home as _dhh - _write_launchd_unsupported_marker() - print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).") - if _spawn_detached_gateway(): - print("✓ Started gateway as a background process instead") - print(" It will NOT auto-start at login or auto-restart on crash.") - print(f" Logs: {_dhh()}/logs/gateway.log") - print(" Stop it with: hermes gateway stop") - return True - print_error("Failed to start the gateway as a background process.") - print(f" Try manually: nohup hermes gateway run --replace > {_dhh()}/logs/gateway.log 2>&1 &") - if exit_on_failure: - sys.exit(1) - return False - - -def _launchd_degrade_or_raise(exc: subprocess.CalledProcessError, what: str) -> None: - """Shared launchctl failure policy: domain unmanageable (5/125) → detached fallback; else re-raise. - - A 5/125 exit is evidence about the *domain* only when launchd is not already supervising this job. - EIO (5) is ``launchctl bootstrap``'s answer for a label that is already loaded, so the ordinary - "reinstall/restart over the live gateway" case lands here with the service up and supervised. - Degrading there is not a graceful fallback: it writes the permanent launchd-unsupported marker and - starts a detached gateway *beside* the supervised one, and the marker makes - :func:`wait_for_launchd_gateway_supervision` answer True unconditionally — so no later - install/update can tell that nothing ties the gateway to launchd any more. A live supervised PID is - direct evidence this macOS does manage the job, so surface the failure instead of branding the host. - """ - if not _launchctl_domain_unsupported(exc.returncode): - raise exc - label = get_launchd_label() - if _launchctl_label_supervising_process(label): - print(f"⚠ {what} failed (exit {exc.returncode}), but launchd still supervises {label}") - print(" Not switching to the detached fallback — this host manages the job.") - print(" Apply the definition with: hermes gateway stop && hermes gateway install --force") - raise exc - _launchd_fallback_to_detached(f"{what} exit {exc.returncode}") - - -def generate_launchd_plist() -> str: - # Stable cwd anchor — never the volatile source checkout (same rot risk as systemd's WorkingDirectory). - working_dir = _stable_service_working_dir() - hermes_home = str(get_hermes_home().resolve()) - log_dir = get_hermes_home() / "logs" - log_dir.mkdir(parents=True, exist_ok=True) - label = get_launchd_label() - venv_dir = _service_venv_dir() - # launchd's default PATH misses Homebrew, nvm, cargo…; prepend venv/bin + node dirs (as in the - # systemd unit) so node stays resolvable even if the shell PATH changes, then the shell PATH. - priority_dirs = _build_service_path_dirs() - _append_node_dir_for_service(priority_dirs) - sane_path = ":".join(dict.fromkeys(priority_dirs + [p for p in os.environ.get("PATH", "").split(":") if p])) - - # ProgramArguments (incl. --profile); the stderr wrapper keeps launchd restart semantics while timestamping stderr. - prog_args_xml = "\n ".join( - f"{part}" - for part in _timestamped_stderr_gateway_command(log_dir / "gateway.error.log", external_supervisor=True) - ) - - # Persist the configured RLIMIT_NOFILE floor: launchd defaults to soft 256, and every plist - # rewrite would otherwise strip a manual limit and reintroduce EMFILE crashes. - nofile_block = "" - try: - from hermes_cli.resource_limits import configured_nofile_soft_limit - nofile_target = configured_nofile_soft_limit() - except Exception: - nofile_target = None - if nofile_target: - nofile_block = f""" - SoftResourceLimits - - NumberOfFiles - {nofile_target} - -""" - - return f""" - - - - Label - {label} - - ProgramArguments - - {prog_args_xml} - - - WorkingDirectory - {working_dir} - - EnvironmentVariables - - PATH - {sane_path} - VIRTUAL_ENV - {venv_dir} - HERMES_HOME - {hermes_home} - HERMES_SUPERVISED_CHILD - 1 - - - LimitLoadToSessionType - - Aqua - Background - - - RunAtLoad - - - KeepAlive - - SuccessfulExit - - - - - ThrottleInterval - 30 - - ExitTimeOut - 60 -{nofile_block} - StandardOutPath - {log_dir}/gateway.log - - StandardErrorPath - {log_dir}/gateway.error.log - - -""" - - -def launchd_plist_is_current() -> bool: - """Check if the installed launchd plist matches the currently generated one.""" - plist_path = get_launchd_plist_path() - if not plist_path.exists(): - return False - installed = plist_path.read_text(encoding="utf-8") - norm = _normalize_launchd_plist_for_comparison - return norm(installed) == norm(generate_launchd_plist()) - - -def _spawn_deferred_launchd_reload( - *, domain: str, label: str, target: str, plist_path: Path, gateway_pid: int -) -> bool: - """Hand the bootout/bootstrap cycle to a transient ``launchctl submit`` job; True if spawned. The - helper waits for the OLD gateway to exit (bootstrap during drain fails EIO), then retries bootstrap - until ``launchctl list`` shows a positive PID or the drain budget elapses.""" - reload_log_path = _launchd_reload_log_path() - with contextlib.suppress(OSError): - reload_log_path.parent.mkdir(parents=True, exist_ok=True) - - # Durable pre-bootout marker: distinguishes "helper never started" from "helper ran but failed". - _append_launchd_reload_log(f"Launchd reload helper started for {target}") - - _reload_budget = int(_launchd_reload_budget()) - q_target, q_label, q_log = shlex.quote(target), shlex.quote(label), shlex.quote(str(reload_log_path)) - stamp = "$(date '+%Y-%m-%d %H:%M:%S %z')" - # Require a POSITIVE PID: `launchctl list` also exits 0 for a registered-but-not-running - # definition, and a crashed job reports `"PID" = -1` (mirrors _parse_launchd_pid_from_list_output). - listed = f"launchctl list {q_label} 2>/dev/null | grep -qE '\\\"PID\\\" = [0-9]+;'" - # Unique per reload so concurrent/repeated reloads never collide. - submit_label = f"{label}.reload.{os.getpid()}.{int(time.time())}" - reload_script = ( - f"sleep 2; " - f"launchctl bootout {q_target} 2>/dev/null; " - # Wait for the OLD gateway to exit: bootout only SIGTERMs and every bootstrap during the drain fails EIO. - f"_wait_deadline=$(($(date +%s) + {_reload_budget})); " - f"while kill -0 {gateway_pid} 2>/dev/null; do if [ $(date +%s) -ge $_wait_deadline ]; then " - f" echo \"[{stamp}] old gateway pid {gateway_pid} still alive after {_reload_budget}s drain wait — bootstrapping anyway\" >> {q_log}; " - f" break; fi; sleep 1; done; " - # Let launchd finish unregistering the label after the process exits. - f"sleep 1; _deadline=$(($(date +%s) + {_reload_budget})); while :; do " - f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; " - f" if {listed}; then break; fi; " - f" echo \"[{stamp}] bootstrap not yet registered for {q_target} — retrying\" >> {q_log}; " - f" if [ $(date +%s) -ge $_deadline ]; then break; fi; sleep 2; done; " - f"if ! {listed}; then " - f" echo \"[{stamp}] FAILED launchd reload for {q_target} — service NOT registered after {_reload_budget}s of retries\" >> {q_log}; " - f"fi; " - # Submitted jobs stay registered after the script exits; removing our own label ends the one-shot job. - f"launchctl remove {shlex.quote(submit_label)} 2>/dev/null" - ) - try: - # `launchctl submit` rather than setsid: setsid does NOT leave the launchd coalition that bootout kills. - # Spawn the reload helper via `launchctl submit` (a transient launchd one-shot job) instead of - # `start_new_session=True`. `start_new_session=True` only calls setsid(2), which creates a new POSIX - # session but does NOT move the child outside the launchd job's process coalition. When `launchctl - # bootout` fires on the gateway label, launchd terminates ALL processes in that coalition — - # including a setsid-detached child (#69098). `launchctl submit` creates a wholly independent - # transient launchd job that launchd manages separately from the gateway, so bootout of the gateway - # job cannot reach the helper. - subprocess.Popen( - [ - "launchctl", "submit", "-l", submit_label, "-o", str(reload_log_path), "-e", str(reload_log_path), - "--", "/bin/bash", "-c", reload_script, - ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except Exception as e: - # Fall through to in-process bootout/bootstrap: risky in the coalition, but better than a never-reloaded plist. - logger.warning("Deferred launchd reload could not be spawned: %s", e) - _append_launchd_reload_log( - f"FAILED to spawn launchd reload helper for {target}: {e} — falling back to in-process bootout/bootstrap" - ) - return False - return True - - -def refresh_launchd_plist_if_needed() -> bool: - """Rewrite the installed plist when the generated one differs, then bootout/bootstrap so launchd - re-reads it immediately.""" - plist_path = get_launchd_plist_path() - if not plist_path.exists() or launchd_plist_is_current(): - return False - - new_plist = generate_launchd_plist() - if _refuse_temp_home_service_write(new_plist, "launchd plist"): - return False - - plist_path.write_text(new_plist, encoding="utf-8") - label = get_launchd_label() - domain = _launchd_domain() - target = f"{domain}/{label}" - - # Inside the gateway's launchd process tree (agent self-update) a direct bootout kills THIS CLI - # before bootstrap runs, leaving the job unloaded with no KeepAlive. - try: - from gateway.status import get_running_pid - gateway_pid = get_running_pid() - except Exception: - gateway_pid = None - - # POSIX ancestry is NOT a reliable "bootout will kill us" test (coalition membership survives - # reparenting), so always prefer the detached helper; in-process is only the spawn-failure fallback. - if ( - gateway_pid is not None - and hasattr(os, "setsid") # POSIX-only; launchd is macOS so always true here - ) and _spawn_deferred_launchd_reload( - domain=domain, label=label, target=target, plist_path=plist_path, gateway_pid=gateway_pid - ): - print( - "↻ Updated gateway launchd service definition; reload deferred to " - "a transient launchd job (survives the bootout of this process)" - ) - return True - - # Bootout/bootstrap so launchd reads the new definition; bootstrap can fail silently under load - # during a drain, and KeepAlive can't revive an unregistered job. - # Captured: best-effort (the job may already be unloaded), keep expected noise off the terminal. - subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_CAPTURE_TEXT) - _reload_budget = _launchd_reload_budget() - # Wait out the old gateway's drain first so the budget isn't burned on guaranteed EIO ("already loaded"). - if gateway_pid is not None and not _wait_for_pid_exit(gateway_pid, _reload_budget): - _append_launchd_reload_log( - f"old gateway pid {gateway_pid} still alive after " - f"{int(_reload_budget)}s drain wait — bootstrapping {target} anyway" - ) - _deadline = time.monotonic() + _reload_budget - if not _retry_launchctl_bootstrap_until_registered(domain, plist_path, label, deadline=_deadline): - _append_launchd_reload_log( - f"FAILED launchd reload of {target} — service NOT registered after " - f"retrying for {int(_reload_budget)}s (in-process fallback path)" - ) - logger.error( - "launchd reload of %s failed — service not registered after %ds of retries; see %s", - target, int(_reload_budget), _launchd_reload_log_path(), - ) - return False - print("↻ Updated gateway launchd service definition to match the current Hermes install") - return True - - -def launchd_install(force: bool = False): - plist_path = get_launchd_plist_path() - - if plist_path.exists() and not force: - if not launchd_plist_is_current(): - print(f"↻ Repairing outdated launchd service at: {plist_path}") - if refresh_launchd_plist_if_needed(): - print("✓ Service definition updated") - else: - # The plist was rewritten but launchd never registered it (or the write was refused): - # a success line here would hide an unloaded service with no KeepAlive. - from hermes_constants import display_hermes_home - print( - "⚠ Service definition could not be reloaded with launchd. " - "Run 'hermes gateway install --force' or check " - f"{display_hermes_home()}/logs/launchd-reload.log for details." - ) - return - print(f"Service already installed at: {plist_path}") - print("Use --force to reinstall") - return - - plist_path.parent.mkdir(parents=True, exist_ok=True) - new_plist = generate_launchd_plist() - if _refuse_temp_home_service_write(new_plist, "launchd plist"): - return - print(f"Installing launchd service to: {plist_path}") - plist_path.write_text(new_plist, encoding="utf-8") - - try: - _launchctl_bootstrap(_launchd_domain(), plist_path, get_launchd_label(), timeout=30) - except subprocess.CalledProcessError as e: - _launchd_degrade_or_raise(e, "launchctl bootstrap") - return - - print() - print("✓ Service installed and loaded!") - _clear_launchd_unsupported_marker() - print() - print("Next steps:") - print(" hermes gateway status # Check status") - from hermes_constants import display_hermes_home as _dhh - print(f" tail -f {_dhh()}/logs/gateway.log # View logs") - - -def launchd_uninstall(): - plist_path = get_launchd_plist_path() - # Captured: uninstalling an already-unloaded job is fine — don't print Boot-out failed: 3. - subprocess.run( - ["launchctl", "bootout", f"{_launchd_domain()}/{get_launchd_label()}"], - check=False, timeout=90, **_CAPTURE_TEXT) - if plist_path.exists(): - plist_path.unlink() - print(f"✓ Removed {plist_path}") - print("✓ Service uninstalled") - - -def launchd_start(): - plist_path = get_launchd_plist_path() - label = get_launchd_label() - - # Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade) - if not plist_path.exists(): - new_plist = generate_launchd_plist() - if _refuse_temp_home_service_write(new_plist, "launchd plist"): - sys.exit(1) - print("↻ launchd plist missing; regenerating service definition") - plist_path.parent.mkdir(parents=True, exist_ok=True) - plist_path.write_text(new_plist, encoding="utf-8") - if _launchd_bootstrap_and_kickstart(plist_path, label): - _launchd_ok("✓ Service started") - return - - refresh_launchd_plist_if_needed() - try: - _launchctl_kickstart_current(label) - except subprocess.CalledProcessError as e: - if not _launchd_error_indicates_unloaded(e): - raise - # Job not loaded in this domain — re-bootstrap the plist and retry. - print("↻ launchd job was unloaded; reloading service definition") - if not _launchd_bootstrap_and_kickstart(plist_path, label): - return - _launchd_ok("✓ Service started") - - -def _launchctl_kickstart_current(label: str) -> None: - subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30) - - -def _launchd_bootstrap_and_kickstart(plist_path: Path, label: str) -> bool: - """Bootstrap then kickstart; False after degrading to detached (domain unsupported). Other errors propagate.""" - try: - _launchctl_bootstrap(_launchd_domain(), plist_path, label, timeout=30) - _launchctl_kickstart_current(label) - except subprocess.CalledProcessError as e: - _launchd_degrade_or_raise(e, "launchctl") - return False - return True - - -def _launchd_ok(message: str) -> None: - """Print a launchd success line and clear the unsupported marker (an OS fix recovers automatically).""" - print(message) - _clear_launchd_unsupported_marker() - - -def launchd_stop(): - target = f"{_launchd_domain()}/{get_launchd_label()}" - _mark_planned_stop() - # bootout unloads the definition so KeepAlive doesn't respawn; `hermes gateway start` re-bootstraps. - try: - # Captured: an already-unloaded job (3/113/125) is handled below, so launchctl's own - # "Boot-out failed: 3" must not print around the ✓ line; e.stderr stays on the raised error. - subprocess.run(["launchctl", "bootout", target], check=True, timeout=90, **_CAPTURE_TEXT) - except subprocess.CalledProcessError as e: - # Job already unloaded (3/113/125), or the domain can't be managed at all (5/125, macOS 26+ - # detached-fallback process, issue #23387) — in both cases just fall through to the PID-based kill - # below. - if not (_launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(e.returncode)): - raise - _wait_for_gateway_exit(timeout=10.0, force_after=5.0) - print("✓ Service stopped") - def _wait_for_gateway_exit(timeout: float = 10.0, force_after: float | None = 5.0) -> bool: """Wait up to ``timeout`` s for the gateway (by gateway.pid, not launchd labels, so multiple @@ -4178,189 +3570,6 @@ def _wait_for_api_server_port_free(*, timeout: float = 10.0) -> bool: return freed -def _launchd_kickstart(label: str, domain: str) -> None: - """``launchctl kickstart -k domain/label``; raises so callers own per-label failure accounting.""" - subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True, timeout=90, **_CAPTURE_TEXT) - - -def _wait_for_launchd_service_pid( - label: str, old_pid: int | None, timeout: float = 10.0, *, domain: str -) -> bool: - """Poll ``domain/label`` (0.5s) until it runs on a fresh PID or ``timeout`` passes — KeepAlive respawn - isn't instantaneous. launchctl ``TimeoutExpired`` propagates; callers own failure accounting.""" - deadline = time.monotonic() + max(timeout, 0.5) - while True: - _loaded, pid = _launchd_print_service_pid(domain, label) - if pid is not None and pid > 0 and pid != old_pid: - return True - if time.monotonic() >= deadline: - return False - time.sleep(0.5) - - -def launchd_restart(): - label = get_launchd_label() - domain = _launchd_domain() - target = f"{domain}/{label}" - from gateway.status import get_running_pid - try: - pid = get_running_pid() - if pid is not None and _request_gateway_self_restart(pid): - _launchd_ok("✓ Service restart requested") - return - if pid is not None and probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED: - # Event loop provably dead: it can't process a graceful shutdown, so a full drain wait - # only stalls the restart (and `hermes update`). Bounded SIGTERM → SIGKILL, ~10s. - print(f"⚠ Gateway PID {pid} event loop is unresponsive — " "skipping drain and forcing a bounded stop...") - _escalate_wedged_gateway(pid) - pid = None - if pid is not None: - # Graceful in-band restart via SIGUSR1 (mirrors systemd); the budget covers both the idle wait - # and the drain. A bare SIGTERM would lose the resume_pending handoff. Announce BEFORE waiting: - # surfaces with no other feedback (desktop updater) read silence as "update stuck". - wait_budget = _get_restart_exit_wait_budget() - print(f"→ Stopping gateway (PID {pid}) — draining in-flight runs (up to {wait_budget:.0f}s)...") - from hermes_cli.update_cmd_drain_report import drain_progress_reporter - if _graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)): - # KeepAlive revives a planned exit, so do NOT kickstart (-k would kill the replacement) — - # but a clean exit doesn't prove supervision, so verify a replacement PID appears first. - if _wait_for_launchd_service_pid(label, pid, timeout=15.0, domain=domain): - _launchd_ok("✓ Service restart requested") - return - print("⚠ launchd did not revive the gateway after its graceful exit — forcing restart") - else: - print(f"⚠ Gateway drain timed out after {wait_budget:.0f}s — forcing launchd restart") - # Captured: an unloaded job (3/113/125) is the expected case below, which - # prints its own ↻ line — and e.stderr feeds the update_cmd failure diagnostic. - _wait_for_api_server_port_free() - subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90, **_CAPTURE_TEXT) - _launchd_ok("✓ Service restarted") - except subprocess.CalledProcessError as e: - if not _launchd_error_indicates_unloaded(e): - _launchd_degrade_or_raise(e, "launchctl kickstart") - return - # Job not loaded — bootstrap and start fresh - print("↻ launchd job was unloaded; reloading") - try: - # After a drain the job is usually still registered (bootstrap would hit EIO): boot it out first. - # Captured: best-effort (the job may already be unloaded after the drain), - # so an expected Boot-out failed: 3 must not leak past the ↻ line below. - subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_CAPTURE_TEXT) - plist_path = str(get_launchd_plist_path()) - subprocess.run(["launchctl", "bootstrap", _launchd_domain(), plist_path], check=True, timeout=30) - subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30) - except subprocess.CalledProcessError as e2: - _launchd_degrade_or_raise(e2, "launchctl") - return - _launchd_ok("✓ Service restarted") - - -# KeepAlive relaunches at most ~once per 10s, so a self-restart leaves the label pid-less that long. -LAUNCHD_SUPERVISION_VERIFY_TIMEOUT = 20.0 - - -def wait_for_launchd_gateway_supervision( - *, - timeout: float = LAUNCHD_SUPERVISION_VERIFY_TIMEOUT, - label: str | None = None, - poll_interval: float = 0.5, - old_pid: int | None = None, -) -> bool: - """Poll launchd until it supervises a live gateway; True at once if the detached fallback is active. - ``launchd_restart`` returns once the restart is *requested* (asynchronous), so it can't see a helper - dying before bootstrap or a ``launchctl bootstrap`` that exits 0 without registering. - - The ``_request_gateway_self_restart`` branch hands the work to the running gateway and returns - immediately, and a plist reload is handed to a detached helper. Both are asynchronous, so a caller that - reads "returned without raising" as "the service is up" cannot see a helper that dies before its first - bootstrap (#88848) — nor a ``launchctl bootstrap`` that exits 0 without registering, which the reporter - measured on macOS 26.6.1. - Judge the outcome the way #80491 taught the helper to judge it: by a live supervised pid, never by an - exit code. :func:`_launchctl_supervised_pid` is already that probe, so this only adds the wait. - - ``old_pid`` is the pid launchd ran for the label *before* the restart: a restart that leaves the same - process running is not a restart, so passing it holds the invoking profile to the same fresh-pid - contract :func:`_wait_for_launchd_service_pid` enforces for sibling labels. With ``old_pid=None`` - (no pre-restart pid was observable) any supervised pid counts, as before. - """ - if _launchd_unsupported_marker_exists(): - return True - - label = label or get_launchd_label() - deadline = time.monotonic() + max(timeout, 0.0) - while True: - pid = _launchctl_supervised_pid(label) - if pid is not None and pid != old_pid: - return True - if time.monotonic() >= deadline: - return False - time.sleep(max(poll_interval, 0.01)) - - -def launchd_status(deep: bool = False): - plist_path = get_launchd_plist_path() - label = get_launchd_label() - try: - result = subprocess.run(["launchctl", "list", label], timeout=10, **_CAPTURE_TEXT) - service_listed = result.returncode == 0 - list_output = result.stdout - except subprocess.TimeoutExpired: - service_listed = False - list_output = "" - - # `launchctl list` exits 0 for any registered definition (even `state = not running`); only a PID proves a process. - launchd_pid = _parse_launchd_pid_from_list_output(list_output) if service_listed else None - - # Hermes PID may be a detached fallback process; when launchd IS supervising both PIDs match — don't double-count. - from gateway.status import get_running_pid - fallback_pid = get_running_pid(cleanup_stale=False) - if launchd_pid is not None and fallback_pid == launchd_pid: - fallback_pid = None - - # Marker from a 5/125 bootstrap/kickstart failure explains *why* launchd can't supervise. - launchd_unsupported = _launchd_unsupported_marker_exists() - - print(f"Launchd plist: {plist_path}") - if launchd_plist_is_current(): - print("✓ Service definition matches the current Hermes install") - else: - print("⚠ Service definition is stale relative to the current Hermes install") - print(" Run: hermes gateway start") - - if not service_listed: - print("✗ Gateway service is not loaded") - print(" Service definition exists locally but launchd has not loaded it.") - print(" Run: hermes gateway start") - if fallback_pid: - print(f" Note: a detached gateway process is running (PID {fallback_pid})") - elif launchd_pid is not None: - print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})") - print(" Auto-start at login and auto-restart on crash are available.") - if launchd_unsupported: - print(" (launchd domain was previously unavailable but is now working)") - elif launchd_unsupported: - print("⚠ Gateway service is registered but launchd is not supervising it") - print(" launchd cannot manage the gateway on this macOS version.") - if fallback_pid: - print(f"✓ Detached fallback process is running (PID {fallback_pid})") - print(" Cron jobs will fire. Stop with: hermes gateway stop") - else: - print("✗ No fallback process is running") - print(" Run: hermes gateway start") - print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.") - else: - print("✓ Gateway service is registered with launchd") - print(list_output) - if fallback_pid: - print(f" Detached gateway process is running (PID {fallback_pid})") - - if deep: - log_file = get_hermes_home() / "logs" / "gateway.log" - if log_file.exists(): - print() - print("Recent logs:") - subprocess.run(["tail", "-20", str(log_file)], timeout=10) - # ============================================================================= # Gateway Runner diff --git a/hermes_cli/gateway_launchd.py b/hermes_cli/gateway_launchd.py new file mode 100644 index 0000000000..209fc3b566 --- /dev/null +++ b/hermes_cli/gateway_launchd.py @@ -0,0 +1,860 @@ +"""Gateway launchd (macOS LaunchAgent) backend: plist generation/refresh, bootstrap, start/stop/restart/status. + +Extracted from ``hermes_cli/gateway.py``. Bodies read facade helpers through ``_gw()`` (late +binding on ``hermes_cli.gateway``) so the seams tests and callers patch on the facade keep +intercepting the moved code. +""" +from __future__ import annotations + +from pathlib import Path +import contextlib +import json +import os +import shlex +import subprocess +import sys +import time + + +def _gw(): + from hermes_cli import gateway # late: the facade imports this module + return gateway + + +def get_launchd_label() -> str: + """Return the launchd service label, scoped per profile.""" + suffix = _gw()._profile_suffix() + return f"ai.hermes.gateway-{suffix}" if suffix else "ai.hermes.gateway" + + +def _probe_launchd_domain_for_label(label: str) -> str: + """Launchd domain managing ``label`` (uncached): ``gui/`` (Aqua), then ``user/`` + (Background/SSH), else the ``launchctl managername`` heuristic. Sibling profiles may live in + different domains, so never reuse the cached ``_launchd_domain()`` for another label.""" + uid = os.getuid() # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows + gui_domain, user_domain = f"gui/{uid}", f"user/{uid}" + + launchctl_errors = (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) + for domain in (gui_domain, user_domain): + try: + subprocess.run(["launchctl", "print", f"{domain}/{label}"], check=True, timeout=5, capture_output=True) + return domain + except launchctl_errors: + pass + + # Not loaded anywhere: Aqua → gui/; anything else (Background, loginwindow) → user/, + # the pre-probing default and the recommended domain on macOS 26+. + try: + result = subprocess.run(["launchctl", "managername"], timeout=5, **_gw()._CAPTURE_TEXT) + if "Aqua" in (result.stdout or ""): + return gui_domain + except launchctl_errors: + pass + return user_domain + + +def _launchd_domain() -> str: + """Domain managing the current profile's gateway; cached per process so start/stop/restart agree. + + See #40831, #23387. + """ + # The cache lives on the facade: tests and callers reset ``hermes_cli.gateway._resolved_launchd_domain``. + gw = _gw() + if gw._resolved_launchd_domain is None: + gw._resolved_launchd_domain = _probe_launchd_domain_for_label(gw.get_launchd_label()) + return gw._resolved_launchd_domain + + +# 125 ("Domain does not support specified action") and 3/113 ("Could not find service") all mean +# the job isn't loaded in the target domain: re-bootstrap the plist and retry. +_LAUNCHD_JOB_UNLOADED_EXIT_CODES = frozenset({3, 113, 125}) + + +# 5 (EIO) / persistent 125 mean either a stale still-registered label (recoverable: bootout + +# bootstrap, which `_launchctl_bootstrap()` tries first) or a domain that genuinely can't manage +# services (macOS 26+). Only when the retry ALSO fails do callers degrade to a detached process. +# launchctl returns 5 ("Input/output error") or a persistent 125 in two very different situations, so exit 5 +# is NOT on its own proof the domain is broken: 1. See #42914. 2. Here launchd cannot supervise the gateway +# at all and we degrade to a detached background process (the `nohup hermes gateway run` workaround). See +# #23387. +_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125}) + + +def _launchd_error_indicates_unloaded(exc: subprocess.CalledProcessError) -> bool: + """True when launchctl failed because the job isn't loaded (retry bootstrap).""" + return exc.returncode in _LAUNCHD_JOB_UNLOADED_EXIT_CODES + + +def _launchctl_domain_unsupported(returncode: int) -> bool: + """True when launchctl can't manage the domain even after a fresh bootstrap (macOS 26+) — degrade to detached.""" + return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES + + +# EIO from `launchctl bootstrap` = label *already* registered (stale load); recoverable, not an unmanageable domain. +_LAUNCHCTL_BOOTSTRAP_EIO = 5 + + +def _launchctl_bootstrap(domain: str, plist_path, label: str, *, timeout: int = 30) -> None: + """Bootstrap a launchd job, recovering from a stale still-registered label (EIO 5). Without the + bootout + retry that case is misread as an unmanageable domain and degrades to detached, silently + losing auto-start and crash-restart.""" + bootstrap = ["launchctl", "bootstrap", domain, str(plist_path)] + try: + subprocess.run(bootstrap, check=True, timeout=timeout) + except subprocess.CalledProcessError as exc: + if exc.returncode != _LAUNCHCTL_BOOTSTRAP_EIO: + raise + # Stale registration — bootout the leftover label and bootstrap once more. + # Captured: the bootout is best-effort (a drained job may already be + # unloaded), so its expected 3/113/125 stderr must not leak to the terminal. + subprocess.run( + ["launchctl", "bootout", f"{domain}/{label}"], + check=False, timeout=timeout, **_gw()._CAPTURE_TEXT) + subprocess.run(bootstrap, check=True, timeout=timeout) + + +def _launchd_reload_log_path() -> Path: + """Path the launchd reload watchdog tails for persistent-orphan detection.""" + return _gw().get_hermes_home() / "logs" / "launchd-reload.log" + + +def _append_launchd_reload_log(message: str) -> None: + """Append a timestamped line to the launchd reload log (best-effort).""" + path = _launchd_reload_log_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + from datetime import datetime as _dt + stamp = _dt.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z") + with path.open("a", encoding="utf-8") as fh: + fh.write(f"[{stamp}] {message}\n") + except OSError: + pass + + +def _launchd_reload_budget() -> float: + """Bootstrap retry window for a plist reload: the failure happens while the old gateway is still + draining (default 180s), so size it to the drain timeout with a 30s floor.""" + return max(30.0, _gw()._get_restart_drain_timeout()) + + +def _launchctl_supervised_pid(label: str) -> int | None: + """PID launchd currently runs for ``label``, or None when it runs none. ``launchctl list`` exits 0 for + a mere registered definition (``state = not running`` on macOS 26+), so a PID — not the exit code — is + the answer. Domain-agnostic on purpose: ``launchctl print`` domain probes fail on macOS-26 per-user + domains, which is why the invoking profile verifies through this and not ``_launchd_print_service_pid``.""" + try: + result = subprocess.run(["launchctl", "list", label], check=False, timeout=10, **_gw()._CAPTURE_TEXT) + except (subprocess.TimeoutExpired, OSError): + return None + if result.returncode != 0: + return None + return _gw()._parse_launchd_pid_from_list_output(result.stdout) + + +def _launchctl_label_supervising_process(label: str) -> bool: + """True when launchd knows ``label`` AND runs a process for it.""" + return _gw()._launchctl_supervised_pid(label) is not None + + +def _retry_launchctl_bootstrap_until_registered( + domain: str, plist_path, label: str, *, deadline: float +) -> bool: + """Retry ``_launchctl_bootstrap`` until the label supervises a process or ``deadline`` passes. Under + load bootstrap can fail even after bootout, during a drain (default 180s) — ~10s is too short.""" + attempt = 0 + while True: + attempt += 1 + try: + _gw()._launchctl_bootstrap(domain, plist_path, label, timeout=30) + if _gw()._launchctl_label_supervising_process(label): + return True + outcome = f"exited 0 but {domain}/{label} has no supervised process (launchctl list)" + except subprocess.CalledProcessError as exc: + outcome = f"failed (rc={exc.returncode}) for {domain}/{label}" + except subprocess.TimeoutExpired: + outcome = f"timed out for {domain}/{label}" + _gw()._append_launchd_reload_log(f"bootstrap attempt {attempt} {outcome} — retrying") + if time.monotonic() >= deadline: + return False + time.sleep(2) + + +# launchd-unsupported marker: written when the domain can't be managed (exit 5/125, macOS 26+) so +# `launchd_status()` can explain missing supervision; cleared on successful bootstrap/kickstart. +def _launchd_unsupported_marker_path() -> Path: + return _gw().get_hermes_home() / ".gateway-launchd-unsupported" + + +def _write_launchd_unsupported_marker() -> None: + """Persist that launchd cannot supervise the gateway on this host.""" + from datetime import datetime, timezone + payload = { + "written_at": datetime.now(timezone.utc).isoformat(), + "reason": "launchd domain unsupported (exit 5/125)", + } + with contextlib.suppress(OSError): + _launchd_unsupported_marker_path().write_text(json.dumps(payload), encoding="utf-8") + + +def _clear_launchd_unsupported_marker() -> None: + """Clear the unsupported marker when launchd bootstrap succeeds.""" + with contextlib.suppress(OSError): + _launchd_unsupported_marker_path().unlink(missing_ok=True) + + +def _launchd_unsupported_marker_exists() -> bool: + return _launchd_unsupported_marker_path().exists() + + +def _gateway_run_command() -> list[str]: + """Build ``python -m hermes_cli.main [--profile X] gateway run --replace``, honoring the active profile.""" + return [_gw().get_python_path(), "-m", "hermes_cli.main", *_gw()._profile_arg().split(), "gateway", "run", "--replace"] + + +def _timestamped_stderr_gateway_command(error_log: Path, *, external_supervisor: bool = False) -> list[str]: + """Wrap gateway run so raw stderr lines are timestamped before file write. ``external_supervisor`` + (launchd ProgramArguments only) adds ``--external-supervisor`` so ``hermes update`` hands back to + launchd, and drops ``--replace``: KeepAlive respawns would re-arm takeover, so two profiles sharing + a token would kill each other forever. + + ``external_supervisor=True`` is for launchd ProgramArguments only: the inner ``gateway run`` must carry + ``--external-supervisor`` so ``hermes update`` sees the flag on the live grandchild argv and hands the + process back to launchd instead of starting a detached watcher (#86893 / #87005). The detached nohup + fallback stays unmarked. + Supervised starts also drop ``--replace`` (issue #79048): a launchd service is respawned by KeepAlive, + so takeover authority would be re-armed on every respawn — two profiles legitimately sharing one + platform token would each terminate the sibling, and launchd would revive the victim forever. Bounded + replacement is the lifecycle commands' job (``launchctl kickstart -k``, drain in ``launchd_restart()``, + bootout+bootstrap in install/refresh), which run before supervision resumes. Mirrors + ``generate_systemd_unit``, whose ExecStart also runs ``gateway run`` without ``--replace``. + """ + inner = _gw()._gateway_run_command() + if external_supervisor: + inner = [part for part in inner if part != "--replace"] + if "--external-supervisor" not in inner: + inner.append("--external-supervisor") + return [_gw().get_python_path(), "-m", "hermes_cli.stderr_timestamp", "--error-log", str(error_log), "--", *inner] + + +def _spawn_detached_gateway() -> bool: + """Launch the gateway detached (launchd fallback for macOS 26+). CLI-managed nohup equivalent: + stdout → gateway.log, timestamped stderr → gateway.error.log, PID via gateway.pid so stop/status work. + + Used when launchctl can no longer bootstrap/kickstart the gateway on macOS 26+ (issue #23387). Mirrors + the `nohup hermes gateway run --replace` workaround but keeps it CLI-managed: stdout goes to + gateway.log, stderr is timestamped into gateway.error.log, and the PID is tracked via the gateway.pid + file that `run_gateway` writes, so stop/status/restart keep working. + """ + from hermes_cli._subprocess_compat import windows_detach_popen_kwargs + log_dir = _gw().get_hermes_home() / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + try: + with open(log_dir / "gateway.log", "ab") as out: + subprocess.Popen( + _timestamped_stderr_gateway_command(log_dir / "gateway.error.log"), + stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.DEVNULL, + **windows_detach_popen_kwargs(), + ) + except OSError: + return False + return True + + +def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool: + """Start the gateway detached when launchd can't manage it; on failure print the manual workaround + and (by default) exit 1.""" + from hermes_constants import display_hermes_home as _dhh + _gw()._write_launchd_unsupported_marker() + print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).") + if _gw()._spawn_detached_gateway(): + print("✓ Started gateway as a background process instead") + print(" It will NOT auto-start at login or auto-restart on crash.") + print(f" Logs: {_dhh()}/logs/gateway.log") + print(" Stop it with: hermes gateway stop") + return True + _gw().print_error("Failed to start the gateway as a background process.") + print(f" Try manually: nohup hermes gateway run --replace > {_dhh()}/logs/gateway.log 2>&1 &") + if exit_on_failure: + sys.exit(1) + return False + + +def _launchd_degrade_or_raise(exc: subprocess.CalledProcessError, what: str) -> None: + """Shared launchctl failure policy: domain unmanageable (5/125) → detached fallback; else re-raise. + + A 5/125 exit is evidence about the *domain* only when launchd is not already supervising this job. + EIO (5) is ``launchctl bootstrap``'s answer for a label that is already loaded, so the ordinary + "reinstall/restart over the live gateway" case lands here with the service up and supervised. + Degrading there is not a graceful fallback: it writes the permanent launchd-unsupported marker and + starts a detached gateway *beside* the supervised one, and the marker makes + :func:`wait_for_launchd_gateway_supervision` answer True unconditionally — so no later + install/update can tell that nothing ties the gateway to launchd any more. A live supervised PID is + direct evidence this macOS does manage the job, so surface the failure instead of branding the host. + """ + if not _launchctl_domain_unsupported(exc.returncode): + raise exc + label = _gw().get_launchd_label() + if _gw()._launchctl_label_supervising_process(label): + print(f"⚠ {what} failed (exit {exc.returncode}), but launchd still supervises {label}") + print(" Not switching to the detached fallback — this host manages the job.") + print(" Apply the definition with: hermes gateway stop && hermes gateway install --force") + raise exc + _launchd_fallback_to_detached(f"{what} exit {exc.returncode}") + + +def generate_launchd_plist() -> str: + # Stable cwd anchor — never the volatile source checkout (same rot risk as systemd's WorkingDirectory). + working_dir = _gw()._stable_service_working_dir() + hermes_home = str(_gw().get_hermes_home().resolve()) + log_dir = _gw().get_hermes_home() / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + label = _gw().get_launchd_label() + venv_dir = _gw()._service_venv_dir() + # launchd's default PATH misses Homebrew, nvm, cargo…; prepend venv/bin + node dirs (as in the + # systemd unit) so node stays resolvable even if the shell PATH changes, then the shell PATH. + priority_dirs = _gw()._build_service_path_dirs() + _gw()._append_node_dir_for_service(priority_dirs) + sane_path = ":".join(dict.fromkeys(priority_dirs + [p for p in os.environ.get("PATH", "").split(":") if p])) + + # ProgramArguments (incl. --profile); the stderr wrapper keeps launchd restart semantics while timestamping stderr. + prog_args_xml = "\n ".join( + f"{part}" + for part in _timestamped_stderr_gateway_command(log_dir / "gateway.error.log", external_supervisor=True) + ) + + # Persist the configured RLIMIT_NOFILE floor: launchd defaults to soft 256, and every plist + # rewrite would otherwise strip a manual limit and reintroduce EMFILE crashes. + nofile_block = "" + try: + from hermes_cli.resource_limits import configured_nofile_soft_limit + nofile_target = configured_nofile_soft_limit() + except Exception: + nofile_target = None + if nofile_target: + nofile_block = f""" + SoftResourceLimits + + NumberOfFiles + {nofile_target} + +""" + + return f""" + + + + Label + {label} + + ProgramArguments + + {prog_args_xml} + + + WorkingDirectory + {working_dir} + + EnvironmentVariables + + PATH + {sane_path} + VIRTUAL_ENV + {venv_dir} + HERMES_HOME + {hermes_home} + HERMES_SUPERVISED_CHILD + 1 + + + LimitLoadToSessionType + + Aqua + Background + + + RunAtLoad + + + KeepAlive + + SuccessfulExit + + + + + ThrottleInterval + 30 + + ExitTimeOut + 60 +{nofile_block} + StandardOutPath + {log_dir}/gateway.log + + StandardErrorPath + {log_dir}/gateway.error.log + + +""" + + +def launchd_plist_is_current() -> bool: + """Check if the installed launchd plist matches the currently generated one.""" + plist_path = _gw().get_launchd_plist_path() + if not plist_path.exists(): + return False + installed = plist_path.read_text(encoding="utf-8") + norm = _gw()._normalize_launchd_plist_for_comparison + return norm(installed) == norm(_gw().generate_launchd_plist()) + + +def _spawn_deferred_launchd_reload( + *, domain: str, label: str, target: str, plist_path: Path, gateway_pid: int +) -> bool: + """Hand the bootout/bootstrap cycle to a transient ``launchctl submit`` job; True if spawned. The + helper waits for the OLD gateway to exit (bootstrap during drain fails EIO), then retries bootstrap + until ``launchctl list`` shows a positive PID or the drain budget elapses.""" + reload_log_path = _launchd_reload_log_path() + with contextlib.suppress(OSError): + reload_log_path.parent.mkdir(parents=True, exist_ok=True) + + # Durable pre-bootout marker: distinguishes "helper never started" from "helper ran but failed". + _gw()._append_launchd_reload_log(f"Launchd reload helper started for {target}") + + _reload_budget = int(_launchd_reload_budget()) + q_target, q_label, q_log = shlex.quote(target), shlex.quote(label), shlex.quote(str(reload_log_path)) + stamp = "$(date '+%Y-%m-%d %H:%M:%S %z')" + # Require a POSITIVE PID: `launchctl list` also exits 0 for a registered-but-not-running + # definition, and a crashed job reports `"PID" = -1` (mirrors _parse_launchd_pid_from_list_output). + listed = f"launchctl list {q_label} 2>/dev/null | grep -qE '\\\"PID\\\" = [0-9]+;'" + # Unique per reload so concurrent/repeated reloads never collide. + submit_label = f"{label}.reload.{os.getpid()}.{int(time.time())}" + reload_script = ( + f"sleep 2; " + f"launchctl bootout {q_target} 2>/dev/null; " + # Wait for the OLD gateway to exit: bootout only SIGTERMs and every bootstrap during the drain fails EIO. + f"_wait_deadline=$(($(date +%s) + {_reload_budget})); " + f"while kill -0 {gateway_pid} 2>/dev/null; do if [ $(date +%s) -ge $_wait_deadline ]; then " + f" echo \"[{stamp}] old gateway pid {gateway_pid} still alive after {_reload_budget}s drain wait — bootstrapping anyway\" >> {q_log}; " + f" break; fi; sleep 1; done; " + # Let launchd finish unregistering the label after the process exits. + f"sleep 1; _deadline=$(($(date +%s) + {_reload_budget})); while :; do " + f" launchctl bootstrap {shlex.quote(domain)} {shlex.quote(str(plist_path))} 2>/dev/null; " + f" if {listed}; then break; fi; " + f" echo \"[{stamp}] bootstrap not yet registered for {q_target} — retrying\" >> {q_log}; " + f" if [ $(date +%s) -ge $_deadline ]; then break; fi; sleep 2; done; " + f"if ! {listed}; then " + f" echo \"[{stamp}] FAILED launchd reload for {q_target} — service NOT registered after {_reload_budget}s of retries\" >> {q_log}; " + f"fi; " + # Submitted jobs stay registered after the script exits; removing our own label ends the one-shot job. + f"launchctl remove {shlex.quote(submit_label)} 2>/dev/null" + ) + try: + # `launchctl submit` rather than setsid: setsid does NOT leave the launchd coalition that bootout kills. + # Spawn the reload helper via `launchctl submit` (a transient launchd one-shot job) instead of + # `start_new_session=True`. `start_new_session=True` only calls setsid(2), which creates a new POSIX + # session but does NOT move the child outside the launchd job's process coalition. When `launchctl + # bootout` fires on the gateway label, launchd terminates ALL processes in that coalition — + # including a setsid-detached child (#69098). `launchctl submit` creates a wholly independent + # transient launchd job that launchd manages separately from the gateway, so bootout of the gateway + # job cannot reach the helper. + subprocess.Popen( + [ + "launchctl", "submit", "-l", submit_label, "-o", str(reload_log_path), "-e", str(reload_log_path), + "--", "/bin/bash", "-c", reload_script, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except Exception as e: + # Fall through to in-process bootout/bootstrap: risky in the coalition, but better than a never-reloaded plist. + _gw().logger.warning("Deferred launchd reload could not be spawned: %s", e) + _gw()._append_launchd_reload_log( + f"FAILED to spawn launchd reload helper for {target}: {e} — falling back to in-process bootout/bootstrap" + ) + return False + return True + + +def refresh_launchd_plist_if_needed() -> bool: + """Rewrite the installed plist when the generated one differs, then bootout/bootstrap so launchd + re-reads it immediately.""" + plist_path = _gw().get_launchd_plist_path() + if not plist_path.exists() or _gw().launchd_plist_is_current(): + return False + + new_plist = _gw().generate_launchd_plist() + if _gw()._refuse_temp_home_service_write(new_plist, "launchd plist"): + return False + + plist_path.write_text(new_plist, encoding="utf-8") + label = _gw().get_launchd_label() + domain = _gw()._launchd_domain() + target = f"{domain}/{label}" + + # Inside the gateway's launchd process tree (agent self-update) a direct bootout kills THIS CLI + # before bootstrap runs, leaving the job unloaded with no KeepAlive. + try: + from gateway.status import get_running_pid + gateway_pid = get_running_pid() + except Exception: + gateway_pid = None + + # POSIX ancestry is NOT a reliable "bootout will kill us" test (coalition membership survives + # reparenting), so always prefer the detached helper; in-process is only the spawn-failure fallback. + if ( + gateway_pid is not None + and hasattr(os, "setsid") # POSIX-only; launchd is macOS so always true here + ) and _spawn_deferred_launchd_reload( + domain=domain, label=label, target=target, plist_path=plist_path, gateway_pid=gateway_pid + ): + print( + "↻ Updated gateway launchd service definition; reload deferred to " + "a transient launchd job (survives the bootout of this process)" + ) + return True + + # Bootout/bootstrap so launchd reads the new definition; bootstrap can fail silently under load + # during a drain, and KeepAlive can't revive an unregistered job. + # Captured: best-effort (the job may already be unloaded), keep expected noise off the terminal. + subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT) + _reload_budget = _launchd_reload_budget() + # Wait out the old gateway's drain first so the budget isn't burned on guaranteed EIO ("already loaded"). + if gateway_pid is not None and not _gw()._wait_for_pid_exit(gateway_pid, _reload_budget): + _gw()._append_launchd_reload_log( + f"old gateway pid {gateway_pid} still alive after " + f"{int(_reload_budget)}s drain wait — bootstrapping {target} anyway" + ) + _deadline = time.monotonic() + _reload_budget + if not _gw()._retry_launchctl_bootstrap_until_registered(domain, plist_path, label, deadline=_deadline): + _gw()._append_launchd_reload_log( + f"FAILED launchd reload of {target} — service NOT registered after " + f"retrying for {int(_reload_budget)}s (in-process fallback path)" + ) + _gw().logger.error( + "launchd reload of %s failed — service not registered after %ds of retries; see %s", + target, int(_reload_budget), _launchd_reload_log_path(), + ) + return False + print("↻ Updated gateway launchd service definition to match the current Hermes install") + return True + + +def launchd_install(force: bool = False): + plist_path = _gw().get_launchd_plist_path() + + if plist_path.exists() and not force: + if not _gw().launchd_plist_is_current(): + print(f"↻ Repairing outdated launchd service at: {plist_path}") + if _gw().refresh_launchd_plist_if_needed(): + print("✓ Service definition updated") + else: + # The plist was rewritten but launchd never registered it (or the write was refused): + # a success line here would hide an unloaded service with no KeepAlive. + from hermes_constants import display_hermes_home + print( + "⚠ Service definition could not be reloaded with launchd. " + "Run 'hermes gateway install --force' or check " + f"{display_hermes_home()}/logs/launchd-reload.log for details." + ) + return + print(f"Service already installed at: {plist_path}") + print("Use --force to reinstall") + return + + plist_path.parent.mkdir(parents=True, exist_ok=True) + new_plist = _gw().generate_launchd_plist() + if _gw()._refuse_temp_home_service_write(new_plist, "launchd plist"): + return + print(f"Installing launchd service to: {plist_path}") + plist_path.write_text(new_plist, encoding="utf-8") + + try: + _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, _gw().get_launchd_label(), timeout=30) + except subprocess.CalledProcessError as e: + _gw()._launchd_degrade_or_raise(e, "launchctl bootstrap") + return + + print() + print("✓ Service installed and loaded!") + _gw()._clear_launchd_unsupported_marker() + print() + print("Next steps:") + print(" hermes gateway status # Check status") + from hermes_constants import display_hermes_home as _dhh + print(f" tail -f {_dhh()}/logs/gateway.log # View logs") + + +def launchd_uninstall(): + plist_path = _gw().get_launchd_plist_path() + # Captured: uninstalling an already-unloaded job is fine — don't print Boot-out failed: 3. + subprocess.run( + ["launchctl", "bootout", f"{_launchd_domain()}/{get_launchd_label()}"], + check=False, timeout=90, **_gw()._CAPTURE_TEXT) + if plist_path.exists(): + plist_path.unlink() + print(f"✓ Removed {plist_path}") + print("✓ Service uninstalled") + + +def launchd_start(): + plist_path = _gw().get_launchd_plist_path() + label = _gw().get_launchd_label() + + # Self-heal if the plist is missing entirely (e.g., manual cleanup, failed upgrade) + if not plist_path.exists(): + new_plist = _gw().generate_launchd_plist() + if _gw()._refuse_temp_home_service_write(new_plist, "launchd plist"): + sys.exit(1) + print("↻ launchd plist missing; regenerating service definition") + plist_path.parent.mkdir(parents=True, exist_ok=True) + plist_path.write_text(new_plist, encoding="utf-8") + if _launchd_bootstrap_and_kickstart(plist_path, label): + _launchd_ok("✓ Service started") + return + + _gw().refresh_launchd_plist_if_needed() + try: + _launchctl_kickstart_current(label) + except subprocess.CalledProcessError as e: + if not _launchd_error_indicates_unloaded(e): + raise + # Job not loaded in this domain — re-bootstrap the plist and retry. + print("↻ launchd job was unloaded; reloading service definition") + if not _launchd_bootstrap_and_kickstart(plist_path, label): + return + _launchd_ok("✓ Service started") + + +def _launchctl_kickstart_current(label: str) -> None: + subprocess.run(["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], check=True, timeout=30) + + +def _launchd_bootstrap_and_kickstart(plist_path: Path, label: str) -> bool: + """Bootstrap then kickstart; False after degrading to detached (domain unsupported). Other errors propagate.""" + try: + _gw()._launchctl_bootstrap(_gw()._launchd_domain(), plist_path, label, timeout=30) + _launchctl_kickstart_current(label) + except subprocess.CalledProcessError as e: + _gw()._launchd_degrade_or_raise(e, "launchctl") + return False + return True + + +def _launchd_ok(message: str) -> None: + """Print a launchd success line and clear the unsupported marker (an OS fix recovers automatically).""" + print(message) + _gw()._clear_launchd_unsupported_marker() + + +def launchd_stop(): + target = f"{_launchd_domain()}/{get_launchd_label()}" + _gw()._mark_planned_stop() + # bootout unloads the definition so KeepAlive doesn't respawn; `hermes gateway start` re-bootstraps. + try: + # Captured: an already-unloaded job (3/113/125) is handled below, so launchctl's own + # "Boot-out failed: 3" must not print around the ✓ line; e.stderr stays on the raised error. + subprocess.run(["launchctl", "bootout", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT) + except subprocess.CalledProcessError as e: + # Job already unloaded (3/113/125), or the domain can't be managed at all (5/125, macOS 26+ + # detached-fallback process, issue #23387) — in both cases just fall through to the PID-based kill + # below. + if not (_launchd_error_indicates_unloaded(e) or _launchctl_domain_unsupported(e.returncode)): + raise + _gw()._wait_for_gateway_exit(timeout=10.0, force_after=5.0) + print("✓ Service stopped") + + +def _launchd_kickstart(label: str, domain: str) -> None: + """``launchctl kickstart -k domain/label``; raises so callers own per-label failure accounting.""" + subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True, timeout=90, **_gw()._CAPTURE_TEXT) + + +def _wait_for_launchd_service_pid( + label: str, old_pid: int | None, timeout: float = 10.0, *, domain: str +) -> bool: + """Poll ``domain/label`` (0.5s) until it runs on a fresh PID or ``timeout`` passes — KeepAlive respawn + isn't instantaneous. launchctl ``TimeoutExpired`` propagates; callers own failure accounting.""" + deadline = time.monotonic() + max(timeout, 0.5) + while True: + _loaded, pid = _gw()._launchd_print_service_pid(domain, label) + if pid is not None and pid > 0 and pid != old_pid: + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.5) + + +def launchd_restart(): + label = _gw().get_launchd_label() + domain = _gw()._launchd_domain() + target = f"{domain}/{label}" + from gateway.status import get_running_pid + try: + pid = get_running_pid() + if pid is not None and _gw()._request_gateway_self_restart(pid): + _launchd_ok("✓ Service restart requested") + return + if pid is not None and _gw().probe_gateway_loop_liveness(pid) == _gw().GATEWAY_LOOP_WEDGED: + # Event loop provably dead: it can't process a graceful shutdown, so a full drain wait + # only stalls the restart (and `hermes update`). Bounded SIGTERM → SIGKILL, ~10s. + print(f"⚠ Gateway PID {pid} event loop is unresponsive — " "skipping drain and forcing a bounded stop...") + _gw()._escalate_wedged_gateway(pid) + pid = None + if pid is not None: + # Graceful in-band restart via SIGUSR1 (mirrors systemd); the budget covers both the idle wait + # and the drain. A bare SIGTERM would lose the resume_pending handoff. Announce BEFORE waiting: + # surfaces with no other feedback (desktop updater) read silence as "update stuck". + wait_budget = _gw()._get_restart_exit_wait_budget() + print(f"→ Stopping gateway (PID {pid}) — draining in-flight runs (up to {wait_budget:.0f}s)...") + from hermes_cli.update_cmd_drain_report import drain_progress_reporter + if _gw()._graceful_restart_via_sigusr1(pid, wait_budget, on_progress=drain_progress_reporter(budget_s=wait_budget)): + # KeepAlive revives a planned exit, so do NOT kickstart (-k would kill the replacement) — + # but a clean exit doesn't prove supervision, so verify a replacement PID appears first. + if _gw()._wait_for_launchd_service_pid(label, pid, timeout=15.0, domain=domain): + _launchd_ok("✓ Service restart requested") + return + print("⚠ launchd did not revive the gateway after its graceful exit — forcing restart") + else: + print(f"⚠ Gateway drain timed out after {wait_budget:.0f}s — forcing launchd restart") + # Captured: an unloaded job (3/113/125) is the expected case below, which + # prints its own ↻ line — and e.stderr feeds the update_cmd failure diagnostic. + _gw()._wait_for_api_server_port_free() + subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90, **_gw()._CAPTURE_TEXT) + _launchd_ok("✓ Service restarted") + except subprocess.CalledProcessError as e: + if not _launchd_error_indicates_unloaded(e): + _gw()._launchd_degrade_or_raise(e, "launchctl kickstart") + return + # Job not loaded — bootstrap and start fresh + print("↻ launchd job was unloaded; reloading") + try: + # After a drain the job is usually still registered (bootstrap would hit EIO): boot it out first. + # Captured: best-effort (the job may already be unloaded after the drain), + # so an expected Boot-out failed: 3 must not leak past the ↻ line below. + subprocess.run(["launchctl", "bootout", target], check=False, timeout=90, **_gw()._CAPTURE_TEXT) + plist_path = str(_gw().get_launchd_plist_path()) + subprocess.run(["launchctl", "bootstrap", _gw()._launchd_domain(), plist_path], check=True, timeout=30) + subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30) + except subprocess.CalledProcessError as e2: + _gw()._launchd_degrade_or_raise(e2, "launchctl") + return + _launchd_ok("✓ Service restarted") + + +# KeepAlive relaunches at most ~once per 10s, so a self-restart leaves the label pid-less that long. +LAUNCHD_SUPERVISION_VERIFY_TIMEOUT = 20.0 + + +def wait_for_launchd_gateway_supervision( + *, + timeout: float = LAUNCHD_SUPERVISION_VERIFY_TIMEOUT, + label: str | None = None, + poll_interval: float = 0.5, + old_pid: int | None = None, +) -> bool: + """Poll launchd until it supervises a live gateway; True at once if the detached fallback is active. + ``launchd_restart`` returns once the restart is *requested* (asynchronous), so it can't see a helper + dying before bootstrap or a ``launchctl bootstrap`` that exits 0 without registering. + + The ``_request_gateway_self_restart`` branch hands the work to the running gateway and returns + immediately, and a plist reload is handed to a detached helper. Both are asynchronous, so a caller that + reads "returned without raising" as "the service is up" cannot see a helper that dies before its first + bootstrap (#88848) — nor a ``launchctl bootstrap`` that exits 0 without registering, which the reporter + measured on macOS 26.6.1. + Judge the outcome the way #80491 taught the helper to judge it: by a live supervised pid, never by an + exit code. :func:`_launchctl_supervised_pid` is already that probe, so this only adds the wait. + + ``old_pid`` is the pid launchd ran for the label *before* the restart: a restart that leaves the same + process running is not a restart, so passing it holds the invoking profile to the same fresh-pid + contract :func:`_wait_for_launchd_service_pid` enforces for sibling labels. With ``old_pid=None`` + (no pre-restart pid was observable) any supervised pid counts, as before. + """ + if _gw()._launchd_unsupported_marker_exists(): + return True + + label = label or _gw().get_launchd_label() + deadline = time.monotonic() + max(timeout, 0.0) + while True: + pid = _gw()._launchctl_supervised_pid(label) + if pid is not None and pid != old_pid: + return True + if time.monotonic() >= deadline: + return False + time.sleep(max(poll_interval, 0.01)) + + +def launchd_status(deep: bool = False): + plist_path = _gw().get_launchd_plist_path() + label = _gw().get_launchd_label() + try: + result = subprocess.run(["launchctl", "list", label], timeout=10, **_gw()._CAPTURE_TEXT) + service_listed = result.returncode == 0 + list_output = result.stdout + except subprocess.TimeoutExpired: + service_listed = False + list_output = "" + + # `launchctl list` exits 0 for any registered definition (even `state = not running`); only a PID proves a process. + launchd_pid = _gw()._parse_launchd_pid_from_list_output(list_output) if service_listed else None + + # Hermes PID may be a detached fallback process; when launchd IS supervising both PIDs match — don't double-count. + from gateway.status import get_running_pid + fallback_pid = get_running_pid(cleanup_stale=False) + if launchd_pid is not None and fallback_pid == launchd_pid: + fallback_pid = None + + # Marker from a 5/125 bootstrap/kickstart failure explains *why* launchd can't supervise. + launchd_unsupported = _gw()._launchd_unsupported_marker_exists() + + print(f"Launchd plist: {plist_path}") + if _gw().launchd_plist_is_current(): + print("✓ Service definition matches the current Hermes install") + else: + print("⚠ Service definition is stale relative to the current Hermes install") + print(" Run: hermes gateway start") + + if not service_listed: + print("✗ Gateway service is not loaded") + print(" Service definition exists locally but launchd has not loaded it.") + print(" Run: hermes gateway start") + if fallback_pid: + print(f" Note: a detached gateway process is running (PID {fallback_pid})") + elif launchd_pid is not None: + print(f"✓ Gateway is supervised by launchd (PID {launchd_pid})") + print(" Auto-start at login and auto-restart on crash are available.") + if launchd_unsupported: + print(" (launchd domain was previously unavailable but is now working)") + elif launchd_unsupported: + print("⚠ Gateway service is registered but launchd is not supervising it") + print(" launchd cannot manage the gateway on this macOS version.") + if fallback_pid: + print(f"✓ Detached fallback process is running (PID {fallback_pid})") + print(" Cron jobs will fire. Stop with: hermes gateway stop") + else: + print("✗ No fallback process is running") + print(" Run: hermes gateway start") + print(" ⚠ Auto-start at login and auto-restart on crash are NOT available.") + else: + print("✓ Gateway service is registered with launchd") + print(list_output) + if fallback_pid: + print(f" Detached gateway process is running (PID {fallback_pid})") + + if deep: + log_file = _gw().get_hermes_home() / "logs" / "gateway.log" + if log_file.exists(): + print() + print("Recent logs:") + subprocess.run(["tail", "-20", str(log_file)], timeout=10)