hermes-agent/hermes_cli/main_dashboard.py

"""Dashboard/serve support: managed-service restart (systemd/respawn), status/listening probes, SSH session token file, named-profile routing, web-dist resolution, update stdio hangup protection.

Split out of ``hermes_cli/main.py``. Names that still live in main (``PROJECT_ROOT``, ...)
are imported lazily inside the functions that use them (avoids an import cycle).
"""

import contextlib
import os
import re
import shlex
import subprocess
import sys

from pathlib import Path
from typing import NoReturn
from hermes_cli.cli_output import line_input
from hermes_cli.process_identity import is_desktop_owned_backend as _is_desktop_owned_backend

_PRE_BUILD_HINT = "  Pre-build first:  npm install --workspace web && npm run build -w web"


def _find_stale_dashboard_pids(*, exclude_pids: set[int] | None = None,
                               scope_home: str | None = None) -> list[int]:
    """PIDs of running ``dashboard``/``serve`` backends the caller may stop.

    *scope_home*: keep only backends whose resolved Hermes home (see
    ``_hermes_home_for_pid``) is this home; unreadable ownership is spared, never guessed.
    ``--stop`` and the post-update cleanup pass their own home so another install's or
    profile's backend on the same machine is never a target (#113978).
    """
    from hermes_cli.dashboard_procs import (
        _caller_ancestor_pids,
        _is_caller_wrapper_shell,
        _pids_owned_by_hermes_home,
        _scan_dashboard_processes,
    )
    pids = [pid for pid, _cmd in _scan_dashboard_processes(exclude_pids=exclude_pids)]
    # The argv substring scan also selects the caller's own wrapper shell (``bash -c
    # 'hermes dashboard --stop'``); killing it takes down the invoking terminal.
    ancestors = _caller_ancestor_pids()
    pids = [pid for pid in pids if not _is_caller_wrapper_shell(pid, ancestors)]
    return _pids_owned_by_hermes_home(pids, scope_home) if scope_home else pids


def _parse_dashboard_runtime(command: str) -> tuple[str, str, int] | None:
    """Best-effort parse of a dashboard/server cmdline into mode, host, and port."""
    mode = None
    for candidate in ("dashboard", "serve"):
        patterns = (f"hermes {candidate}", f"hermes_cli.main {candidate}", f"hermes_cli/main.py {candidate}")
        if any(pattern in command for pattern in patterns):
            mode = candidate
            break
    if mode is None:
        return None

    port = 9119
    host = "127.0.0.1"

    port_match = re.search(r"(?:^|\s)--port(?:=|\s+)(\d+)", command)
    if port_match:
        try:
            port = int(port_match.group(1))
        except ValueError:
            return None

    host_match = re.search(r"(?:^|\s)--host(?:=|\s+)(\"[^\"]+\"|'[^']+'|\S+)", command)
    if host_match:
        host = host_match.group(1).strip("\"'") or "127.0.0.1"

    return mode, host, port


def _dashboard_probe_host(host: str | None) -> str:
    """Map wildcard binds to a loopback address suitable for local probing."""
    normalized = (host or "127.0.0.1").strip().strip("[]")
    if normalized in {"", "0.0.0.0", "::"}:
        return "127.0.0.1"
    return normalized


_DASHBOARD_SYSTEMD_UNIT = "hermes-dashboard.service"

_SYSTEMCTL_ERRORS = (FileNotFoundError, subprocess.TimeoutExpired, OSError)


def _run_probe(cmd: list[str], *, timeout: int) -> subprocess.CompletedProcess:
    """Captured, text-decoded ``subprocess.run`` for short local probes (systemctl, ps)."""
    return subprocess.run(
        cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout)


def _restart_managed_dashboard_service(reason: str, unit: str = _DASHBOARD_SYSTEMD_UNIT) -> bool:
    """Restart a systemd-managed dashboard instead of raw-killing its PID.

    True when a unit was found and handled (success or printed failure) — which
    deliberately stops the caller's ``os.kill`` fallback: systemd treats a direct
    SIGTERM as a clean stop, so ``Restart=on-failure`` won't bring it back.
    """
    if sys.platform == "win32":
        return False

    def _systemctl(*args: str, timeout: int = 10) -> subprocess.CompletedProcess:
        return _run_probe(["systemctl", *args], timeout=timeout)

    # User manager first (Hermes installs Linux services in the user scope by
    # default), system manager only when the unit isn't there. Keep the selected
    # scope for ALL probes and the restart — a user unit must never be restarted
    # through the system manager (or raw-killed).
    scope: tuple[str, ...] | None = None
    for candidate in (("--user",), ()):
        try:
            result = _systemctl(*candidate, "list-unit-files", unit, "--no-legend", "--no-pager")
        except _SYSTEMCTL_ERRORS:
            continue
        if result.returncode != 0:
            continue
        unit_rows = (result.stdout or "").splitlines()
        if any(row.split()[0:1] == [unit] for row in unit_rows if row.split()):
            scope = candidate
            break

    if scope is None:
        return False

    try:
        active = _systemctl(*scope, "is-active", unit)
        enabled = _systemctl(*scope, "is-enabled", unit)
    except _SYSTEMCTL_ERRORS:
        return False

    active_state = (active.stdout or "").strip()
    enabled_state = (enabled.stdout or "").strip()
    if active_state != "active" and enabled_state not in {
        "enabled", "enabled-runtime", "linked", "linked-runtime", "static", "generated",
    }:
        return False

    print(f"\n⟲ Restarting managed dashboard service ({reason})")

    scope_label = "systemctl --user" if scope else "sudo systemctl"
    commands = [("systemctl", *scope, "restart", unit)]
    if not scope:
        # System units may require privilege escalation; user units must use
        # the user manager directly and never prompt for sudo.
        commands.append(("sudo", "-n", "systemctl", "restart", unit))

    errors: list[str] = []
    for command in commands:
        try:
            result = _run_probe(list(command), timeout=60)
        except _SYSTEMCTL_ERRORS as e:
            errors.append(f"{' '.join(command)}: {e}")
            continue
        if result.returncode == 0:
            print(f"    ✓ restarted {unit}")
            return True
        errors.append(f"{' '.join(command)}: {(result.stderr or result.stdout or '').strip()}")

    print(f"    ✗ failed to restart {unit}")
    for err in errors:
        if err.strip():
            print(f"      {err}")
    print(
        "  Dashboard is managed by systemd; not raw-killing its PID because "
        "systemd would treat that as a clean stop."
    )
    print(f"  Restart manually: {scope_label} restart {unit}")
    return True


def _pid_unified_cgroup_entries(pid: int):
    """Yield the ``0::<path>`` cgroup paths from ``/proc/<pid>/cgroup``; nothing when unreadable."""
    try:
        cgroup_path = Path(f"/proc/{pid}/cgroup")
        if not cgroup_path.is_file():
            return
        text = cgroup_path.read_text(encoding="utf-8-sig", errors="replace")
    except (OSError, PermissionError):
        return
    for line in text.splitlines():
        parts = line.strip().split("::", 1)
        if len(parts) == 2:
            yield parts[1]


def _get_systemd_service_for_pid(pid: int) -> str | None:
    """The systemd service unit name *pid* belongs to (``hermes-serve.service``), or None.

    None when the PID isn't part of a service, the file is unreadable, or off Linux.
    """
    for cg_path in _pid_unified_cgroup_entries(pid):
        if cg_path.endswith(".service"):
            svc_name = cg_path.rsplit("/", 1)[-1]
            if svc_name:
                return svc_name
    return None


def _extract_scope_from_cgroup(cgroup_entry: str) -> str | None:
    """``user`` / ``system`` from a cgroup path (``/user.slice/…`` vs ``/system.slice/…``), else None."""
    if "/system.slice/" in cgroup_entry:
        return "system"
    if "/user.slice/" in cgroup_entry:
        return "user"
    return None


def _get_pid_cgroup_path(pid: int) -> str | None:
    """The unified (``0::``) cgroup path from ``/proc/<pid>/cgroup``, or None."""
    return next(_pid_unified_cgroup_entries(pid), None)


def _try_restart_systemd_service(svc_name: str, cgroup_path: str | None = None) -> bool:
    """Restart *svc_name* via systemctl (``--user`` for user-scope units). True on success.

    Unknown scope tries system first, then user.
    """
    scope = _extract_scope_from_cgroup(cgroup_path) if cgroup_path else None
    system_cmd = ["systemctl", "restart", svc_name]
    user_cmd = ["systemctl", "--user", "restart", svc_name]
    candidates = {"user": [user_cmd], "system": [system_cmd]}.get(scope, [system_cmd, user_cmd])
    for cmd in candidates:
        try:
            if _run_probe(cmd, timeout=15).returncode == 0:
                return True
        except _SYSTEMCTL_ERRORS:
            continue
    return False


# launchd plist directories that can supervise a ``hermes dashboard`` / ``hermes serve`` backend on
# macOS, with the launchctl domain their jobs load into (LaunchAgents: ``gui/<uid>`` or ``user/<uid>``,
# probed per label like the gateway helpers; LaunchDaemons: ``system``). Both LaunchAgents dirs are
# per-user domains, so they share the ``agent`` kind.
def _launchd_plist_dirs() -> list[tuple[str, Path]]:
    return [
        ("agent", Path.home() / "Library" / "LaunchAgents"),
        ("agent", Path("/Library/LaunchAgents")),
        ("daemon", Path("/Library/LaunchDaemons")),
    ]


def _loaded_launchd_backend_jobs(
    plist_dirs: list[tuple[str, Path]] | None = None,
) -> list[tuple[str, str, list[str], int | None]]:
    """``(domain, label, program_arguments, live_pid)`` for every LOADED launchd job whose
    ``ProgramArguments`` is a ``hermes dashboard`` / ``hermes serve`` backend. macOS only (empty
    elsewhere). Reads the plists (unreadable/malformed ones are skipped) and asks ``launchctl print``
    per candidate label — a job that is not loaded in any domain is not returned, so an operator's
    stale plist never claims a process."""
    if sys.platform != "darwin":
        return []
    import plistlib
    from xml.parsers.expat import ExpatError

    from hermes_cli.gateway import _launchd_print_service_pid
    uid = os.getuid()  # windows-footgun: ok — darwin-only branch
    jobs: list[tuple[str, str, list[str], int | None]] = []
    for kind, plist_dir in (plist_dirs if plist_dirs is not None else _launchd_plist_dirs()):
        try:
            plists = sorted(plist_dir.glob("*.plist"))
        except OSError:
            continue
        for plist_path in plists:
            try:
                with open(plist_path, "rb") as f:
                    data = plistlib.load(f)
            # ExpatError is NOT a ValueError: plistlib propagates it unwrapped for
            # XML that is not well-formed (e.g. a hand-edited plist with a raw
            # `&` in `ProgramArguments`), and one such operator file must skip —
            # not abort — the whole post-pull cleanup scan.
            except (OSError, ValueError, plistlib.InvalidFileException, ExpatError):
                continue
            if not isinstance(data, dict):
                continue
            label = str(data.get("Label") or "").strip()
            args = data.get("ProgramArguments")
            if not label or not isinstance(args, list) or not args:
                continue
            argv = [str(a) for a in args]
            if _parse_dashboard_runtime(shlex.join(argv)) is None:
                continue
            domains = ("system",) if kind == "daemon" else (f"gui/{uid}", f"user/{uid}")
            for domain in domains:
                try:
                    loaded, live_pid = _launchd_print_service_pid(domain, label)
                except _SYSTEMCTL_ERRORS:
                    loaded, live_pid = False, None
                if loaded:
                    jobs.append((domain, label, argv, live_pid))
                    break
    return jobs


def _launchd_job_owning_backend(
    pid: int, cmdline: list[str] | None, jobs: list[tuple[str, str, list[str], int | None]],
    ancestors: "list[int] | tuple[int, ...]" = (),
) -> tuple[str, str, int | None] | None:
    """``(domain, label, live_pid)`` of the loaded launchd job that owns *pid*: launchd reports *pid*
    (or one of its *ancestors* — a plist may wrap the backend in ``/bin/sh -c …`` without ``exec``)
    as the job's live process, OR the process runs the job's ``ProgramArguments`` — a detached copy
    of a supervised backend (an earlier respawn) holds the port the job needs, and respawning it again
    would only re-create that conflict. ``--no-open`` is ignored on both sides: the respawn path adds
    it, so an earlier respawn's argv is the plist's plus that flag. None when no loaded job claims
    the process."""
    def _norm(argv: list[str]) -> list[str]:
        return [a for a in argv if a != "--no-open"]

    for domain, label, argv, live_pid in jobs:
        if live_pid is not None and (live_pid == pid or live_pid in ancestors):
            return (domain, label, live_pid)
        if cmdline is not None and _norm(list(cmdline)) == _norm(argv):
            return (domain, label, live_pid)
    return None


def _restart_launchd_job(domain: str, label: str, old_pid: int | None, *, timeout: float = 15.0) -> bool:
    """Bring a launchd-supervised backend back after its process was stopped: ``launchctl kickstart
    <domain>/<label>`` (no ``-k`` — a KeepAlive job may already have respawned it, and a kill would
    take that fresh process down), then require launchd to report a live PID other than *old_pid*
    within *timeout*. A kickstart that returns 0 only means "restart requested"; a job that is loaded
    but never comes back on a fresh PID is a failure the operator must hear about."""
    from hermes_cli.gateway import _wait_for_launchd_service_pid
    try:
        if _run_probe(["launchctl", "kickstart", f"{domain}/{label}"], timeout=30).returncode != 0:
            return False
        return _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=timeout, domain=domain)
    except _SYSTEMCTL_ERRORS:
        return False


def _dashboard_cmdline_for_pid(pid: int) -> list[str] | None:
    """Exact argv of a running process: ``/proc/<pid>/cmdline`` (Linux), ``ps -o command=`` + shlex
    (macOS), None on Windows (no graceful taskkill window; Desktop manages its backend)."""
    if sys.platform == "win32":
        return None
    try:
        cmdline_path = f"/proc/{pid}/cmdline"
        if os.path.exists(cmdline_path):
            with open(cmdline_path, "rb") as f:
                raw = f.read()
            argv = [part.decode("utf-8", errors="replace") for part in raw.split(b"\x00") if part]
            return argv or None
        result = _run_probe(["ps", "-p", str(pid), "-o", "command="], timeout=10)
        if result.returncode != 0:
            return None
        command = (result.stdout or "").strip()
        if not command:
            return None
        try:
            argv = shlex.split(command)
        except ValueError:
            argv = command.split()
        return argv or None
    except (OSError, ValueError, subprocess.TimeoutExpired):
        return None


def _respawn_dashboard_processes(commands: list[list[str]]) -> list[list[str]]:
    """Respawn manually-started dashboards after ``hermes update``, detached, logging to
    ``logs/dashboard-restart.log``; returns the argvs that failed to spawn. Callers pre-filter via
    ``_filter_dashboard_respawn_candidates`` (no Desktop ``--port 0`` backends, capped per profile).

    See #78821.
    """
    from hermes_constants import get_hermes_home
    respawned: list[list[str]] = []
    failed: list[tuple[list[str], str]] = []
    log_path = get_hermes_home() / "logs" / "dashboard-restart.log"
    with contextlib.suppress(OSError):
        log_path.parent.mkdir(parents=True, exist_ok=True)

    for command in commands:
        try:
            # Keep restarted dashboards headless; reopening a browser after a
            # background update is noisy and fails in SSH/headless sessions.
            if "dashboard" in command and "--no-open" not in command:
                command = [*command, "--no-open"]
            with open(log_path, "ab") as log_f:
                subprocess.Popen(
                    command, stdin=subprocess.DEVNULL, stdout=log_f, stderr=subprocess.STDOUT,
                    start_new_session=True, close_fds=True)
            respawned.append(command)
        except (OSError, ValueError) as exc:
            failed.append((command, str(exc)))

    for command in respawned:
        print(f"    ✓ restarted: {shlex.join(command)}")
    for command, err_msg in failed:
        print(f"    ✗ failed to restart ({shlex.join(command)}): {err_msg}")
    return [command for command, _ in failed]


class _UpdateOutputStream:
    """stdout/stderr wrapper for ``hermes update``: mirrors to ``logs/update.log`` and, once the
    terminal vanishes (BrokenPipe/OSError/ValueError), drops screen output instead of the update."""

    _BROKEN = (BrokenPipeError, OSError, ValueError)

    def __init__(self, original, log_file):
        self._original = original
        self._log = log_file
        self._original_broken = False

    def write(self, data):
        # Mirror to the log file first — it's the most reliable destination.
        if self._log is not None:
            with contextlib.suppress(Exception):
                self._log.write(data)
        if not self._original_broken:
            try:
                return self._original.write(data)
            except self._BROKEN:
                self._original_broken = True  # terminal vanished; keep updating
        return len(data) if isinstance(data, (str, bytes)) else 0

    def flush(self):
        if self._log is not None:
            with contextlib.suppress(Exception):
                self._log.flush()
        if self._original_broken:
            return
        try:
            self._original.flush()
        except self._BROKEN:
            self._original_broken = True

    def isatty(self):
        if self._original_broken:
            return False
        try:
            return self._original.isatty()
        except Exception:
            return False

    def fileno(self):
        # Defer to the underlying stream; callers handle failures as when unwrapped.
        return self._original.fileno()

    def __getattr__(self, name):
        return getattr(self._original, name)


def _install_hangup_protection(gateway_mode: bool = False):
    """Protect ``cmd_update`` from SIGHUP (→ SIG_IGN, inherited by pip/git children) and broken pipes
    (stdio wrapped in ``_UpdateOutputStream``). SIGINT/SIGTERM are left alone — legitimate cancels.
    Gateway updates are already detached, but still need the log mirror for the
    Desktop progress watchdog. Returns state for ``_finalize_update_output``."""
    state = {
        "prev_stdout": sys.stdout, "prev_stderr": sys.stderr, "log_file": None, "installed": False}

    import signal as _signal

    if not gateway_mode and hasattr(_signal, "SIGHUP"):
        # Non-main thread: update still runs, just without hangup protection.
        with contextlib.suppress(ValueError, OSError):
            _signal.signal(_signal.SIGHUP, _signal.SIG_IGN)

    # Any failure here is non-fatal; we just skip the wrap.
    try:
        # Late-bound import so tests can monkeypatch
        # hermes_cli.config.get_hermes_home to simulate setup failure.
        from hermes_cli.config import get_hermes_home as _get_hermes_home
        logs_dir = _get_hermes_home() / "logs"
        logs_dir.mkdir(parents=True, exist_ok=True)
        log_file = open(logs_dir / "update.log", "a", buffering=1, encoding="utf-8")

        import datetime as _dt

        stage = "continued on the pulled code" if os.environ.get("HERMES_UPDATE_POST_SWAP") == "1" else "started"
        log_file.write(f"\n=== hermes update {stage} {_dt.datetime.now().isoformat(timespec='seconds')} ===\n")

        state["log_file"] = log_file
        sys.stdout = _UpdateOutputStream(state["prev_stdout"], log_file)
        sys.stderr = _UpdateOutputStream(state["prev_stderr"], log_file)
        state["installed"] = True
    except Exception:
        state["log_file"] = None

    return state


def _finalize_update_output(state):
    """Restore stdio and close the update.log handle opened by ``_install_hangup_protection``."""
    if not state:
        return
    if state.get("installed"):
        with contextlib.suppress(Exception):
            sys.stdout = state.get("prev_stdout", sys.stdout)
        with contextlib.suppress(Exception):
            sys.stderr = state.get("prev_stderr", sys.stderr)
    log_file = state.get("log_file")
    if log_file is not None:
        with contextlib.suppress(Exception):
            log_file.flush()
            log_file.close()


def _report_dashboard_status() -> int:
    """Print live listening dashboard/serve processes and return the count.

    Serve-mode backends are INCLUDED: ``--stop`` kills them, so hiding them from
    ``--status`` let an operator kill what they couldn't see.

    Ledger-registered serves (profiled launches the argv scan can't match) surface via the spawn-ledger
    augmentation in _scan_dashboard_processes. See #81564.
    """
    from hermes_cli.dashboard_procs import _scan_dashboard_processes
    from gateway.status import _pid_exists
    live: list[tuple[int, str, str]] = []
    for pid, command in _scan_dashboard_processes():
        runtime = _parse_dashboard_runtime(command)
        if runtime is None:
            continue
        mode, host, port = runtime
        if port <= 0 or not _pid_exists(pid) or not _dashboard_listening(host, port):
            continue
        live.append((pid, command, mode))

    if not live:
        print("No hermes dashboard or serve processes running.")
        return 0

    print(f"{len(live)} hermes dashboard/serve process(es) running:")
    for pid, command, mode in live:
        print(f"    PID {pid} [{mode}]: {command}")
    return len(live)


def _dashboard_listening(host: str, port: int) -> bool:
    """True when something accepts TCP connections at host:port (even a 401 proves a dashboard is up)."""
    import socket
    try:
        with socket.create_connection((_dashboard_probe_host(host), port), timeout=1.5):
            return True
    except OSError:
        return False


def _cancel(message: str = "  Cancelled.") -> NoReturn:
    print(message)
    sys.exit(1)


def _maybe_setup_dashboard_auth_interactively(args) -> None:
    """Offer to configure dashboard auth when the gate engages and no provider exists.

    ``start_server`` fails closed for a non-loopback bind / ``dashboard.public_url``
    without a ``DashboardAuthProvider``; prompt an interactive operator first.
    No-op (fail-closed backstop stays) when the gate doesn't engage, a provider
    exists, or stdin/stdout isn't a TTY.
    """
    host = getattr(args, "host", "127.0.0.1") or "127.0.0.1"

    try:
        from hermes_cli.web_server import should_require_dashboard_auth
        if not should_require_dashboard_auth(host):
            return
    except Exception:
        return  # if we can't tell, defer to start_server's own gate

    try:
        from hermes_cli.dashboard_auth import list_providers
        if list_providers():
            return
    except Exception:
        return

    if not (sys.stdin.isatty() and sys.stdout.isatty()):
        return

    print(f"\n⚠ Dashboard authentication is required for this configuration ({host}).")
    print(
        "  Non-loopback binds and configured external dashboard.public_url "
        "values require authentication (--insecure does not bypass this)."
    )
    print()
    print("  How do you want to authenticate the dashboard?")
    print("    [1] Username & password (quickest; for a trusted LAN / VPN)")
    print("    [2] OAuth via Nous Portal (run `hermes dashboard register`)\n    [3] Cancel\n")

    try:
        choice = input("  Choice [1]: ").strip() or "1"
    except (EOFError, KeyboardInterrupt):
        _cancel("\n  Cancelled.")

    if choice == "2":
        print()
        print(
            "  Run this on the host where the dashboard lives, then start "
            "the dashboard again:\n"
            "    hermes dashboard register\n"
            "  It provisions a Nous Portal OAuth client and writes "
            "HERMES_DASHBOARD_OAUTH_CLIENT_ID into ~/.hermes/.env for you.\n"
            "  Docs: https://hermes-agent.nousresearch.com/docs/"
            "user-guide/features/web-dashboard#authentication-gated-mode"
        )
        sys.exit(0)

    if choice != "1":
        _cancel()

    import getpass
    import secrets
    print()
    try:
        username = line_input("  Username [admin]: ").strip() or "admin"
        password = getpass.getpass("  Password: ")
        confirm = getpass.getpass("  Confirm password: ")
    except (EOFError, KeyboardInterrupt):
        _cancel("\n  Cancelled.")

    if not password:
        _cancel("  ✗ Empty password — aborting.")
    if password != confirm:
        _cancel("  ✗ Passwords don't match — aborting.")

    try:
        from plugins.dashboard_auth.basic import hash_password
    except Exception as exc:
        _cancel(f"  ✗ Could not load the password provider: {exc}")

    password_hash = hash_password(password)
    # A stable token-signing secret so sessions survive a dashboard restart.
    secret = secrets.token_urlsafe(32)

    try:
        from hermes_cli.config import load_config, save_config
        from hermes_cli.plugins_cmd import ensure_basic_auth_plugin_enabled_in_config
        cfg = load_config()
        basic = cfg.setdefault("dashboard", {}).setdefault("basic_auth", {})
        basic["username"] = username
        basic["password_hash"] = password_hash
        basic["password"] = ""  # never persist plaintext
        if not str(basic.get("secret", "") or "").strip():
            basic["secret"] = secret
        # The bundled basic provider is a backend plugin that honours
        # plugins.disabled; unblock it so discover_plugins below registers it,
        # and tell an operator who deliberately disabled it.
        if ensure_basic_auth_plugin_enabled_in_config(cfg):
            print("  ✓ Re-enabled the bundled 'basic' auth plugin (was in plugins.disabled)")
        save_config(cfg)
    except Exception as exc:
        _cancel(f"  ✗ Failed to write config.yaml: {exc}")

    # Re-run plugin discovery so the provider registers before start_server's gate.
    try:
        from hermes_cli.plugins import discover_plugins
        discover_plugins(force=True)
    except Exception as exc:
        print(f"  ⚠ Plugin re-discovery failed ({exc}); the gate may still "
              "fail closed. Set the password again or restart the dashboard.")

    print()
    print(f"  ✓ Username/password auth configured (user: {username}).")
    print("    Saved to config.yaml under dashboard.basic_auth.")
    print("    Sign in at the dashboard with these credentials.\n")


def _read_ssh_session_token_file(path: str) -> str:
    """Read and unlink a Desktop SSH token from its private runtime directory."""
    if sys.platform == "win32":
        from hermes_cli.windows_ssh_runtime import read_token
        return read_token(path)

    import stat as _stat

    if not os.path.isabs(path):
        raise SystemExit("--ssh-session-token-file must be absolute")

    # The Desktop client writes the token under the account's $HOME/.hermes/
    # desktop-ssh, independent of HERMES_HOME and the active profile. Anchor
    # validation there, NOT get_hermes_home(): a non-default profile or a Docker
    # /opt/data root re-homes get_hermes_home() and would reject every token.
    # See #69551.
    token_root = Path.home() / ".hermes" / "desktop-ssh"
    try:
        relative = Path(path).relative_to(token_root)
    except ValueError as exc:
        raise SystemExit("--ssh-session-token-file must be under the desktop-ssh directory") from exc
    if len(relative.parts) != 2 or not re.fullmatch(r"[0-9a-f]{32}", relative.parts[0]):
        raise SystemExit("--ssh-session-token-file has an invalid runtime path")
    if not re.fullmatch(r"[0-9a-f]{16}\.token", relative.parts[1]):
        raise SystemExit("--ssh-session-token-file has an invalid filename")

    directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
    file_flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
    uid = os.getuid() if hasattr(os, "getuid") else None

    def _check_dir(fd: int, what: str) -> None:
        st = os.fstat(fd)
        if not _stat.S_ISDIR(st.st_mode):
            raise SystemExit(f"--ssh-session-token-file has an unsafe {what}")
        if uid is not None and st.st_uid != uid:
            raise SystemExit(f"--ssh-session-token-file {what} has the wrong owner")
        if what == "parent directory" and (st.st_mode & 0o777) != 0o700:
            raise SystemExit("--ssh-session-token-file parent has unsafe permissions")

    root_fd = -1
    directory_fd = -1
    file_fd = -1
    try:
        try:
            root_fd = os.open(token_root, directory_flags)
            _check_dir(root_fd, "runtime root")
            directory_fd = os.open(relative.parts[0], directory_flags, dir_fd=root_fd)
            _check_dir(directory_fd, "parent directory")
            file_fd = os.open(relative.parts[1], file_flags, dir_fd=directory_fd)
        except SystemExit:
            raise
        except OSError as exc:
            if exc.errno == getattr(__import__("errno"), "ELOOP", -1):
                raise SystemExit("--ssh-session-token-file is a symlink") from exc
            raise SystemExit("--ssh-session-token-file is not accessible") from exc

        file_stat = os.fstat(file_fd)
        if not _stat.S_ISREG(file_stat.st_mode):
            raise SystemExit("--ssh-session-token-file is not a regular file")
        if file_stat.st_size != 64:
            raise SystemExit("--ssh-session-token-file contains an invalid token")
        if uid is not None and file_stat.st_uid != uid:
            raise SystemExit("--ssh-session-token-file has the wrong owner")
        if uid is not None and (file_stat.st_mode & 0o777) & ~0o600:
            raise SystemExit("--ssh-session-token-file has unsafe permissions")

        with os.fdopen(file_fd, "r", encoding="utf-8-sig") as token_stream:
            file_fd = -1
            token = token_stream.read(65)

        if not re.fullmatch(r"[0-9a-f]{64}", token):
            raise SystemExit("--ssh-session-token-file contains an invalid token")
        return token
    finally:
        if file_fd >= 0:
            os.close(file_fd)
        if directory_fd >= 0:
            with contextlib.suppress(OSError):
                os.unlink(relative.parts[1], dir_fd=directory_fd)
            os.close(directory_fd)
        if root_fd >= 0:
            os.close(root_fd)


def _is_electron_packaged_web_dist(path: str) -> bool:
    """True when *path* is an Electron-packaged renderer dist (``app.asar[.unpacked]/dist``).

    A standalone ``hermes dashboard`` inheriting that ``HERMES_WEB_DIST`` would
    serve the desktop frontend in the browser ("Desktop IPC bridge is unavailable").
    """
    if not path:
        return False
    return "app.asar" in path.replace("\\", "/")


def _host_backend_attachment():
    """Live host serve/dashboard record to attach to, or ``None``.

    Record-based discovery replaces the old bare-TCP probe: "something accepts a connection on
    this port" proved nothing about WHO answers (a foreign service, or a recycled PID's new
    owner). The record carries ``(pid, createTime)`` so liveness is proved against the same
    incarnation, and its token fingerprint must still match the 0600 token file the owner wrote.
    The record only nominates a CANDIDATE; :func:`_attach_to_host_backend` makes it prove itself.
    """
    try:
        from gateway import host_rendezvous as hr

        record = hr.read_record(hr.ROLE_SERVE)
        if record is None or not record.port:
            return None
        return record if hr.record_token_is_consistent(record) else None
    except Exception:
        return None


def _explicit_endpoint_flags(argv=None) -> set:
    """Which of ``--host``/``--port`` the operator actually typed.

    argparse defaults are indistinguishable from a typed value in ``args``, and the difference is
    load-bearing: an unset ``--port`` may attach to whatever port the host owner bound, but a
    typed ``--port 8899`` or ``--host 0.0.0.0`` (LAN access) must never be silently answered with
    a loopback attach on some other port.
    """
    typed = set()
    for token in (sys.argv[1:] if argv is None else argv):
        name = str(token).split("=", 1)[0]
        if name in ("--host", "--port"):
            typed.add(name[2:])
    return typed


def _endpoint_conflict(args, record, typed: set) -> str:
    """Why an explicitly requested endpoint cannot be served by ``record`` ('' when it can)."""
    if "port" in typed:
        wanted_port = getattr(args, "port", None)
        # ``--port 0`` is "any free port", not a demand for a specific one.
        if isinstance(wanted_port, int) and wanted_port > 0 and wanted_port != record.port:
            return f"--port {wanted_port} (the host owner is on port {record.port})"
    if "host" in typed:
        wanted_host = str(getattr(args, "host", "") or "")
        owner_host = record.host or "127.0.0.1"
        loopback = {"127.0.0.1", "localhost", "::1"}
        wildcard = {"0.0.0.0", "::", "*"}
        # A wildcard owner already answers on loopback; anything else must match exactly.
        reachable = wanted_host == owner_host or (owner_host in wildcard and wanted_host in loopback)
        if not reachable:
            return f"--host {wanted_host} (the host owner is bound to {owner_host})"
    return ""


def _attach_to_host_backend(args, headless_backend: bool) -> None:
    """Multiplex-only: a second `hermes serve`/`dashboard` attaches to the host backend.

    Exactly ONE backend runs per host and multiplexes every profile, so a second invocation —
    for ANY profile, the default included — reports the live one and exits 0 instead of binding
    a second port. ``--isolated`` opts out (Desktop's SSH backend proves ownership with it) and
    Desktop pool backends (HERMES_DESKTOP=1) keep their own lifecycle.

    Exit 0 means "the host backend answered and serves what you asked for", and nothing else:

    * the owner must ANSWER on its recorded port and identify itself (a record alone cannot see a
      graceful-shutdown window or a foreign listener that inherited the port) — a supervisor or
      `hermes update` relaunch landing in that window would otherwise exit 0 with NOTHING
      listening, reporting success for a dead service;
    * an explicitly typed ``--port``/``--host`` the owner cannot serve is a REFUSAL naming the
      owner, never a silent redirect. It exits 78 (EX_CONFIG), the deliberate-refusal code
      ``RestartPreventExitStatus=78`` parks on: exit 1 under ``Restart=always`` was an infinite
      restart loop with nothing listening on the ingress port (#119824);
    * a `hermes dashboard` user is never handed a headless backend's URL (no SPA behind it).

    Returns normally — leaving the caller to BIND — when no owner answers.
    """
    if getattr(args, "isolated", False) or _is_desktop_owned_backend():
        return
    record = _host_backend_attachment()
    if record is None:
        return

    from gateway import host_rendezvous as hr
    from gateway.restart import GATEWAY_FATAL_CONFIG_EXIT_CODE

    identity = hr.probe_owner(record)
    if identity is None:
        # Unprovable liveness (no psutil), a closed port, a foreign listener: all mean "no owner
        # answered". Fall through to the bind — never exit 0 on an attach that did not happen.
        return

    typed = _explicit_endpoint_flags()
    conflict = _endpoint_conflict(args, record, typed)
    if conflict:
        print(f"Refusing to start: this host is already served by {hr.describe(record)}.")
        print(f"  You asked for {conflict}.")
        print("  Stop that backend, or drop the flag to use the running one.")
        sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)

    if not headless_backend and not identity.get("servesSpa"):
        print(f"Refusing to start: this host is already served by {hr.describe(record)}, "
              "which is a headless `hermes serve` backend with no dashboard UI.")
        print("  Stop it and run `hermes dashboard`, or use --isolated for a dedicated server.")
        sys.exit(GATEWAY_FATAL_CONFIG_EXIT_CODE)

    try:
        from hermes_cli.profiles import get_active_profile_name
        profile = get_active_profile_name()
    except Exception:
        profile = "default"
    wanted = getattr(args, "open_profile", "") or profile
    url = f"http://{hr.dial_host(record)}:{record.port}/?profile={wanted}"

    kind = "backend" if headless_backend else "dashboard"
    print(f"Hermes {kind} already running on this host: PID {record.pid}, port {record.port}.")
    print(f"  Managing profile '{wanted}': {url}")
    if not headless_backend and not args.no_open:
        with contextlib.suppress(Exception):
            import webbrowser
            webbrowser.open(url)
    sys.exit(0)


def _route_named_profile_dashboard(
    args, _headless_backend: bool, _ssh_owner_nonce: str, _token_file: str) -> None:
    """Route a named-profile launch to the single MACHINE dashboard (per-request ``?profile=`` scoping
    makes one server per profile pure fragmentation).

    No-record fallback to :func:`_attach_to_host_backend`, which already attached (and exited)
    when the host publishes a live rendezvous record: re-exec pinned to ``-p default`` (so
    ``_apply_profile_override`` can't re-route via the sticky active_profile file). ``--isolated``
    opts out; Desktop pool backends (HERMES_DESKTOP=1) stay per-profile. Returns normally when no
    routing applies.
    """
    try:
        from hermes_cli.profiles import get_active_profile_name
        _launch_profile = get_active_profile_name()
    except Exception:
        _launch_profile = "default"

    if (
        _launch_profile in ("default", "custom")
        or getattr(args, "isolated", False)
        or getattr(args, "open_profile", "")
        or _is_desktop_owned_backend()
    ):
        return

    print(
        f"Routing to the machine dashboard (profile '{_launch_profile}' "
        f"preselected). Use --isolated for a dedicated per-profile server."
    )
    reexec_argv = [
        sys.executable, "-m", "hermes_cli.main",
        "-p", "default",
        # Preserve the lean serve path so a named-profile `serve` doesn't
        # silently rebuild the UI as `dashboard`.
        "serve" if _headless_backend else "dashboard",
        "--port", str(args.port),
        "--host", args.host,
        "--open-profile", _launch_profile]
    for enabled, extra in (
        (_ssh_owner_nonce, ["--ssh-owner-nonce", _ssh_owner_nonce]),
        (_token_file, ["--ssh-session-token-file", _token_file]),
        (args.no_open, ["--no-open"]),
        (getattr(args, "insecure", False), ["--insecure"]),
        (getattr(args, "skip_build", False), ["--skip-build"])):
        if enabled:
            reexec_argv.extend(extra)
    from tools.environments.local import build_subprocess_env
    # HERMES_HOME is pinned to the machine root below — the factory must not
    # re-inject a profile home.
    env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False)
    # Pin the child to the machine ROOT, resolved explicitly rather than by
    # dropping HERMES_HOME: in the Docker layout the root is /opt/data, and an
    # unset HERMES_HOME would fall back to $HOME/.hermes = /opt/data/.hermes — an
    # empty auto-seeded home with only the default profile and no install stamp.
    # get_default_hermes_root() strips a trailing profiles/<name> for both layouts.
    try:
        from hermes_constants import get_default_hermes_root
        env["HERMES_HOME"] = str(get_default_hermes_root())
    except Exception:
        env.pop("HERMES_HOME", None)  # prior behaviour rather than blocking the reroute
    # On Windows os.execvpe() spawns via CreateProcess then exits, which under
    # Python 3.14+ can crash with STATUS_ACCESS_VIOLATION; use Popen + exit.
    if sys.platform == "win32":
        proc = subprocess.Popen(reexec_argv, env=env)
        sys.exit(proc.wait())
    else:
        os.execvpe(sys.executable, reexec_argv, env)


def _resolve_dashboard_web_dist(args, _headless_backend: bool) -> None:
    """Build or validate the web UI dist before the server imports.

    ``serve`` sets HERMES_SERVE_HEADLESS so mount_spa() stays off. Otherwise build
    unless HERMES_WEB_DIST / --skip-build promise a dist — then verify index.html
    (else the server serves 404s). --skip-build on the default location gets ONE
    recovery build; a caller-managed HERMES_WEB_DIST can't be populated.
    """
    from hermes_cli.main import PROJECT_ROOT
    from hermes_cli.main_web_build import _build_web_ui
    skip_build = getattr(args, "skip_build", False)
    if _headless_backend:
        os.environ["HERMES_SERVE_HEADLESS"] = "1"  # set before web_server import
    elif "HERMES_WEB_DIST" not in os.environ and not skip_build:
        if not _build_web_ui(PROJECT_ROOT / "web", fatal=True):
            sys.exit(1)
    elif skip_build:
        _dist_root = (
            # --build-mode skip trusts the caller to have pre-built the web UI. Verify the dist actually
            # exists; otherwise the server will start and serve 404s with no obvious cause (issue #23817).
            Path(os.environ["HERMES_WEB_DIST"])
            if "HERMES_WEB_DIST" in os.environ
            else PROJECT_ROOT / "hermes_cli" / "web_dist"
        )
        if not (_dist_root / "index.html").exists():
            # Only the default dist location is recoverable (desktop launches with
            # --build-mode skip after a wipe of web_dist); a custom HERMES_WEB_DIST
            # is a caller-managed directory the build cannot populate.
            # The caller promised a pre-built dist but there isn't one. Instead of hard-failing (issue
            # #59288 — desktop launches with --build-mode skip after a wipe of web_dist), warn and attempt
            # ONE recovery build through the normal build path.
            _recoverable = "HERMES_WEB_DIST" not in os.environ
            if _recoverable:
                print(f"⚠ --skip-build was passed but no web dist found at: {_dist_root}")
                print("  Attempting one recovery build of the web UI...")
                _build_web_ui(PROJECT_ROOT / "web", fatal=True)
            if not (_dist_root / "index.html").exists():
                print(f"✗ --skip-build was passed but no web dist found at: {_dist_root}")
                if _recoverable:
                    print("  The recovery build did not produce a usable dist.")
                print(_PRE_BUILD_HINT)
                print("  Or drop --skip-build to build automatically.")
                sys.exit(1)
            print("  ✓ Recovery build produced a web dist")
        print(f"→ Skipping web UI build (--skip-build); using dist at {_dist_root}")
    else:
        # HERMES_WEB_DIST without --skip-build: the env var points at a
        # caller-managed dist, so validate it like the --skip-build branch.
        # HERMES_WEB_DIST is set without --skip-build: the build is skipped (the env var points at a
        # caller-managed dist), so validate it the same way the --skip-build branch does — otherwise the
        # server starts and serves 404s with no obvious cause (same failure mode as #23817, via the env-var
        # path).
        _dist_root = Path(os.environ["HERMES_WEB_DIST"]).expanduser()
        if not (_dist_root / "index.html").exists():
            print(f"✗ HERMES_WEB_DIST is set but no web dist found at: {_dist_root}")
            print(_PRE_BUILD_HINT)
            print("  Or unset HERMES_WEB_DIST to build and use the default web UI dist.")
            sys.exit(1)
        # web_server reads HERMES_WEB_DIST raw at import (no expanduser), so a
        # validated "~/dist" would otherwise pass here and still 404 there.
        os.environ["HERMES_WEB_DIST"] = str(_dist_root)
        print(f"→ Using web dist from HERMES_WEB_DIST: {_dist_root}")