test(update): hostile cells + docs for the POSIX pre-swap pause; leave unprivileged system units running

hermes_cli/update_cmd_posix_pause.py

@@ -244,7 +244,10 @@ def _left_running(units: list[dict], jobs: list[dict]) -> tuple[list[dict], list
     from hermes_cli.gateway import _is_pid_ancestor_of_current_process
     keep_units, keep_jobs, notices = [], [], []
     for unit in units:
-        if _escape_cgroup(unit):
+        if _manage_cmd(unit["scope"]) is None:
+            notices.append(f"  ↷ {unit['unit']} keeps running until after the update (stopping a system "
+                           "unit needs root or passwordless sudo)")
+        elif _escape_cgroup(unit):
             keep_units.append(unit)
         else:
             notices.append(f"  ↷ {unit['unit']} keeps running until after the update (this update runs "

tests/e2e/core/upgrade/git/test_hostile_pause.py

@@ -20,6 +20,16 @@ and the durable pause record ``update_pause_record`` keeps in the root hermes ho
     a toggle exists; the update tree is SIGKILLed once HEAD is the release and the gateway is
     paused. The next launch syncs the dependencies and restarts exactly the paused gateway on the
     new commit.
+(g) the dependency sync fails after the commit (contract amendment A6): the release's dependency
+    preparation raises while a toggle exists and changes ``uv.lock``'s bytes (the dependency stamp),
+    so the venv is stale at the new HEAD. The update exits 0 with a ``dependencies`` follow-up and
+    the paused gateway stays stopped (the tree gate holds it: it would boot new code on stale
+    dependencies). A ``hermes gateway run`` launched then (the replayed argv) syncs the owed
+    dependencies in its own launch before serving; the next ``hermes`` launch adopts that gateway
+    instead of starting a twin, and the record is discharged.
+(h) the Desktop hand-off (``scripts/desktop-update/posix.sh``, what the Desktop app spawns before it
+    quits) runs the same ``hermes update --gateway``: the gateway is paused and restarted on the new
+    commit, and the hand-off's result file reports success.
 """
 
 from __future__ import annotations
@@ -52,6 +62,13 @@ SLOW_DEPS = ".e2e-slow-deps-sync"
 DEPS_STARTED = ".e2e-deps-sync-started"
 UPDATE_TIMEOUT = 1500
 _DEPS_MARK = "# e2e (test_hostile_pause): slow dependency sync toggle"
+BREAK_DEPS = ".e2e-break-deps-sync"
+DEPS_OWED = "dependencies not installed yet"  # update_completion._settle_after_commit
+_BREAK_MARK = "# e2e (test_hostile_pause): failing dependency sync toggle"
+_BREAK_DEPS = f"""    {_BREAK_MARK}
+    if __import__("os").path.exists(__import__("os").path.join(__import__("os").path.expanduser("~"), "{BREAK_DEPS}")):
+        raise RuntimeError("e2e: dependency sync fails on purpose ({BREAK_DEPS})")
+"""
 _SLOW_DEPS = f"""    {_DEPS_MARK}
     _e2e_home = __import__("os").path.expanduser("~")
     if __import__("os").path.exists(__import__("os").path.join(_e2e_home, "{SLOW_DEPS}")):
@@ -231,11 +248,26 @@ def _publish_slow_deps_release(w: G.World, tag: str) -> str:
         "pm/client.py": text, f"docs/e2e-hostile-pause-{tag}.txt": f"release {tag}\n"})
 
 
+def _publish_broken_deps_release(w: G.World, tag: str) -> str:
+    text = w.git("show", "HEAD:pm/client.py") + "\n"
+    if _BREAK_MARK not in text:
+        m = re.search(r"^def ensure_tools_for_sync\(\) -> None:\n    \"\"\".*?\"\"\"\n", text, re.S | re.M)
+        assert m, "premise: pm/client.py has no ensure_tools_for_sync() with a docstring to inject after"
+        text = text[:m.end()] + _BREAK_DEPS + text[m.end():]
+    # pm's dependency stamp hashes uv.lock's bytes: a trailing TOML comment makes the venv stale
+    # at the new HEAD without changing what any dependency resolves to.
+    lock = w.git("show", "HEAD:uv.lock") + f"\n# e2e hostile pause {tag}: dependency stamp moves\n"
+    return w.publish(f"release: e2e hostile pause broken deps {tag}", {
+        "pm/client.py": text, "uv.lock": lock, f"docs/e2e-hostile-pause-{tag}.txt": f"release {tag}\n"})
+
+
 def _diag(w: G.World, host: NamespaceHost, cp=None, **extra) -> str:
     parts = [f"{k}={v}" for k, v in extra.items()]
     parts.append("pause records: " + json.dumps(_record(w), default=str)[:3000])
     parts.append("--- sandbox ---\n" + host.ps_text())
-    return "\n".join(parts) + "\n" + w.diag(cp)
+    text = "\n".join(parts) + "\n" + w.diag(cp)
+    print(text[:12000])  # the cell's receipt: ``-rA`` shows it for passing cells too
+    return text
 
 
 # -- cells -----------------------------------------------------------------------------------------
@@ -404,3 +436,78 @@ def test_sigkill_after_commit_before_restart_next_launch_restarts_on_new_code(w)
         assert not gw_alive_at_kill, f"the gateway still ran old code while HEAD named the release:\n{diag}"
         assert served, f"the next launch did not restart the paused gateway on the release:\n{diag}"
         assert int(served["pid"]) != gw_pid and len(gws) == 1, f"lost or duplicated gateway:\n{diag}"
+
+
+def test_deps_failure_after_commit_holds_the_paused_set_until_a_launch_syncs(w):
+    w.reset_clean()
+    toggle = w.sb.home / BREAK_DEPS
+    with _namespace(w) as host:
+        before = _start_gateway(host, w, "g")
+        gw_pid = int(before["pid"])
+        target = _publish_broken_deps_release(w, "g")
+        toggle.write_text("on\n", encoding="utf-8")
+        try:
+            cp = _ns_cli(host, w, "update", "--yes", "--branch", "main", timeout=UPDATE_TIMEOUT)
+        finally:
+            toggle.unlink(missing_ok=True)
+        out = G.output(cp)
+        held = _gateways(host)
+        rec_after_update = _record(w)
+        diag = _diag(w, host, cp, before=before, gateways_after_update=held)
+        assert w.head() == target, f"premise: the release never committed:\n{diag}"
+        assert DEPS_OWED in out, f"premise: the dependency sync did not fail after the commit:\n{diag}"
+        assert cp.returncode == 0, f"A6: a dependency failure after the commit failed the update:\n{diag}"
+        assert PAUSED in out, f"premise: the gateway was not paused:\n{diag}"
+        assert not held, f"a paused gateway was started on the new code with stale dependencies:\n{diag}"
+        assert any((r.get("token") or {}).get("resume_needed") for r in rec_after_update), \
+            f"the held set is not owed in the durable record:\n{diag}"
+
+        # The replayed argv's own launch: does it sync the owed dependencies before importing?
+        log = w.sb.root / "gateway-g2.log"
+        host.spawn([w.sb.hermes, "gateway", "run"], log=log)
+        served = None
+        with contextlib.suppress(AssertionError):
+            served = H.wait_for(lambda: _serving(w, target), timeout=600, interval=1.0, what="a gateway on the release")
+        launch_log = _read(log)
+        status = _ns_cli(host, w, "status", timeout=600)
+        time.sleep(3.0)
+        gws = _gateways(host)
+        diag = _diag(w, host, None, served=served, gateways=gws, launch_log=launch_log[-4000:],
+                     status=G.output(status)[-3000:])
+        assert served and int(served["pid"]) != gw_pid, f"the gateway's own launch did not come up on the release:\n{diag}"
+        assert len(gws) == 1, f"the next launch started a twin beside the running gateway:\n{diag}"
+        assert not [r for r in _record(w) if (r.get("token") or {}).get("resume_needed")], \
+            f"the record still owes a gateway that is serving:\n{diag}"
+
+
+def test_desktop_handoff_update_pauses_and_restarts_through_the_same_path(w):
+    w.reset_clean()
+    with _namespace(w) as host:
+        before = _start_gateway(host, w, "h")
+        old_pid = int(before["pid"])
+        target = _publish_release(w, "h")
+        result = w.sb.hermes_home / ".hermes-update-result.json"
+        result.unlink(missing_ok=True)
+        log = w.sb.hermes_home / "logs" / "desktop-update-handoff.log"
+        log0 = len(_read(log))
+        with _Timeline(host, w, old_pid) as tl:
+            cp = host.run(["bash", str(w.checkout / "scripts" / "desktop-update" / "posix.sh"),
+                           "--install-root", str(w.checkout), "--desktop-pid", "0", "--no-ui"],
+                          timeout=UPDATE_TIMEOUT, cwd=w.checkout)
+        handoff = _read(log)[log0:]
+        res = {}
+        with contextlib.suppress(OSError, ValueError):
+            res = json.loads(result.read_text(encoding="utf-8"))
+        after = None
+        with contextlib.suppress(AssertionError):
+            after = H.wait_for(lambda: _serving(w, target), timeout=60, interval=1.0, what="a gateway on the release")
+        gws = _gateways(host)
+        diag = _diag(w, host, cp, timeline="\n" + tl.text(), result=res, after=after, gateways=gws,
+                     handoff_log=handoff[-6000:])
+        assert w.head() == target, f"premise: the hand-off did not land the release:\n{diag}"
+        assert res.get("ok") is True, f"the hand-off result is not a success:\n{diag}"
+        assert PAUSED in handoff and RESTARTED in handoff, f"the hand-off update did not pause/restart:\n{diag}"
+        assert tl.first(lambda alive, head: alive and head == target) is None, \
+            f"the OLD gateway process was alive while HEAD named the release:\n{diag}"
+        assert after and int(after["pid"]) != old_pid and len(gws) == 1, \
+            f"not exactly one NEW gateway on the release after the hand-off:\n{diag}"

tests/hermes_cli/test_update_posix_pause.py

@@ -0,0 +1,103 @@
+"""POSIX pre-swap gateway pause: what survives a killed updater, and what a paused update asks.
+
+Real processes and a real record; the live stop/restart of real gateways is proven by
+``tests/e2e/core/upgrade/git/test_hostile_pause.py`` (a real ``hermes update`` in a sandbox).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import plistlib
+import signal
+import subprocess
+import sys
+import threading
+import time
+
+import pytest
+
+from tests.hermes_cli.test_update_pause_record import _child, _reap_children  # noqa: F401 - autouse reaper
+
+pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the POSIX pause; Windows has its own")
+
+_UNIT = {"kind": "systemd", "scope": "user", "unit": "hermes-gateway-p2probe.service", "pid": 4242}
+
+
+@pytest.mark.live_system_guard_bypass
+def test_a_killed_updaters_supervised_units_are_adopted_by_the_next_update(tmp_path):
+    """The updater stopped a systemd unit and was SIGKILLed: the next ``hermes update`` must own
+    restarting that unit (the record's only debt), never drop it."""
+    owner = _child("""
+        import time
+        from hermes_cli import update_pause_record as r
+        r.write(r.stamp_tree({"platform": "posix", "resume_needed": True, "posix_units": [%r]}),
+                owner=r.identity())
+        print("written", flush=True)
+        time.sleep(120)
+    """ % _UNIT, env={"HERMES_HOME": str(tmp_path)})
+    assert owner.stdout.readline().strip() == "written"
+    owner.send_signal(signal.SIGKILL)  # windows-footgun: ok — module skips on Windows
+    owner.wait(timeout=10)
+
+    nxt = _child("""
+        import json
+        from hermes_cli import update_pause_record as r
+        adopted, claims = r.adopt_orphans()
+        token = r.record_pause({"platform": "posix", "resume_needed": True, "unmapped": []}, adopted, claims)
+        print(json.dumps({"units": token.get("posix_units"), "platform": token.get("platform")}))
+    """, env={"HERMES_HOME": str(tmp_path)})
+    out, _ = nxt.communicate(timeout=60)
+    got = json.loads(out.strip().splitlines()[-1])
+    assert got == {"units": [_UNIT], "platform": "posix"}, f"the adopted unit was lost: {got}"
+
+
+def test_a_prompt_while_the_gateways_are_paused_takes_its_default_at_once(tmp_path, monkeypatch):
+    """``/update`` relays prompts through the gateway; while this update holds it stopped nobody can
+    answer, so the prompt must not park the update (and the downtime) for its 300 s timeout."""
+    from hermes_cli import update_cmd, update_cmd_posix_pause
+
+    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+    monkeypatch.setattr(update_cmd_posix_pause, "_RUN", {"platform": "posix", "posix_stopped": True, "resume_needed": True})
+    got: list[str] = []
+    t = threading.Thread(target=lambda: got.append(update_cmd._gateway_prompt("Restore local changes now?", "n")),
+                         daemon=True)
+    t.start()
+    t.join(timeout=10)
+    assert got == ["n"], "a prompt waited for a gateway this update had stopped"
+    assert not (tmp_path / ".update_prompt.json").exists()
+
+
+@pytest.mark.platforms("macos")
+@pytest.mark.live_system_guard_bypass
+def test_a_launchd_job_is_paused_through_launchd_and_restarted_through_it(tmp_path):
+    """A KeepAlive job merely killed is respawned by launchd on the old code mid-update; the pause
+    boots it out (no respawn) and the restart bootstraps the same plist (a fresh PID)."""
+    from hermes_cli.gateway import _gw
+    from hermes_cli.update_cmd_posix_pause import _alive, _start_job, _stop_job
+    label = f"ai.hermes.p2probe-{os.getpid()}"
+    plist = tmp_path / f"{label}.plist"
+    plist.write_bytes(plistlib.dumps({"Label": label, "ProgramArguments": ["/bin/sleep", "600"],
+                                      "RunAtLoad": True, "KeepAlive": True}))
+    domain = f"gui/{os.getuid()}"
+    if subprocess.run(["launchctl", "print", domain], capture_output=True, check=False).returncode:
+        domain = f"user/{os.getuid()}"
+
+    def pid() -> int | None:
+        return _gw()._launchd_print_service_pid(domain, label)[1]
+
+    subprocess.run(["launchctl", "bootstrap", domain, str(plist)], check=True, timeout=30)
+    try:
+        deadline = time.monotonic() + 15
+        while not pid() and time.monotonic() < deadline:
+            time.sleep(0.2)
+        first = pid()
+        assert first, "premise: launchd never started the probe job"
+        job = {"kind": "launchd", "label": label, "domain": domain, "plist": str(plist), "pid": first}
+        _stop_job(job)
+        time.sleep(3.0)  # KeepAlive would have respawned a merely-killed job by now
+        assert not _alive(first, None) and not pid(), "the paused job is running again under launchd"
+        _start_job(job)
+        assert pid() and pid() != first, "the restart did not bring the job back under launchd"
+    finally:
+        subprocess.run(["launchctl", "bootout", f"{domain}/{label}"], capture_output=True, check=False)

website/docs/getting-started/updating.md

@@ -93,14 +93,27 @@ This suppresses both cached update notices and passive update-check network requ
 For an admitted source checkout, `hermes update` runs these phases:
 
 1. **Pre-update snapshot** — Hermes saves selected state files for every profile in that profile's `state-snapshots/` directory. These include pairing data, cron jobs, `config.yaml`, `.env`, and `auth.json`. Automatic quick snapshots skip individual files larger than 1 GiB. `updates.pre_update_backup` selects `quick`, `full`, or `off`. Full archives use the [backup exclusions](../reference/faq.md#hermes-backup-vs-hermes-profile-export). Recovery uses [Snapshots and rollback](../user-guide/checkpoints-and-rollback.md). Quick snapshots recover state files, not application code. The snapshot is best-effort: if it fails, the update prints a `⚠ Pre-update snapshot FAILED` warning and continues, and the receipt records `pre_update_backup` as a failed step (a deliberate `off`/`--no-backup` lands in the receipt's skips with its reason instead).
-2. **Code update** — applies the configured source branch or stable release tag and updates submodules.
+2. **Code update** — applies the configured source branch or stable release tag and updates submodules. On Linux and macOS every gateway running from this install is paused immediately before the first change to the checkout (see [Gateway downtime during an update](#gateway-downtime-during-an-update)); an update that has nothing to apply, or that fails before it changes the checkout, stops nothing.
 3. **Post-pull syntax validation + auto-rollback** — after the pull, Hermes compiles the nine critical files every `hermes` invocation imports at startup. If any fails to parse (e.g. an orphan merge-conflict marker, an accidentally truncated file), Hermes rolls the install back so your shell stays bootable: it resets the checkout to the commit it ran before the pull (`git reset --hard <pre-pull-sha>`), or, when the update switched away from a detached checkout or a parked feature branch, returns there with `git checkout --detach <sha>` / `git checkout <branch>` (a branch it cannot check out again is restored detached at the same commit). Re-run `hermes update` once the upstream fix lands.
 4. **Dependency preparation** — PM provisions required tools and prepares a complete Python environment from the new lock, existing extras, and enabled plugin requirements. It validates that environment before publishing its selection. A plugin never fails the update. A plugin that no longer fits the new core is added to `plugins.disabled` in every profile that enables it (a memory provider has `memory.provider` cleared). That covers a `requires-python` that excludes Hermes's Python, a `manifest_version` newer than this Hermes supports, and dependencies that the resolver proves can't resolve alongside core and earlier plugins in config order, or that fail their own build. A download or network failure gets one retry and is disabled if it fails again. The update prints `⚠ Disabled plugin '<name>' in <home>: <reason>`, records it in the receipt's warnings, and continues. Re-enable it with `hermes plugins enable <name>` once the plugin ships a compatible release, or once the network is back. A `requires_hermes` range the running version misses never disables a plugin, because a source checkout without release tags can read as an older release. That plugin sits out instead (`⚠ Left plugin '<name>' … out of this update`), stays enabled, and rejoins once Hermes reports a version it accepts. A secondary profile whose config cannot be read sits out the same way until the config is fixed. Only a core that cannot build on its own fails this step.
 5. **Config migration** — detects new config options added since your version and prompts you to set them
 6. **Desktop rebuild (stage-and-swap)** — if the Hermes Desktop app was built from this checkout, it is rebuilt so the GUI matches the new code. The rebuild packs into a temporary staging directory next to `apps/desktop/release/`, verifies the staged app, and only then renames it over the previous build (on Windows a real-time scanner briefly holding `release/win-unpacked` is ridden out with a few short retries). A rebuild that fails at any point — corrupt Electron download, missing dependency, disk full — leaves the previous app untouched and launchable; the update prints a `⚠` line, still finishes the remaining steps (config migration, gateway restart) and exits 0 with a `build` follow-up in its receipt, and the next launch, `hermes update` or `hermes desktop --build-only --force-build` retries the rebuild. On macOS the rebuilt bundle is then copied (with `ditto`, signature intact) over a stale `/Applications/Hermes.app` or `~/Applications/Hermes.app`, so the copy Finder and the Dock launch matches the backend; an installed copy that is currently running is left alone and the update tells you to quit it and run `hermes update` again. If a copy an earlier update installed has since been removed (including dragged to the Trash), the update rebuilds it if needed, puts it back and says so, so Finder, the Dock and Spotlight find Hermes again; `hermes uninstall` removes the app for good. On Windows, when the update finishes from inside the Desktop app it would rebuild (the app completing an interrupted update at launch), the rebuild is skipped with a notice instead: Windows locks a running app's files, and stopping the app would end the update with it. The rest of the update completes; quit Hermes Desktop and run `hermes desktop` from a terminal, or use **Update now** in **Settings → About**, to rebuild and reopen it.
-7. **Gateway auto-restart**: running gateways are refreshed after the update completes. Service-managed gateways (systemd on Linux, launchd on macOS) restart through the service manager. Manual gateways are relaunched when Hermes can map their PID to a profile. Manually launched `hermes serve` / `hermes dashboard` backends are different: the updater leaves them running and asks their owner to restart them. See [Manual backend restart reminders](#manual-backend-restart-reminders). Backends owned by a running Desktop app remain the app's responsibility.
+7. **Gateway auto-restart**: gateways paused for the update (Linux, macOS and Windows) are started again as soon as the dependencies are prepared and the launchers refreshed, before the product builds; any other running gateway is refreshed after the update completes. Service-managed gateways (systemd on Linux, launchd on macOS) restart through the service manager. Manual gateways are relaunched when Hermes can map their PID to a profile. Manually launched `hermes serve` / `hermes dashboard` backends are different: the updater leaves them running and asks their owner to restart them. See [Manual backend restart reminders](#manual-backend-restart-reminders). Backends owned by a running Desktop app remain the app's responsibility.
 8. **Multiplex migration (multi-profile installs)** — once the fleet is verified on the new code, an install with two or more profiles that still run **one gateway per profile** is folded into a single multiplexed default gateway when nothing blocks it (same as `hermes gateway migrate --multiplex --yes`); if a blocker exists (a bot token shared by two profiles, a secondary profile binding a port with no `/p/<profile>/` ingress) the update prints the blockers with their fixes and changes nothing. Single-profile installs are never touched. See [Migrating from per-profile gateways](../user-guide/multi-profile-gateways.md#migrating-from-per-profile-gateways).
 
+### Gateway downtime during an update
+
+On Linux and macOS, `hermes update` stops every gateway running from this install right before it first changes the checkout, and starts exactly that set again on the new code once the dependencies are prepared (before the web UI, TUI and Desktop builds, which can take minutes). Your bots are offline for that window: typically the git update plus the dependency sync. Before this, a gateway kept running old code while git rewrote the files under it, and any module it imported lazily in that window came from the new code.
+
+- The update prints one line naming what it paused (`⏸ Paused for the update: hermes-gateway.service`) and one when it is back (`▶ Restarted paused gateway(s): …`).
+- systemd units and launchd agents are stopped and started through systemd/launchd, so the service manager cannot restart them on old code mid-update. A gateway you started by hand (`hermes gateway run`) is drained and then started again in the background from the same command line and Hermes home; its output goes to `logs/gateway-update-resume.log` in that home instead of your terminal.
+- `/update` from chat survives the pause: when the update runs inside the gateway's own systemd unit, it first moves itself into a scope of its own. A prompt asked while the gateways are paused takes its default immediately, because no gateway can relay it.
+- The paused set is recorded in the Hermes home before the first stop. If the update is killed, the next `hermes` command (or `hermes update`) starts exactly those gateways again. A failed update that never changed the checkout restarts them on the old code. If the dependency sync fails after the checkout moved, they restart once a launch has synced the dependencies.
+- A gateway the updater cannot stop through its supervisor (an s6 or custom systemd unit, `--external-supervisor`, a launchd job that started this update) is left running and refreshed after the update, as before.
+- `--no-gateway-restart` stops nothing; you manage the gateways.
+
+Windows already paused its gateways before the update (they lock files the dependency sync replaces); they now restart at the same point.
+
 ### Why the gateway restart can take a while
 
 The restart is drain-first: the running gateway refuses new turns, then waits for in-flight work (chat turns, cron jobs, API runs) to finish before exiting, capped by `agent.restart_after_turn_timeout` (30 minutes by default) so a long-running job is never cut off mid-run. While that wait is in progress the updater prints, every 30 seconds, what the gateway is still holding for — for example:
@@ -306,7 +319,8 @@ An update launched *by* the gateway (a cron job, the Desktop updater, any automa
 child of the gateway process) cannot survive its own fleet restart: the gateway drains on
 `SIGUSR1` and systemd's `KillMode=mixed` then kills everything left in the cgroup, updater
 included. `hermes update --no-gateway-restart` runs the full pipeline (pull, dependencies,
-Node workspaces, web UI, maintenance) and skips only the restart and fleet verification. The
+Node workspaces, web UI, maintenance) and skips only the restart and fleet verification; on Linux
+and macOS it also skips the pause before the checkout changes, so nothing is stopped. The
 pending-restart marker is kept, so the next CLI start warns and the next normal `hermes update`
 (or `hermes gateway restart`) catches the fleet up. Pair it with a separate restart step, for
 example a timer 10–15 minutes after the update job. The receipt records the deferral; a stale