diff --git a/tests/e2e/core/windows_update/test_pause_durable.py b/tests/e2e/core/windows_update/test_pause_durable.py new file mode 100644 index 0000000000..2aed7efeb7 --- /dev/null +++ b/tests/e2e/core/windows_update/test_pause_durable.py @@ -0,0 +1,153 @@ +"""A killed or failed ``hermes update`` never strands — or wrongly starts — a paused gateway. + +Failure class: Windows pause durability. ``hermes update`` stops every running gateway before it +touches the checkout. Cell (a): the updater is killed (``taskkill /F /T`` — console closed, Desktop +kill) right after the pause; the next plain ``hermes`` command must bring the gateway back. +Cell (b): the pull fails half-way (a held file makes git's fast-forward die after writing part of +the tree); the gateway must NOT be started onto that torn tree. +""" + +from __future__ import annotations + +import subprocess +import threading +import time + +import psutil +import pytest + +from tests.e2e.core.windows_update._machine import ( + REQUIRES_OPT_IN, + fail_with, + harness_git, + new_machine, +) +from tests.fakes.fake_llm_provider import FakeLLMServer + +pytestmark = [pytest.mark.platforms("windows"), pytest.mark.integration, + pytest.mark.live_system_guard_bypass, REQUIRES_OPT_IN] + +_PAUSED = "Paused gateway profile" +_RECORD = ".hermes-update-paused-gateways.json" +_EARLY, _HELD = "AGENTS.md", "website/package.json" # checkout order: the early file is written first + + +def _alive(pid: int) -> bool: + try: + return psutil.Process(pid).is_running() and psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.Error: + return False + + +def _running_gateway(machine, not_pid: int, timeout: float) -> dict | None: + try: + return machine.wait_gateway_running(not_pid=not_pid, timeout=timeout) + except AssertionError: + return None + + +def _update_killed_after_pause(machine) -> tuple[bool, str]: + """Start ``hermes update`` and ``taskkill /F /T`` it the moment it reports the pause.""" + log = machine.logs / "update-killed.log" + proc = subprocess.Popen([str(machine.hermes_exe), "update", "--yes"], cwd=machine.profile, + env=machine.env(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + seen = threading.Event() + lines: list[str] = [] + assert proc.stdout is not None + + def pump() -> None: + with log.open("w", encoding="utf-8") as fh: + for raw in iter(proc.stdout.readline, b""): + text = raw.decode("utf-8", "replace").rstrip() + lines.append(text) + fh.write(text + "\n") + fh.flush() + if _PAUSED in text: + seen.set() + + threading.Thread(target=pump, daemon=True).start() + deadline = time.monotonic() + 600 + while not seen.is_set() and proc.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60) + proc.wait(timeout=60) + return seen.is_set(), "\n".join(lines[-40:]) + + +def _mint_two_file_change(machine) -> str: + """A target commit on top of the installed HEAD that rewrites two existing tracked files.""" + base = machine.installed_head() + index = machine.root / "torn.index" + env = {"GIT_INDEX_FILE": str(index), "GIT_AUTHOR_NAME": "Hermes E2E", "GIT_AUTHOR_EMAIL": "e2e@hermes.invalid", + "GIT_COMMITTER_NAME": "Hermes E2E", "GIT_COMMITTER_EMAIL": "e2e@hermes.invalid"} + harness_git("-C", str(machine.serve), "read-tree", base, env=env) + for path in (_EARLY, _HELD): + src = machine.root / ("torn-" + path.replace("/", "_")) + src.write_bytes((machine.install_dir / path).read_bytes() + b"\n") + blob = harness_git("-C", str(machine.serve), "hash-object", "-w", "--no-filters", str(src)) + harness_git("-C", str(machine.serve), "update-index", "--cacheinfo", f"100644,{blob},{path}", env=env) + tree = harness_git("-C", str(machine.serve), "write-tree", env=env) + index.unlink(missing_ok=True) + target = harness_git("-C", str(machine.serve), "commit-tree", tree, "-p", base, "-m", "e2e: torn target", env=env) + harness_git("-C", str(machine.serve), "update-ref", "refs/heads/main", target) + return base + + +@pytest.fixture(scope="module") +def journey(tmp_path_factory): + out: dict = {} + with FakeLLMServer() as srv: + machine = new_machine(tmp_path_factory.mktemp("pd"), srv.base_url, label="pd", system_git=True) + out["machine"] = machine + try: + install = machine.install() + assert install.returncode == 0, fail_with(machine, f"install.ps1 exited {install.returncode}", install) + with machine.gateway_phase(): + # (a) kill the updater right after it paused a real gateway + machine.spawn_gateway() + old = int(machine.wait_gateway_running().get("pid") or 0) + machine.advance() + out["a_paused"], out["a_tail"] = _update_killed_after_pause(machine) + out["a_old_stopped"] = not _alive(old) + out["a_record"] = (machine.hermes_home / _RECORD).is_file() + out["a_launch"] = machine.hermes("gateway", "status", label="next-launch") + out["a_after"] = _running_gateway(machine, old, 120) + machine.kill_owned() + + # (b) the pull fails with a torn tree + machine.spawn_gateway() + old = int(machine.wait_gateway_running().get("pid") or 0) + out["b_pre"] = _mint_two_file_change(machine) + with (machine.install_dir / _HELD).open("rb"): # no FILE_SHARE_DELETE: git cannot replace it + out["b_update"] = machine.hermes("update", "--yes", label="update-torn", timeout=900) + out["b_head"] = machine.installed_head() + out["b_status"] = harness_git("-C", str(machine.install_dir), "status", "--porcelain", + "--untracked-files=no") + out["b_after"] = _running_gateway(machine, old, 60) + out["b_record"] = (machine.hermes_home / _RECORD).is_file() + machine.kill_owned() + yield out + finally: + machine.teardown() + + +def test_killed_updater_gateway_resumed_by_next_launch(journey) -> None: + m = journey["machine"] + assert journey["a_paused"] and journey["a_old_stopped"], fail_with( + m, f"premise: the update never paused the gateway before the kill\n{journey['a_tail']}") + assert journey["a_after"] is not None, fail_with( + m, "after `taskkill /F` of an updater that paused the gateway, the next `hermes` launch left it " + f"stopped (pause record present after kill: {journey['a_record']})", journey["a_launch"]) + + +def test_failed_pull_does_not_start_gateway_on_torn_tree(journey) -> None: + m, run = journey["machine"], journey["b_update"] + torn = journey["b_head"] == journey["b_pre"] and _EARLY in journey["b_status"] + assert run.returncode != 0 and torn, fail_with( + m, f"premise: the held file did not tear the pull (rc={run.returncode}, HEAD={journey['b_head']}, " + f"pre={journey['b_pre']}, status={journey['b_status']!r})", run) + assert journey["b_after"] is None, fail_with( + m, f"the gateway was restarted onto a torn checkout ({journey['b_status']!r}) after the failed update " + f"(pid {journey['b_after'].get('pid') if journey['b_after'] else None})", run) + assert journey["b_record"], fail_with(m, "the paused set was not kept for recovery after the torn pull", run) diff --git a/tests/hermes_cli/test_update_pause_record.py b/tests/hermes_cli/test_update_pause_record.py new file mode 100644 index 0000000000..a60d2df716 --- /dev/null +++ b/tests/hermes_cli/test_update_pause_record.py @@ -0,0 +1,108 @@ +"""Paused-gateway record and update-claim adoption, with real processes and a real checkout.""" + +from __future__ import annotations + +import os +import signal +import subprocess +import sys +import textwrap +import time +from pathlib import Path + +import pytest + +from hermes_cli import update_pause_record as pause_record + +REPO = Path(__file__).resolve().parents[2] +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals; Windows cells live in wine2e") + + +def _child(code: str, *argv: str, env: dict | None = None) -> subprocess.Popen: + return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *argv], cwd=REPO, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, text=True, env={**os.environ, "PYTHONPATH": str(REPO), **(env or {})}) + + +def test_record_written_by_a_killed_updater_is_orphaned_only_after_its_death(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + owner = _child(""" + import sys, time + from hermes_cli import update_pause_record as r + r.write(r.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}), owner=r.identity()) + print("written", flush=True) + time.sleep(120) + """, env={"HERMES_HOME": str(tmp_path)}) + try: + assert owner.stdout.readline().strip() == "written" + body = pause_record.read() + assert body["owner"]["pid"] == owner.pid and body["owner"]["ct"].startswith("ct:") + assert body["token"]["profiles"] == {"default": 4242} + assert pause_record.orphaned_record() is None # live owner: its own resume owns the set + finally: + owner.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows + owner.wait(timeout=10) + orphan = pause_record.orphaned_record() + assert orphan is not None and orphan["token"]["profiles"] == {"default": 4242} + + +def _git(root: Path, *args: str) -> str: + return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True, encoding="utf-8", + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}).stdout.strip() + + +def test_resume_waits_for_a_whole_tree(tmp_path): + root = tmp_path / "checkout" + root.mkdir() + _git(root, "init", "-q") + for name in ("a.py", "b.py", "local.txt"): + (root / name).write_text("v1\n", encoding="utf-8") + _git(root, "add", ".") + _git(root, "commit", "-qm", "v1") + (root / "local.txt").write_text("user edit\n", encoding="utf-8") # dirty before the update: not git's doing + token = pause_record.stamp_tree({"resume_needed": True}, root) + assert pause_record.tree_is_whole(token, root) == (True, "") + + (root / "a.py").write_text("v2\n", encoding="utf-8") # git wrote a.py, then died before b.py and HEAD + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "a.py" in why + + (root / "a.py").write_text("v1\n", encoding="utf-8") + marker = root / ".git" / "hermes-update-pull" + marker.write_text("pid\n", encoding="utf-8") + assert pause_record.tree_is_whole(token, root)[0] is False + marker.unlink() + + _git(root, "commit", "-qam", "v2") # HEAD moved: only a dependency sync for it makes it whole + whole, why = pause_record.tree_is_whole(token, root) + assert not whole and "dependencies" in why + + +_HOLDER = """ + import subprocess, sys, time + from hermes_cli.update_lock import UpdateLock + lock = UpdateLock() + assert lock.acquire() and lock.acquired + host = subprocess.Popen([sys.executable, sys.argv[1], "hermes_cli.main", *sys.argv[2:]], stdout=subprocess.PIPE, text=True) + print(host.stdout.read().strip(), flush=True) + host.wait() +""" +# A stand-in for the host the update relaunched: its argv names the host command, and the +# ``hermes update`` its agent starts is its child. +_HOST = """ +import subprocess, sys +probe = "from hermes_cli.update_lock import UpdateLock; l = UpdateLock(); print(l.acquire(), l.holder is not None)" +print(subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True).stdout.strip()) +""" + + +# The intermediate is a stand-in script, not a real gateway/update: nothing touches the checkout. +@pytest.mark.live_system_guard_bypass +@pytest.mark.parametrize("host_argv, adopts", [(("gateway", "run"), False), (("status",), True)]) +def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_path, host_argv, adopts): + script = tmp_path / "host.py" + script.write_text(_HOST, encoding="utf-8") + holder = _child(_HOLDER, str(script), *host_argv, env={"HERMES_HOME": str(tmp_path)}) + out, _ = holder.communicate(timeout=60) + assert holder.returncode == 0, out + assert out.strip() == ("True False" if adopts else "False True"), out