test(update): durable Windows pause — record ownership, whole-tree gate, claim adoption; wine2e cells
(cherry picked from commit fe97d553c4ee91916a617bb1f1c1eefeed0e6bf0) (cherry picked from commit f2adbee6548fdc4b96164718ae04ae10582d2ec6)
This commit is contained in:
Teknium
153
tests/e2e/core/windows_update/test_pause_durable.py
Normal file
153
tests/e2e/core/windows_update/test_pause_durable.py
Normal file
@@ -0,0 +1,153 @@
|
||||
"""A killed or failed ``hermes update`` never strands — or wrongly starts — a paused gateway.
|
||||
|
||||
Failure class: Windows pause durability. ``hermes update`` stops every running gateway before it
|
||||
touches the checkout. Cell (a): the updater is killed (``taskkill /F /T`` — console closed, Desktop
|
||||
kill) right after the pause; the next plain ``hermes`` command must bring the gateway back.
|
||||
Cell (b): the pull fails half-way (a held file makes git's fast-forward die after writing part of
|
||||
the tree); the gateway must NOT be started onto that torn tree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from tests.e2e.core.windows_update._machine import (
|
||||
REQUIRES_OPT_IN,
|
||||
fail_with,
|
||||
harness_git,
|
||||
new_machine,
|
||||
)
|
||||
from tests.fakes.fake_llm_provider import FakeLLMServer
|
||||
|
||||
pytestmark = [pytest.mark.platforms("windows"), pytest.mark.integration,
|
||||
pytest.mark.live_system_guard_bypass, REQUIRES_OPT_IN]
|
||||
|
||||
_PAUSED = "Paused gateway profile"
|
||||
_RECORD = ".hermes-update-paused-gateways.json"
|
||||
_EARLY, _HELD = "AGENTS.md", "website/package.json" # checkout order: the early file is written first
|
||||
|
||||
|
||||
def _alive(pid: int) -> bool:
|
||||
try:
|
||||
return psutil.Process(pid).is_running() and psutil.Process(pid).status() != psutil.STATUS_ZOMBIE
|
||||
except psutil.Error:
|
||||
return False
|
||||
|
||||
|
||||
def _running_gateway(machine, not_pid: int, timeout: float) -> dict | None:
|
||||
try:
|
||||
return machine.wait_gateway_running(not_pid=not_pid, timeout=timeout)
|
||||
except AssertionError:
|
||||
return None
|
||||
|
||||
|
||||
def _update_killed_after_pause(machine) -> tuple[bool, str]:
|
||||
"""Start ``hermes update`` and ``taskkill /F /T`` it the moment it reports the pause."""
|
||||
log = machine.logs / "update-killed.log"
|
||||
proc = subprocess.Popen([str(machine.hermes_exe), "update", "--yes"], cwd=machine.profile,
|
||||
env=machine.env(), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
seen = threading.Event()
|
||||
lines: list[str] = []
|
||||
assert proc.stdout is not None
|
||||
|
||||
def pump() -> None:
|
||||
with log.open("w", encoding="utf-8") as fh:
|
||||
for raw in iter(proc.stdout.readline, b""):
|
||||
text = raw.decode("utf-8", "replace").rstrip()
|
||||
lines.append(text)
|
||||
fh.write(text + "\n")
|
||||
fh.flush()
|
||||
if _PAUSED in text:
|
||||
seen.set()
|
||||
|
||||
threading.Thread(target=pump, daemon=True).start()
|
||||
deadline = time.monotonic() + 600
|
||||
while not seen.is_set() and proc.poll() is None and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)
|
||||
proc.wait(timeout=60)
|
||||
return seen.is_set(), "\n".join(lines[-40:])
|
||||
|
||||
|
||||
def _mint_two_file_change(machine) -> str:
|
||||
"""A target commit on top of the installed HEAD that rewrites two existing tracked files."""
|
||||
base = machine.installed_head()
|
||||
index = machine.root / "torn.index"
|
||||
env = {"GIT_INDEX_FILE": str(index), "GIT_AUTHOR_NAME": "Hermes E2E", "GIT_AUTHOR_EMAIL": "e2e@hermes.invalid",
|
||||
"GIT_COMMITTER_NAME": "Hermes E2E", "GIT_COMMITTER_EMAIL": "e2e@hermes.invalid"}
|
||||
harness_git("-C", str(machine.serve), "read-tree", base, env=env)
|
||||
for path in (_EARLY, _HELD):
|
||||
src = machine.root / ("torn-" + path.replace("/", "_"))
|
||||
src.write_bytes((machine.install_dir / path).read_bytes() + b"\n")
|
||||
blob = harness_git("-C", str(machine.serve), "hash-object", "-w", "--no-filters", str(src))
|
||||
harness_git("-C", str(machine.serve), "update-index", "--cacheinfo", f"100644,{blob},{path}", env=env)
|
||||
tree = harness_git("-C", str(machine.serve), "write-tree", env=env)
|
||||
index.unlink(missing_ok=True)
|
||||
target = harness_git("-C", str(machine.serve), "commit-tree", tree, "-p", base, "-m", "e2e: torn target", env=env)
|
||||
harness_git("-C", str(machine.serve), "update-ref", "refs/heads/main", target)
|
||||
return base
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def journey(tmp_path_factory):
|
||||
out: dict = {}
|
||||
with FakeLLMServer() as srv:
|
||||
machine = new_machine(tmp_path_factory.mktemp("pd"), srv.base_url, label="pd", system_git=True)
|
||||
out["machine"] = machine
|
||||
try:
|
||||
install = machine.install()
|
||||
assert install.returncode == 0, fail_with(machine, f"install.ps1 exited {install.returncode}", install)
|
||||
with machine.gateway_phase():
|
||||
# (a) kill the updater right after it paused a real gateway
|
||||
machine.spawn_gateway()
|
||||
old = int(machine.wait_gateway_running().get("pid") or 0)
|
||||
machine.advance()
|
||||
out["a_paused"], out["a_tail"] = _update_killed_after_pause(machine)
|
||||
out["a_old_stopped"] = not _alive(old)
|
||||
out["a_record"] = (machine.hermes_home / _RECORD).is_file()
|
||||
out["a_launch"] = machine.hermes("gateway", "status", label="next-launch")
|
||||
out["a_after"] = _running_gateway(machine, old, 120)
|
||||
machine.kill_owned()
|
||||
|
||||
# (b) the pull fails with a torn tree
|
||||
machine.spawn_gateway()
|
||||
old = int(machine.wait_gateway_running().get("pid") or 0)
|
||||
out["b_pre"] = _mint_two_file_change(machine)
|
||||
with (machine.install_dir / _HELD).open("rb"): # no FILE_SHARE_DELETE: git cannot replace it
|
||||
out["b_update"] = machine.hermes("update", "--yes", label="update-torn", timeout=900)
|
||||
out["b_head"] = machine.installed_head()
|
||||
out["b_status"] = harness_git("-C", str(machine.install_dir), "status", "--porcelain",
|
||||
"--untracked-files=no")
|
||||
out["b_after"] = _running_gateway(machine, old, 60)
|
||||
out["b_record"] = (machine.hermes_home / _RECORD).is_file()
|
||||
machine.kill_owned()
|
||||
yield out
|
||||
finally:
|
||||
machine.teardown()
|
||||
|
||||
|
||||
def test_killed_updater_gateway_resumed_by_next_launch(journey) -> None:
|
||||
m = journey["machine"]
|
||||
assert journey["a_paused"] and journey["a_old_stopped"], fail_with(
|
||||
m, f"premise: the update never paused the gateway before the kill\n{journey['a_tail']}")
|
||||
assert journey["a_after"] is not None, fail_with(
|
||||
m, "after `taskkill /F` of an updater that paused the gateway, the next `hermes` launch left it "
|
||||
f"stopped (pause record present after kill: {journey['a_record']})", journey["a_launch"])
|
||||
|
||||
|
||||
def test_failed_pull_does_not_start_gateway_on_torn_tree(journey) -> None:
|
||||
m, run = journey["machine"], journey["b_update"]
|
||||
torn = journey["b_head"] == journey["b_pre"] and _EARLY in journey["b_status"]
|
||||
assert run.returncode != 0 and torn, fail_with(
|
||||
m, f"premise: the held file did not tear the pull (rc={run.returncode}, HEAD={journey['b_head']}, "
|
||||
f"pre={journey['b_pre']}, status={journey['b_status']!r})", run)
|
||||
assert journey["b_after"] is None, fail_with(
|
||||
m, f"the gateway was restarted onto a torn checkout ({journey['b_status']!r}) after the failed update "
|
||||
f"(pid {journey['b_after'].get('pid') if journey['b_after'] else None})", run)
|
||||
assert journey["b_record"], fail_with(m, "the paused set was not kept for recovery after the torn pull", run)
|
||||
108
tests/hermes_cli/test_update_pause_record.py
Normal file
108
tests/hermes_cli/test_update_pause_record.py
Normal file
@@ -0,0 +1,108 @@
|
||||
"""Paused-gateway record and update-claim adoption, with real processes and a real checkout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli import update_pause_record as pause_record
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals; Windows cells live in wine2e")
|
||||
|
||||
|
||||
def _child(code: str, *argv: str, env: dict | None = None) -> subprocess.Popen:
|
||||
return subprocess.Popen([sys.executable, "-c", textwrap.dedent(code), *argv], cwd=REPO, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE, text=True, env={**os.environ, "PYTHONPATH": str(REPO), **(env or {})})
|
||||
|
||||
|
||||
def test_record_written_by_a_killed_updater_is_orphaned_only_after_its_death(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
owner = _child("""
|
||||
import sys, time
|
||||
from hermes_cli import update_pause_record as r
|
||||
r.write(r.stamp_tree({"resume_needed": True, "profiles": {"default": 4242}}), owner=r.identity())
|
||||
print("written", flush=True)
|
||||
time.sleep(120)
|
||||
""", env={"HERMES_HOME": str(tmp_path)})
|
||||
try:
|
||||
assert owner.stdout.readline().strip() == "written"
|
||||
body = pause_record.read()
|
||||
assert body["owner"]["pid"] == owner.pid and body["owner"]["ct"].startswith("ct:")
|
||||
assert body["token"]["profiles"] == {"default": 4242}
|
||||
assert pause_record.orphaned_record() is None # live owner: its own resume owns the set
|
||||
finally:
|
||||
owner.send_signal(signal.SIGKILL) # windows-footgun: ok — module skips on Windows
|
||||
owner.wait(timeout=10)
|
||||
orphan = pause_record.orphaned_record()
|
||||
assert orphan is not None and orphan["token"]["profiles"] == {"default": 4242}
|
||||
|
||||
|
||||
def _git(root: Path, *args: str) -> str:
|
||||
return subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, text=True, encoding="utf-8",
|
||||
env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}).stdout.strip()
|
||||
|
||||
|
||||
def test_resume_waits_for_a_whole_tree(tmp_path):
|
||||
root = tmp_path / "checkout"
|
||||
root.mkdir()
|
||||
_git(root, "init", "-q")
|
||||
for name in ("a.py", "b.py", "local.txt"):
|
||||
(root / name).write_text("v1\n", encoding="utf-8")
|
||||
_git(root, "add", ".")
|
||||
_git(root, "commit", "-qm", "v1")
|
||||
(root / "local.txt").write_text("user edit\n", encoding="utf-8") # dirty before the update: not git's doing
|
||||
token = pause_record.stamp_tree({"resume_needed": True}, root)
|
||||
assert pause_record.tree_is_whole(token, root) == (True, "")
|
||||
|
||||
(root / "a.py").write_text("v2\n", encoding="utf-8") # git wrote a.py, then died before b.py and HEAD
|
||||
whole, why = pause_record.tree_is_whole(token, root)
|
||||
assert not whole and "a.py" in why
|
||||
|
||||
(root / "a.py").write_text("v1\n", encoding="utf-8")
|
||||
marker = root / ".git" / "hermes-update-pull"
|
||||
marker.write_text("pid\n", encoding="utf-8")
|
||||
assert pause_record.tree_is_whole(token, root)[0] is False
|
||||
marker.unlink()
|
||||
|
||||
_git(root, "commit", "-qam", "v2") # HEAD moved: only a dependency sync for it makes it whole
|
||||
whole, why = pause_record.tree_is_whole(token, root)
|
||||
assert not whole and "dependencies" in why
|
||||
|
||||
|
||||
_HOLDER = """
|
||||
import subprocess, sys, time
|
||||
from hermes_cli.update_lock import UpdateLock
|
||||
lock = UpdateLock()
|
||||
assert lock.acquire() and lock.acquired
|
||||
host = subprocess.Popen([sys.executable, sys.argv[1], "hermes_cli.main", *sys.argv[2:]], stdout=subprocess.PIPE, text=True)
|
||||
print(host.stdout.read().strip(), flush=True)
|
||||
host.wait()
|
||||
"""
|
||||
# A stand-in for the host the update relaunched: its argv names the host command, and the
|
||||
# ``hermes update`` its agent starts is its child.
|
||||
_HOST = """
|
||||
import subprocess, sys
|
||||
probe = "from hermes_cli.update_lock import UpdateLock; l = UpdateLock(); print(l.acquire(), l.holder is not None)"
|
||||
print(subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True).stdout.strip())
|
||||
"""
|
||||
|
||||
|
||||
# The intermediate is a stand-in script, not a real gateway/update: nothing touches the checkout.
|
||||
@pytest.mark.live_system_guard_bypass
|
||||
@pytest.mark.parametrize("host_argv, adopts", [(("gateway", "run"), False), (("status",), True)])
|
||||
def test_update_started_from_a_relaunched_gateway_does_not_share_the_claim(tmp_path, host_argv, adopts):
|
||||
script = tmp_path / "host.py"
|
||||
script.write_text(_HOST, encoding="utf-8")
|
||||
holder = _child(_HOLDER, str(script), *host_argv, env={"HERMES_HOME": str(tmp_path)})
|
||||
out, _ = holder.communicate(timeout=60)
|
||||
assert holder.returncode == 0, out
|
||||
assert out.strip() == ("True False" if adopts else "False True"), out
|
||||
Reference in New Issue
Block a user