Files
hermes-agent/tests/hermes_cli/test_update_lock_protocol.py
Teknium 29a0305834 fix(update): a launch claimer never reclaims a dead marker while the checkout lock is held (R6/M1)
venv_sync.prepare_launch and _prepare_borrowed_launch claimed the marker with
UpdateLock() (no install_root), and UpdateLock._claim_marker unlinked a dead
marker without asking whether the checkout lock was held. The claim then
failed acquire_checkout and released its own claim: the killed update's
marker was gone while its completion child still ran, and the Desktop gate
reopened over it.

- _claim_marker refuses as held (pid 0) and keeps the marker when the checkout
  lock is held by another process tree (our own hold or an inherited lock fd
  may still reclaim).
- UpdateLock(checkout_first=False): install_root names the guarding checkout
  without taking its lock first; both venv_sync sites pass install_root=root.
- _lock_holder reports a dead recorded updater over a held lock as held, so
  hermes update / launch repair / source completion refusals use the held
  wording instead of naming the dead pid (m6).
2026-10-06 04:36:02 -07:00

545 lines
22 KiB
Python

"""Marker mutation protocol (A7) and checkout custody (R2/R11) with real processes and files.
Each test reproduces a review finding on the pre-A7 head: a stale reader deleting a claim
published after its read (R3), a hand-off owner's killed claim surviving its orphaned delegate
(A7 rule 5, the native Windows ``orphaned_update`` artifact), git / the Node build writing a
checkout a contender already owns (R2), a completion from another home entering its build
while the checkout is owned (R2), and a live pre-root-marker profile claim going unseen (R11).
"""
from __future__ import annotations
import os
import signal
import subprocess
import sys
import textwrap
import time
from pathlib import Path
import pytest
from hermes_cli.update_lock import UpdateLock, marker_mutex_path, process_create_time
REPO_ROOT = Path(__file__).resolve().parents[2]
posix_only = pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals / fd inheritance")
def _script(tmp_path: Path, code: str) -> Path:
path = tmp_path / f"child_{abs(hash(code))}.py"
path.write_text(textwrap.dedent(code), encoding="utf-8")
return path
def _python(tmp_path: Path, code: str, *args, **kwargs) -> subprocess.Popen:
return subprocess.Popen([sys.executable, str(_script(tmp_path, code)), str(REPO_ROOT), *map(str, args)],
stdin=subprocess.DEVNULL, **kwargs)
def _wait_for(path: Path, proc: subprocess.Popen | None = None, timeout: float = 30.0) -> str:
end = time.monotonic() + timeout
while not path.exists():
assert proc is None or proc.poll() is None, f"child exited {proc.returncode} before {path.name}"
assert time.monotonic() < end, f"timed out waiting for {path}"
time.sleep(0.02)
return path.read_text(encoding="utf-8")
def _alive(pid: int) -> bool:
from hermes_cli.update_lock import _pid_alive
return _pid_alive(pid)
_STALE_READER = """
import linecache, os, signal, sys
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli import update_lock
src = update_lock.__file__
def trace(frame, event, arg):
if event == "line" and frame.f_code.co_filename == src \\
and ".unlink()" in linecache.getline(src, frame.f_lineno):
sys.settrace(None)
os.kill(os.getpid(), signal.SIGSTOP) # a scheduling pause, nothing replaced
return trace
sys.settrace(trace)
update_lock.read_live_update(path=Path(sys.argv[2]))
"""
_CLAIMANT = """
import sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import UpdateLock
assert UpdateLock(path=Path(sys.argv[2])).acquire()
Path(sys.argv[3]).touch()
time.sleep(60)
"""
@posix_only
def test_a_stale_reclaim_never_deletes_a_claim_published_after_its_read(tmp_path):
"""R3: the reader judged a dead marker, then paused right at its unlink. A second real
process claims meanwhile. Pre-A7 the resumed reader deleted that live claim; now the
decision and the unlink share one hold of the marker mutex, so the claimant waits."""
marker = tmp_path / ".hermes-update-in-progress"
marker.write_bytes(b"0\n0\n") # malformed: dead
reader = _python(tmp_path, _STALE_READER, marker)
claimant = None
try:
_pid, status = os.waitpid(reader.pid, os.WUNTRACED)
assert os.WIFSTOPPED(status)
claimant = _python(tmp_path, _CLAIMANT, marker, tmp_path / "claimed")
time.sleep(1.0)
os.kill(reader.pid, signal.SIGCONT)
assert reader.wait(timeout=30) == 0
_wait_for(tmp_path / "claimed", claimant)
assert marker.read_text(encoding="utf-8").startswith(f"{claimant.pid}\n"), \
"a stale reader deleted a live claim published after its read"
assert marker_mutex_path(marker).exists(), "the sidecar is never deleted"
finally:
if reader.poll() is None:
os.kill(reader.pid, signal.SIGCONT)
reader.kill()
reader.wait()
if claimant is not None:
claimant.kill()
claimant.wait()
_MUTEX_HOLDER = """
import sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import marker_mutex
with marker_mutex(Path(sys.argv[2])):
Path(sys.argv[3]).touch()
time.sleep(1.5)
"""
def test_marker_mutations_wait_for_the_mutex_another_process_holds(tmp_path):
"""A7 rule 1 on every OS (POSIX flock / Windows share-none open of ``<marker>.lock``):
while another process holds the marker mutex, a reclaim of even a dead marker waits."""
marker = tmp_path / ".hermes-update-in-progress"
marker.write_bytes(b"0\n0\n")
holder = _python(tmp_path, _MUTEX_HOLDER, marker, tmp_path / "held")
try:
_wait_for(tmp_path / "held", holder)
started = time.monotonic()
lock = UpdateLock(path=marker)
assert lock.acquire() and lock.acquired
assert time.monotonic() - started >= 0.9, "the reclaim ran inside another process's hold"
lock.release()
assert marker_mutex_path(marker).exists(), "the sidecar is never deleted"
finally:
holder.wait(timeout=30)
_ORPHAN_UPDATE = """
import sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import UpdateLock
base = Path(sys.argv[2])
while not (base / "delegate-written").exists(): time.sleep(0.02)
lock = UpdateLock(path=base / "marker")
assert lock.acquire(), lock.holder
(base / "ready").touch()
while not (base / "go").exists(): time.sleep(0.02)
lock.release()
"""
_HANDOFF_OWNER = """
import os, subprocess, sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import process_create_time
base = Path(sys.argv[2])
marker = base / "marker"
marker.write_text(f"{os.getpid()}\\n{int(time.time())}\\nct:{process_create_time():.3f}\\n")
child = subprocess.Popen([sys.executable, sys.argv[3], sys.argv[1], str(base)], start_new_session=True)
with open(marker, "a") as fh: # the hand-off scripts name their update child right after spawn
fh.write(f"delegate:{child.pid} ct:{process_create_time(child.pid):.3f}\\n")
(base / "delegate-written").write_text(str(child.pid))
child.wait()
"""
@posix_only
def test_an_orphaned_delegate_removes_its_killed_owners_marker(tmp_path):
"""A7 rule 5 / Windows ``orphaned_update``: the hand-off script claims, names its `hermes
update` child as delegate, and is killed alone. Pre-A7 the child adopted the claim without
writing it, so its exit released nothing: '<dead owner> … delegate:<exited child>' survived
(native artifact crash-12db). The delegate's release now removes a claim whose owner died."""
owner = _python(tmp_path, _HANDOFF_OWNER, tmp_path, _script(tmp_path, _ORPHAN_UPDATE))
try:
child = int(_wait_for(tmp_path / "delegate-written", owner))
_wait_for(tmp_path / "ready", owner)
owner.kill()
owner.wait()
(tmp_path / "go").touch()
end = time.monotonic() + 30
while _alive(child) and time.monotonic() < end:
time.sleep(0.05)
assert not _alive(child)
assert not (tmp_path / "marker").exists(), "the orphaned update's exit left its dead claim behind"
finally:
owner.kill()
owner.wait()
@posix_only
def test_owner_release_hands_the_claim_to_a_live_delegate(tmp_path):
"""A7 rule 5, the other half: the owner's release never deletes the claim a live delegate
still runs under; it rewrites it with that delegate as owner (and keeps the run line)."""
marker = tmp_path / "marker"
delegate = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], stdin=subprocess.DEVNULL)
try:
lock = UpdateLock(path=marker)
assert lock.acquire() and lock.acquired
ct = f"{process_create_time(delegate.pid):.3f}"
marker.write_bytes(marker.read_bytes() + f"delegate:{delegate.pid} ct:{ct}\nrun:desk-1\n".encode())
started = marker.read_text(encoding="utf-8").splitlines()[1]
lock.release()
assert marker.read_text(encoding="utf-8") == f"{delegate.pid}\n{started}\nct:{ct}\nrun:desk-1\n"
finally:
delegate.kill()
delegate.wait()
_GIT_OWNER = """
import sys
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import UpdateLock
from hermes_cli.update_cmd import _git_run
root = Path(sys.argv[2])
assert UpdateLock(path=root / "owner-marker", install_root=root).acquire()
_git_run(["git"], ["commit", "-qm", "after"], cwd=root, check=True)
"""
def _git(root: Path, *args: str) -> str:
env = {**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull}
return subprocess.check_output(["git", "-C", str(root), *args], text=True, env=env).strip()
@posix_only
def test_git_keeps_checkout_custody_after_its_updater_is_killed(tmp_path):
"""R2: `_git_run`'s git (and the hook it runs) inherits the checkout lock. Pre-fix the
killed updater's lock fell free while its git still committed, and a contender acquired."""
root = tmp_path / "checkout"
root.mkdir()
_git(root, "init", "-q")
_git(root, "config", "user.name", "probe")
_git(root, "config", "user.email", "probe@invalid.local")
(root / "tracked").write_text("before\n")
_git(root, "add", "tracked")
_git(root, "commit", "-qm", "before")
(root / "tracked").write_text("after\n")
_git(root, "add", "tracked")
hook = root / ".git/hooks/pre-commit"
hook.write_text(f"#!{sys.executable}\nimport os, time\nfrom pathlib import Path\nr = Path({str(tmp_path)!r})\n"
"(r / 'hook').write_text(str(os.getpid()))\n"
"while not (r / 'go').exists(): time.sleep(0.02)\n")
hook.chmod(0o700)
owner = _python(tmp_path, _GIT_OWNER, root, env={**os.environ, "GIT_CONFIG_NOSYSTEM": "1",
"GIT_CONFIG_GLOBAL": os.devnull})
contender = UpdateLock(path=tmp_path / "contender-marker", install_root=root)
try:
hook_pid = int(_wait_for(tmp_path / "hook", owner))
owner.kill()
owner.wait()
assert _alive(hook_pid)
assert contender.acquire() is False, "a contender owns the checkout while git still commits"
(tmp_path / "go").touch()
end = time.monotonic() + 30
while not contender.acquire():
assert time.monotonic() < end, "the checkout stayed locked after git exited"
time.sleep(0.05)
finally:
(tmp_path / "go").touch()
contender.release()
owner.kill()
owner.wait()
_BUILD_OWNER = """
import os, sys
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import UpdateLock
from hermes_cli.source_build import run_source_script
root = Path(sys.argv[2])
assert UpdateLock(path=root / "owner-marker", install_root=root).acquire()
run_source_script(root, "build.mjs", env={**os.environ, "PATH": sys.argv[3]}, label="probe build")
"""
@posix_only
def test_the_source_build_keeps_checkout_custody_after_its_updater_is_killed(tmp_path):
"""R2: the Node build `run_source_script` starts keeps the lock fd. A real `node` stand-in
(an executable on the build's PATH) outlives the killed updater; the checkout stays locked."""
root = tmp_path / "checkout"
(root / ".git").mkdir(parents=True)
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
node = bin_dir / "node"
node.write_text(f"#!{sys.executable}\nimport os, time\nfrom pathlib import Path\nr = Path({str(tmp_path)!r})\n"
"(r / 'node').write_text(str(os.getpid()))\n"
"while not (r / 'go').exists(): time.sleep(0.02)\n")
node.chmod(0o700)
owner = _python(tmp_path, _BUILD_OWNER, root, bin_dir)
contender = UpdateLock(path=tmp_path / "contender-marker", install_root=root)
try:
node_pid = int(_wait_for(tmp_path / "node", owner))
owner.kill()
owner.wait()
assert _alive(node_pid)
assert contender.acquire() is False, "a contender owns the checkout while the build still writes"
finally:
(tmp_path / "go").touch()
contender.release()
owner.kill()
owner.wait()
_HOLDER = """
import sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli.update_lock import UpdateLock
assert UpdateLock(path=Path(sys.argv[3]), install_root=sys.argv[2]).acquire()
print("held", flush=True)
time.sleep(120)
"""
def test_a_completion_from_another_home_never_builds_an_owned_checkout(tmp_path, monkeypatch):
"""R2 admission: `complete_source_checkout` ACQUIRES the checkout lock. Pre-fix it claimed
only its own home's marker and entered the build body beside the owning update."""
from hermes_cli import source_completion
root = tmp_path / "checkout"
(root / ".git").mkdir(parents=True)
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "other-home"))
entered = []
monkeypatch.setattr(source_completion, "_complete_locked", lambda *a, **k: entered.append(a))
holder = subprocess.Popen([sys.executable, str(_script(tmp_path, _HOLDER)), str(REPO_ROOT), str(root),
str(tmp_path / "owner-marker")],
stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, text=True, encoding="utf-8")
try:
assert holder.stdout is not None and holder.stdout.readline().strip() == "held"
with pytest.raises(RuntimeError, match="an update is still running"):
source_completion.complete_source_checkout(root, desktop=False, assume_yes=True)
assert entered == [], "the completion body ran while another update owned the checkout"
finally:
holder.kill()
holder.wait()
def test_a_live_legacy_profile_claim_is_honored(tmp_path, monkeypatch):
"""R11: before the root marker, an update under a named profile claimed
``<root>/profiles/<p>/.hermes-update-in-progress`` (v1) and never took the checkout lock.
While such a claim is live, a new update from that root refuses."""
root = tmp_path / "root"
profile = root / "profiles" / "work"
profile.mkdir(parents=True)
monkeypatch.setenv("HERMES_HOME", str(profile))
legacy = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], stdin=subprocess.DEVNULL)
try:
(profile / ".hermes-update-in-progress").write_text(f"{legacy.pid}\n{int(time.time())}\n")
lock = UpdateLock(path=root / ".hermes-update-in-progress")
assert lock.acquire() is False
assert lock.holder is not None and lock.holder.pid == legacy.pid
assert not (root / ".hermes-update-in-progress").exists()
finally:
legacy.kill()
legacy.wait()
# Dead: no longer honored (and left for the old updater that wrote it).
lock = UpdateLock(path=root / ".hermes-update-in-progress")
assert lock.acquire() is True
lock.release()
# --- R6: a dead marker is never reclaimed while the checkout lock is held ---------------------
_LOCK_HOLDER = """
import sys, time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli import update_lock
# A killed updater's completion child: holds the checkout lock, never wrote the marker.
assert update_lock._acquire_checkout(Path(sys.argv[2])) is None
Path(sys.argv[3]).write_text("held", encoding="utf-8")
time.sleep(120)
"""
def test_dead_marker_is_kept_and_reported_held_while_the_checkout_lock_is_held(tmp_path):
from hermes_cli import update_lock
install = tmp_path / "install"
install.mkdir()
marker = tmp_path / ".hermes-update-in-progress"
dead = subprocess.Popen([sys.executable, "-c", "pass"])
dead.wait()
marker.write_text(f"{dead.pid}\n{int(time.time())}\nct:1.5\n", encoding="utf-8")
holder = _python(tmp_path, _LOCK_HOLDER, install, tmp_path / "ready")
try:
_wait_for(tmp_path / "ready", holder)
verdict = update_lock.read_live_update(path=marker, install_root=install)
assert marker.exists(), "a reader deleted the dead marker while the update tree still held the checkout"
assert verdict is not None and verdict.held, f"expected the hand-off scripts' 'held', got {verdict!r}"
finally:
holder.kill()
holder.wait()
assert update_lock.read_live_update(path=marker, install_root=install) is None
assert not marker.exists(), "with the tree gone the dead marker is reclaimed"
def _dead_marker(marker: Path) -> None:
dead = subprocess.Popen([sys.executable, "-c", "pass"])
dead.wait()
marker.write_text(f"{dead.pid}\n{int(time.time())}\nct:1.5\n", encoding="utf-8")
def test_a_claimer_never_reclaims_a_dead_marker_while_the_checkout_lock_is_held(tmp_path):
"""M1 (round 7): the launch path claims the marker before it takes the checkout lock
(venv_sync). Its claim step deleted the dead marker over a held lock, then dropped its own
claim when the checkout refused: the marker was gone while the killed update's tree ran."""
from hermes_cli import update_lock
install = tmp_path / "install"
install.mkdir()
marker = tmp_path / ".hermes-update-in-progress"
_dead_marker(marker)
dead_bytes = marker.read_bytes()
holder = _python(tmp_path, _LOCK_HOLDER, install, tmp_path / "ready")
try:
_wait_for(tmp_path / "ready", holder)
claim = UpdateLock(path=marker, install_root=install)
assert claim._claim_marker() is False, "claimed over a dead marker while the checkout lock is held"
assert marker.read_bytes() == dead_bytes, "the claim step replaced the held update's marker"
assert claim.holder is not None and claim.holder.held
launch = UpdateLock(path=marker, install_root=install, checkout_first=False) # venv_sync
assert launch.acquire() is False
launch.release()
assert launch.holder is not None and launch.holder.held and launch.holder.pid == 0
assert marker.read_bytes() == dead_bytes
finally:
holder.kill()
holder.wait()
with UpdateLock(path=marker, install_root=install, checkout_first=False) as launch:
assert launch.acquired, "with the tree gone the dead marker is reclaimed and claimed"
assert not marker.exists()
assert update_lock.checkout_lock_held(install) is False
_ORPHANING_HOLDER = """
import subprocess, sys
from pathlib import Path
sys.path.insert(0, sys.argv[1])
from hermes_cli import update_lock
root = Path(sys.argv[2])
assert update_lock._acquire_checkout(root) is None # the record names this (soon dead) updater
release = Path(sys.argv[3]).with_name("release")
wait = "import sys, time, pathlib\\nwhile not pathlib.Path(sys.argv[1]).exists(): time.sleep(0.05)"
child = subprocess.Popen([sys.executable, "-c", wait, str(release)],
pass_fds=update_lock.checkout_lock_fds(root))
Path(sys.argv[3]).write_text(str(child.pid), encoding="utf-8")
"""
@posix_only
def test_a_refusal_over_a_dead_updaters_lock_uses_the_held_wording(tmp_path):
"""m6: `hermes update`, launch repair and source completion refuse through the lock's
holder record. Its updater is dead and a child it started holds the lock: say so, never
name the dead process."""
from hermes_cli import update_lock
install = tmp_path / "install"
install.mkdir()
updater = _python(tmp_path, _ORPHANING_HOLDER, install, tmp_path / "child")
updater.wait(timeout=30)
dead_pid = updater.pid
_wait_for(tmp_path / "child")
try:
lock = UpdateLock(path=tmp_path / ".hermes-update-in-progress", install_root=install)
assert lock.acquire() is False
assert lock.holder is not None and lock.holder.held and lock.holder.pid == 0
text = update_lock.describe_holder(lock.holder)
assert f"process {dead_pid}" not in text
assert "still holds the checkout" in text
finally:
(tmp_path / "release").touch() # the orphaned child is not ours to signal
# --- D17: a "held?" probe never makes a concurrent update fail --------------------------------
_PROBER = """
import fcntl, os, sys, time
from pathlib import Path
path = sys.argv[2]
end = time.monotonic() + float(sys.argv[3])
Path(sys.argv[4]).write_text("go", encoding="utf-8")
n = 0
while time.monotonic() < end:
fd = os.open(path, os.O_RDONLY)
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
time.sleep(0.0001) # held for the instant of one `flock -n` probe
fcntl.flock(fd, fcntl.LOCK_UN)
except OSError:
pass
os.close(fd)
n += 1
time.sleep(0.001) # the next probe's process spawn
print(n)
"""
@posix_only
def test_held_probe_never_fails_a_concurrent_acquire(tmp_path):
from hermes_cli import update_lock
install = tmp_path / "install"
install.mkdir()
lock_path = update_lock.checkout_lock_path(install)
lock_path.touch()
prober = _python(tmp_path, _PROBER, lock_path, 4, tmp_path / "go", stdout=subprocess.PIPE, text=True)
_wait_for(tmp_path / "go", prober)
attempts = refused = 0
try:
while prober.poll() is None:
attempts += 1
if update_lock._acquire_checkout(install) is not None:
refused += 1
else:
update_lock._release_checkout()
finally:
prober.kill()
prober.wait()
assert attempts > 50
assert refused == 0, f"{refused}/{attempts} acquires failed against a probe with no real holder"
# --- oversized numeric marker fields never raise ---------------------------------------------
def test_oversized_marker_numbers_never_raise_and_are_reclaimed(tmp_path):
from hermes_cli import update_lock
install = tmp_path / "install"
install.mkdir()
marker = tmp_path / ".hermes-update-in-progress"
for text in ("1" * 5000 + "\n1\nct:1.5\n", "4242\n" + "9" * 5000 + "\nct:1.5\n",
"4242\n1\nct:1.5\ndelegate:" + "7" * 5000 + " ct:1.5\n"):
marker.write_text(text, encoding="utf-8")
assert update_lock.read_live_update(path=marker, install_root=install) is None
marker.write_text(text, encoding="utf-8")
lock = UpdateLock(path=marker, install_root=install)
assert lock.acquire() is True, lock.holder
lock.release()