hermes-agent/hermes_cli/update_pause_record.py

"""Durable record of the gateways and services ``hermes update`` paused on Windows.

The pause token used to live only in the updater's memory (plus an ``atexit`` resume), so a
killed updater (console closed, ``taskkill``, power loss) left every paused gateway and SCM
service stopped with nothing on disk saying so. The record is written under the ROOT Hermes
home BEFORE anything is stopped, names its owner (pid + creation time, the update-marker
identity format), is rewritten as the token changes, and is removed only after a verified
resume. A later launch whose owner check finds the updater dead (and no update live) resumes it.

One record per checkout (the file name carries a key of the install root): a checkout sharing
the home never imports, relabels or certifies another checkout's paused set — its tree gate
says nothing about the other tree.

Resuming is gated on a whole tree: no interrupted-pull marker; at the pre-update HEAD, no tracked
change beyond the ones present at pause time (git died before moving HEAD); at a moved HEAD,
dependencies current for it where a launch syncs them (a build step may rewrite tracked files there). Every pause folded into
a record keeps its own baseline (``baselines``): each one taken at the current HEAD must hold.
Otherwise the record stays for the next launch, which runs after the interrupted-pull restore and
the dependency sync.

Custody: every mutation (write, claim, retire, discharge) happens while holding a kernel lock on
``<record dir>/.hermes-update-paused-gateways.lock`` (flock / msvcrt, released by the kernel when
the holder dies; the file is never deleted), and the read → judge → mutate decision is made
inside that hold. A recovering launch claims a record by publishing ``<record>.<pid>.<nonce>.claim``
with its own identity already inside (atomic replace), then retires the source; a crash between the
two leaves two files with ONE obligation id (``pause_id``), and an update that folds orphaned sets
into its own record lists their ids under ``absorbed`` before retiring them. Readers treat a file
whose id another file carries as a copy, never as a second obligation, and the next mutator
retires it. Completing an obligation first adds its ids to ``<record stem>.retired`` (atomic), then
unlinks its files: an unlink Windows refuses (a reader holding the file without
FILE_SHARE_DELETE) leaves a copy that is redundant by id, never an obligation that executes again.
The list is read only under the mutex, and only its absence means "nothing retired": a list that
cannot be read means unknown, so nothing is claimed and the list is never rewritten from that read.
A list that reads but does not parse is unknown too while any record or claim exists; with none
left it guards nothing and is set aside as ``<record stem>.corrupt``.
"""

from __future__ import annotations

import glob
import hashlib
import json
import os
import shutil
import subprocess
import sys
import threading
import time
import uuid
from contextlib import contextmanager, redirect_stdout, suppress
from pathlib import Path

RECORD_STEM = ".hermes-update-paused-gateways"
MUTEX_NAME = RECORD_STEM + ".lock"
_MUTEX_WAIT_S = 10.0
# Re-entry depth per THREAD: another thread of this process (a gateway's consumer, a watcher)
# must wait for the kernel lock like any other holder, never ride on this thread's hold.
_mutex_held = threading.local()


def install_root() -> Path:
    return Path(__file__).resolve().parents[1]


def _install_key(root: Path | None = None) -> str:
    return hashlib.sha256(os.path.normcase(str(root or install_root())).encode("utf-8")).hexdigest()[:12]


def record_path() -> Path:
    from hermes_constants import get_default_hermes_root
    return get_default_hermes_root() / f"{RECORD_STEM}.{_install_key()}.json"


def identity(pid: int | None = None) -> dict:
    """``{"pid", "ct"}`` with ``ct`` spelled like the update marker's line 3 (``ct:<s.3f>``).

    The creation time comes from the probe :func:`identity_is_live` judges with
    (``update_lock.process_create_time``: psutil, else stdlib), so a host without psutil still
    writes a provable identity instead of ``ct=None``."""
    from hermes_cli import update_lock
    pid = os.getpid() if pid is None else int(pid)
    created = update_lock.process_create_time(pid)
    return {"pid": pid, "ct": None if created is None else f"ct:{created:.3f}"}


def identity_is_live(ident: dict | None) -> bool:
    """The update marker's incarnation rule (``update_lock.incarnation_live``) for a recorded identity.

    Unprovable (alive, no readable creation time) counts as live: a record is never taken from, nor
    a paused process restarted over, a process that may still be the one recorded.
    """
    from hermes_cli import update_lock
    ident = ident or {}
    return update_lock.incarnation_live(ident.get("pid") or 0, ident.get("ct") or None) is not False


class RecordConflict(OSError):
    """The record names another pause whose owner is still live."""


class RecordBusy(OSError):
    """Another process held the record mutex past the bounded wait."""


class RetiredUnknown(OSError):
    """The retired-id list exists but cannot be read: which obligations are complete is unknown."""


class RecordUnknown(OSError):
    """A saved pause or claim exists but cannot be read or parsed: what it still owes is unknown."""


@contextmanager
def _mutex(wait_s: float = _MUTEX_WAIT_S):
    """Exclusive kernel lock on the record directory's sidecar (A7). Re-entrant per thread; each
    thread's first entry opens its own descriptor, so threads exclude each other like processes."""
    depth = getattr(_mutex_held, "depth", 0)
    if depth:
        _mutex_held.depth = depth + 1
        try:
            yield
        finally:
            _mutex_held.depth -= 1
        return
    from hermes_cli import update_lock
    path = record_path().with_name(MUTEX_NAME)
    path.parent.mkdir(parents=True, exist_ok=True)
    fd = os.open(path, os.O_RDWR | os.O_CREAT | getattr(os, "O_BINARY", 0), 0o644)
    try:
        deadline = time.monotonic() + wait_s
        while not update_lock._try_lock(fd):
            if time.monotonic() > deadline:
                raise RecordBusy(f"{path} is held by another process or thread")
            time.sleep(0.05)
        _mutex_held.depth = 1
        try:
            yield
        finally:
            _mutex_held.depth = 0
            update_lock._unlock(fd)
    finally:
        os.close(fd)


def _git_executable() -> str:
    """The git ``hermes update`` runs, found without installing anything: PATH's, else the copy
    install.ps1 staged in PM's store (``_subprocess_compat.expose_pm_git`` puts it on PATH only
    inside the updater). A bare ``git`` on such an install dies with WinError 2, so every gate
    read would fail and the paused gateways would never be restarted."""
    found = shutil.which("git")
    if found:
        return found
    with suppress(Exception):  # health: allow BLE001 -- no PM, no store, no Windows git package: PATH's answer ("git") stands
        import pm
        from pm import paths
        from pm.lock import Lockfile

        recorded = pm.installed_package("git", allow_outdated=True)
        if recorded is not None and recorded.binary is not None and recorded.binary.is_file():
            return str(recorded.binary)
        package, target = pm.get_package("git"), pm.current_target()
        version = Lockfile(paths.lockfile_path()).version("git")
        staged = package.binary(paths.store_root() / package.store_entry(version, target), target)
        if staged is not None and staged.is_file():
            return str(staged)
    return "git"


def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None:
    """A read-only git query in the updater's custody (``update_custody.run_git``: job-bound on
    Windows inside an update, custody config in argv); ``None`` when git cannot run."""
    from hermes_cli.update_custody import run_git
    try:
        return run_git([_git_executable(), "-C", str(root)], args, capture_output=True, text=True,
                       encoding="utf-8", errors="replace", stdin=subprocess.DEVNULL, timeout=30, check=False)
    except (OSError, ValueError, subprocess.SubprocessError):
        return None


def head_sha(root: Path) -> str | None:
    result = _git(root, "rev-parse", "--verify", "HEAD")
    if result is None or result.returncode != 0:
        return None
    return result.stdout.strip() or None


def tracked_changes(root: Path) -> list[str] | None:
    """Tracked paths that differ from HEAD (index or worktree); ``None`` when git cannot say."""
    result = _git(root, "status", "--porcelain=v1", "-z", "--untracked-files=no")
    if result is None or result.returncode != 0:
        return None
    paths = set()
    entries = result.stdout.split("\0")
    index = 0
    while index < len(entries):
        entry = entries[index]
        index += 1
        if len(entry) < 4:
            continue
        paths.add(entry[3:])
        if entry[0] in "RC":  # rename/copy: the source path follows as its own entry
            index += 1
    return sorted(paths)


def stamp_tree(token: dict, root: Path | None = None) -> dict:
    """Record what "the tree before this update" was, for the whole-tree gate: one baseline per
    pause (idempotent for this token's ``pause_id``), next to the baselines of the pauses it absorbed."""
    root = root or install_root()
    pause_id = token.setdefault("pause_id", uuid.uuid4().hex)
    baselines = token.setdefault("baselines", [])
    if not any(b.get("pause_id") == pause_id for b in baselines) and (state := tree_state(root)):
        baselines.append({"pause_id": pause_id, **state})
    return token


def tree_state(root: Path) -> dict | None:
    """HEAD, the tracked changes and their bytes now; ``None`` when *root* is not a git checkout."""
    if not (root / ".git").exists():
        return None
    dirty = tracked_changes(root)
    # The bytes, not just the names: an autostashed edit and git's half-written bytes share a path.
    return {"pre_sha": head_sha(root), "dirty_at_pause": dirty,
            "dirty_digests": {path: _digest(root / path) for path in dirty or []}}


def _digest(path: Path) -> str | None:
    try:
        return hashlib.sha256(path.read_bytes()).hexdigest()
    except OSError:  # deleted (or unreadable) at pause: it must read the same way to count as unchanged
        return None


def mark_move(token: dict | None, target: str) -> None:
    """Persist, BEFORE git writes the tree, the commit this run's checkout move goes to, on this
    pause's own baseline: an unmoved HEAD is later judged against what THAT move could write, never
    against refs a later fetch replaced. Raises when it cannot be recorded: the move must not run."""
    if not token or not token.get("pause_id") or not target:
        return
    for baseline in token.get("baselines") or []:
        if baseline.get("pause_id") == token["pause_id"]:
            baseline["move_targets"] = sorted({*baseline.get("move_targets", []), target})
    write({**token, "resume_needed": True})


_MAX_MOVE_TARGETS = 16


def _paths_between(root: Path, targets) -> set[str] | None:
    """Tracked paths that differ between HEAD and any of *targets* — all a fast-forward, reset or
    merge toward them can write; ``None`` when git cannot say."""
    paths: set[str] = set()
    for target in sorted(targets):
        diff = _git(root, "diff", "--name-only", "-z", "--no-renames", "HEAD", target, "--")
        if diff is None or diff.returncode != 0:
            return None
        paths.update(filter(None, diff.stdout.split("\0")))
    return paths


def _paths_a_move_could_write(root: Path) -> set[str] | None:
    """For a pause with no recorded move (:func:`mark_move`: none attempted, or a record from before
    it was kept): the paths between HEAD and a commit the checkout fetched (``FETCH_HEAD``) or tracks
    (``refs/remotes``). ``None`` when git cannot say or nothing was fetched (target unknown)."""
    where = _git(root, "rev-parse", "--git-path", "FETCH_HEAD")
    refs = _git(root, "for-each-ref", "--format=%(objectname)", "refs/remotes")
    if where is None or where.returncode != 0 or refs is None or refs.returncode != 0:
        return None
    try:
        fetched = (root / where.stdout.strip()).read_text(encoding="utf-8-sig", errors="replace").split("\n")
    except FileNotFoundError:
        fetched = []
    except OSError:
        return None
    targets = {line.split()[0] for line in fetched if line.split()} | set(refs.stdout.split())
    if not targets or len(targets) > _MAX_MOVE_TARGETS:
        return None
    return _paths_between(root, targets)


def _seen_at_pause(root: Path, baseline: dict, path: str) -> bool:
    """*path* is as the pause saw it: dirty then, with the same bytes now (a baseline from before
    digests were kept has only the name)."""
    if path not in (baseline.get("dirty_at_pause") or []):
        return False
    digests = baseline.get("dirty_digests")
    return digests is None or digests.get(path) == _digest(root / path)


def _half_written(root: Path, changes: list[str], at_head: list[dict]) -> list[str]:
    """Tracked changes at an unmoved HEAD that git may have left half-written, for the first
    baseline that has any. A change the pause did not see counts only on a path the update's move
    could write: a build/sync step (or the user) rewriting any other tracked file is not git's
    doing, and holding the set for it would keep gateways stopped until someone cleans the file.
    The move is the one recorded before it ran, so a later fetch never narrows it; with none
    recorded the fetched refs stand in, and when even those are unknown every unseen change counts."""
    for baseline in at_head:
        unexpected = sorted(path for path in changes if not _seen_at_pause(root, baseline, path))
        if not unexpected:
            continue
        # move_paths: files a step rewrote toward this HEAD (stash push/apply, reset, checkout --).
        targets, paths = baseline.get("move_targets"), baseline.get("move_paths")
        writable = _paths_between(root, targets) if targets else set() if paths else _paths_a_move_could_write(root)
        torn = unexpected if writable is None else [path for path in unexpected if path in {*writable, *(paths or ())}]
        if torn:
            return torn
    return []


def torn_by_move(root: Path, found: dict, changes: list[str] | None) -> list[str] | None:
    """Paths a move from *found*'s HEAD to the current one wrote that do not hold the current
    commit's bytes: tracked changes on them that were not already there, byte for byte, when the
    move started. ``None`` when git cannot say."""
    writable = _paths_between(root, [found["pre_sha"]]) if found.get("pre_sha") else None
    if writable is None or changes is None:
        return None
    return sorted(path for path in changes if path in writable and not _seen_at_pause(root, found, path))


def _landed_torn(root: Path, head: str, changes: list[str], landed: list[dict]) -> list[str]:
    """Tracked changes at a HEAD a recorded move reached that are not that commit's bytes. git can
    move HEAD past a file it failed to write (``checkout`` exits 0 after "unable to unlink old"),
    so a moved HEAD vouches for nothing by itself. The verdict the updater took the moment git
    returned stands while those paths stay changed: a later dependency sync or stash restore
    rewriting a file is not the move's doing. A move the updater died before judging is judged now."""
    for baseline in landed:
        verdict = (baseline.get("landed") or {}).get(head)
        if verdict is None:
            verdict = torn_by_move(root, baseline, changes)
        torn = list(changes) if verdict is None else [path for path in verdict if path in changes]
        if torn:
            return torn
    return []


def _deps_hold_resume(root: Path) -> bool:
    """At a moved HEAD: are stale dependencies a reason to keep the paused set stopped?

    Only where waiting converges. A self-managed install (``updateMechanism: self``) syncs its
    dependencies at the start of every launch (``venv_sync.prepare_launch``), before startup
    recovery runs, so a deferred set comes back on the next launch. Anywhere else nothing a launch
    does makes them current (a developer checkout owns its own venv): the set resumes now, as it
    did before the pause was durable, and a currency the probe cannot establish never holds it
    either — the restarted gateway's own launch syncs or reports the remedy.
    """
    if (os.environ.get("HERMES_DISABLE_LAZY_INSTALLS", "").lower() in ("1", "true", "yes")
            or not (root / ".git").exists() or not (root / "pyproject.toml").is_file()):
        return False  # prepare_launch skips the sync: waiting would never end
    try:
        from hermes_cli.steward import read_install_stamp
        if read_install_stamp(root).get("updateMechanism") != "self":
            return False
        import pm
        return not pm.venv_is_current(project_root=root)
    except Exception:  # health: allow BLE001 -- unknown currency must not strand the set: resume, the gateway's own launch syncs or names the remedy
        return False


def tree_is_whole(token: dict, root: Path | None = None) -> tuple[bool, str]:
    """May paused gateways start on this checkout now? ``(verdict, reason when not)``."""
    from hermes_cli._early_recovery import interrupted_pull_marker
    root = root or install_root()
    if (root / ".git").exists():
        if interrupted_pull_marker(root).exists():
            return False, "the checkout is mid-pull (interrupted-pull marker present)"
        head = head_sha(root)
        if head is None:
            return False, "the checkout HEAD is unreadable"
        baselines = token.get("baselines") or []
        at_head = [b for b in baselines if b.get("pre_sha") == head]
        # HEAD reached by a move these pauses recorded: judged against that commit's tree.
        landed = [b for b in baselines if b.get("pre_sha") != head
                  and (head in (b.get("landed") or {}) or head in (b.get("move_targets") or []))]
        if at_head or landed:
            # HEAD never moved for the at_head pauses: a tracked change one of them did not see is
            # git's half-written checkout. Each is judged on its own set — a killed run that moved
            # HEAD elsewhere vouches for nothing here, and this run's set never certifies an older one's.
            changes = tracked_changes(root)
            if changes is None:
                return False, "git cannot read the checkout state"
            torn = _half_written(root, changes, at_head) or _landed_torn(root, head, changes, landed)
            if torn:
                return False, f"the checkout has {len(torn)} file(s) git left half-written (e.g. {torn[0]})"
            if at_head:
                return True, ""
    if _deps_hold_resume(root):
        return False, "dependencies are not current for the updated code yet"
    return True, ""


def _atomic_write(path: Path, body: dict) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
    try:
        with open(tmp, "w", encoding="utf-8") as fh:
            json.dump(body, fh, indent=1, sort_keys=True)
            fh.flush()
            os.fsync(fh.fileno())
        os.replace(tmp, path)
    except BaseException:
        # A refused replace (a Windows reader without FILE_SHARE_DELETE) or a failed write must
        # not leave the half-published temp beside the record; the error itself still propagates.
        with suppress(OSError):
            tmp.unlink()
        raise


def read(path: Path | None = None) -> dict | None:
    """The saved body; ``None`` only when the file is absent. One that exists but cannot be read
    (a Windows sharing violation, an AV scanner) or does not parse raises :class:`RecordUnknown`:
    it may be the newest copy of an obligation (its partial progress) or the only one, so it never
    lets an older copy execute nor a new pause replace it. Callers fail closed and retry later."""
    path = path or record_path()
    try:
        text = path.read_text(encoding="utf-8-sig")
    except FileNotFoundError:
        return None
    except OSError as exc:
        raise RecordUnknown(f"cannot read the paused-gateway record {path}: {exc}") from exc
    try:
        body = json.loads(text)
    except ValueError:
        body = None
    if not isinstance(body, dict) or not isinstance(body.get("token"), dict):
        raise RecordUnknown(f"the paused-gateway record {path} is malformed; fix or remove it to resume")
    return body


UNOWNED = {"pid": 0, "ct": None}


def write(token: dict, *, owner: dict | None = None, path: Path | None = None) -> None:
    """Persist *token*. The owner of the same pause is kept; another pause's orphaned set is merged
    into *token* (in place, so this run's resume brings it back); a live one refuses."""
    path = path or record_path()
    with _mutex():
        existing = read(path)
        same = existing is not None and existing["token"].get("pause_id") == token.get("pause_id")
        if existing is not None and not same:
            if existing.get("install_root") != str(install_root()):
                raise RecordConflict(f"{path} holds gateways paused for another checkout ({existing.get('install_root')})")
            if identity_is_live(existing.get("owner")):
                raise RecordConflict(f"{path} holds gateways paused by live process {existing['owner'].get('pid')}")
            merge_into(token, drop_never_stopped(dict(existing["token"])))
        if owner is None:
            owner = existing["owner"] if same and existing is not None else identity()
        _atomic_write(path, {"schema": 1, "owner": owner, "install_root": str(install_root()), "token": token})


def mark_stop_requested(token: dict, pids, markers: dict | None = None) -> None:
    """Persist, BEFORE the first stop request, the stops this update intends and where each mapped
    gateway's request (its planned-stop marker) will be written. Intent is not a request: a live
    process counts as asked only once the producer or consumer checkpoints it, or while the
    valid request this update wrote is still on disk (:func:`_asked`)."""
    token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *(str(int(p)) for p in pids)})
    token["stop_sent"] = sorted(map(str, token.get("stop_sent") or []))
    token["stopper_pid"] = os.getpid()
    token["stop_markers"] = {**(token.get("stop_markers") or {}), **{str(int(p)): str(m) for p, m in (markers or {}).items()}}
    write(token, owner=identity())


def mark_stop_sent(token: dict, pid) -> None:
    """Persist that *pid*'s stop request was issued (its planned-stop marker is on disk, or the
    POSIX pause records it just before its socket request): from here a live *pid* is draining and
    keeps its restart debt until it exits. Best effort: the marker itself is evidence until this
    lands; its consumer checkpoints before unlinking it."""
    token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), str(int(pid))})
    try:
        write(token, owner=identity())
    except OSError as exc:
        print(f"  ⚠ Could not record the stop request for gateway {pid}: {exc}")


def mark_stop_consumed(path: Path, marker: dict) -> None:
    """Checkpoint a validated gateway consume BEFORE its marker disappears.

    Called under the pause mutex by the consumer. Preserve the holder's identity;
    this is evidence of a stop, not a transfer of the updater's custody.
    """
    from gateway.status import _same_hermes_home
    pid = str(marker["target_pid"])
    for src, body in _files(record_path()):
        token = body["token"]
        recorded_path = (token.get("stop_markers") or {}).get(pid)
        if (str(token.get("stopper_pid")) != str(marker.get("stopper_pid"))
                or pid not in (token.get("stop_requested") or [])
                or not recorded_path or not _same_hermes_home(recorded_path, path)):
            continue
        token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), pid})
        _atomic_write(src, body)  # Failure leaves the request intact for recovery.


def _accepted_path(marker_path: Path) -> Path:
    return marker_path.with_name(marker_path.name + ".accepted")


def mark_stop_accepted(path: Path, marker: dict) -> None:
    """The consumer's receipt when it cannot checkpoint into the record (busy mutex, refused
    replace): this incarnation accepted the request and drains until it exits, however long past
    the request's TTL. Written beside the marker, in the gateway's own home, never into the record.

    A home that refuses the new file (an ACL, a read-only directory) still lets the kept request be
    rewritten in place: stamped ``accepted`` it is the receipt. Torn mid-write it reads malformed,
    which recovery holds as unknown, never as "not asked". The consumer accepts the stop either way."""
    try:
        _atomic_write(_accepted_path(path), dict(marker))
        return
    except OSError as exc:
        refused = exc
    try:
        with open(path, "r+", encoding="utf-8") as fh:  # r+: never creates a request that was not there
            json.dump({**marker, "accepted": True}, fh)
            fh.truncate()
            fh.flush()
            os.fsync(fh.fileno())
    except OSError as exc:
        print(f"  ⚠ Could not record the accepted stop request {path}: {refused}; {exc}", file=sys.stderr)


def discharge(token: dict, path: Path | None = None) -> None:
    """Remove the record only while it still names this pause (decided under the mutex)."""
    path = path or record_path()
    with _mutex():
        existing = read(path)
        if existing is not None and existing["token"].get("pause_id") == token.get("pause_id"):
            _retire(path, [(path, existing)])


def sync(token: dict) -> None:
    """Mirror the token after a resume attempt: done → delete; anything still owed → rewrite."""
    if not token.get("pause_id") or token.get("recovery"):
        return  # a recovering launch keeps what it owes in its own claim (_resume_claimed)
    try:
        if token.get("resume_needed") or token.get("resume_deferred"):
            owed = {**token, "resume_needed": True}
            owed.pop("resume_deferred", None)
            write(owed)
        else:
            discharge(token)
    except OSError as exc:
        print(f"  ⚠ Could not update the paused-gateway record {record_path()}: {exc}")


def _live_update_elsewhere() -> bool:
    """A live update other than the one this process belongs to. The update tree holding the
    checkout lock (this process plus the partner whose marker it adopted) is not "elsewhere"."""
    from hermes_cli import update_lock
    root = install_root()
    return not update_lock.holds_checkout_lock(root) and update_lock.update_in_progress(root)


def _claims(path: Path) -> list[Path]:
    return sorted(path.parent.glob(f"{glob.escape(path.name)}.*.claim"))


def _retired_path(path: Path) -> Path:
    return path.with_suffix(".retired")


def _retired(path: Path) -> set[str]:
    """The completed obligation ids (call under the mutex). Only an absent list is empty: one that
    exists but cannot be read raises :class:`RetiredUnknown`, so a completed obligation is never
    claimed again and the list is never rewritten from a read that failed."""
    target = _retired_path(path)
    try:
        text = target.read_text(encoding="utf-8-sig", errors="replace")
    except FileNotFoundError:
        return set()
    except OSError as exc:  # a refused read (sharing violation, AV scanner): unknown
        raise RetiredUnknown(f"cannot read the retired paused-gateway list {target}: {exc}") from exc
    try:
        ids = json.loads(text).get("ids")
    except (ValueError, AttributeError):  # not JSON / not an object: malformed, like a non-list
        ids = None
    if not isinstance(ids, list):
        return _quarantine_retired(path, target)
    return {str(i) for i in ids}


def _quarantine_retired(path: Path, target: Path) -> set[str]:
    """A retired list that reads but does not parse. While a record or claim of this checkout
    exists the list may be the only proof it is complete: unknown, fail closed. With none left it
    guards nothing, and keeping it would abort every Windows update: set it aside and go on."""
    if path.exists() or _claims(path):
        raise RetiredUnknown(f"the retired paused-gateway list {target} is malformed")
    corrupt = target.with_suffix(".corrupt")
    try:
        os.replace(target, corrupt)
    except OSError as exc:
        raise RetiredUnknown(f"cannot set aside the malformed retired paused-gateway list {target}: {exc}") from exc
    print(f"  ⚠ Set aside a malformed paused-gateway history (no paused set refers to it): {corrupt}",
          file=sys.stderr)
    return set()


def _retire(path: Path, carriers: list[tuple[Path, dict]]) -> None:
    """Complete the obligations *carriers* hold (call under the mutex): their ids (and every id
    they absorbed) go on the durable retired list FIRST; a failed read or write raises and deletes
    nothing. Then the files go; one that cannot be deleted is a redundant copy from here on."""
    ids = {str(i) for _src, body in carriers
           for i in [body["token"].get("pause_id"), *(body["token"].get("absorbed") or [])] if i}
    retired = _retired(path)
    if not ids <= retired:
        _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(retired | ids)})
    for src, _body in carriers:
        with suppress(OSError):
            src.unlink()


def _holder(src: Path, body: dict) -> dict:
    """Who holds *src*: a record's owner, a claim's claimer (published with the claim itself)."""
    held_by = body.get("owner") if src.suffix == ".json" else body.get("claimer")
    return held_by if isinstance(held_by, dict) else UNOWNED


def _files(path: Path) -> list[tuple[Path, dict]]:
    """This checkout's record and claims; one that cannot be read raises :class:`RecordUnknown`."""
    found = []
    for src in (path, *_claims(path)):
        body = read(src)
        if body is not None and body.get("install_root") == str(install_root()):
            found.append((src, body))
    return found


def _redundant(found: list[tuple[Path, dict]], held: set[Path], retired: set[str] = frozenset()) -> set[Path]:
    """Files that only copy an obligation another file carries or that was completed: a source an
    update absorbed but died before retiring, one of two same-id files a claim transfer left (the
    furthest-progressed copy is kept), a retired id's leftover. Never a held file."""
    carriers: dict[str, Path] = {}
    for src, body in found:
        for oid in body["token"].get("absorbed") or []:
            carriers.setdefault(str(oid), src)
    redundant: set[Path] = set()
    by_id: dict[str, list[Path]] = {}
    for src, body in found:
        oid = str(body["token"].get("pause_id") or "")
        if not oid:
            continue
        if oid in retired or (oid in carriers and carriers[oid] != src):
            redundant.add(src)
        else:
            by_id.setdefault(oid, []).append(src)
    rev = {src: int(body.get("rev") or 0) for src, body in found}
    for copies in by_id.values():
        keep = next((s for s in copies if s in held), max(copies, key=lambda s: rev[s]))
        redundant.update(s for s in copies if s != keep)
    return redundant - held


def _survey(path: Path) -> tuple[list[tuple[Path, dict]], set[Path], set[Path]]:
    found = _files(path)
    held = {src for src, body in found if identity_is_live(_holder(src, body))}
    return found, held, _redundant(found, held, _retired(path))


def orphans(path: Path | None = None) -> list[tuple[Path, dict]]:
    """``(file, body)`` for every obligation of this checkout whose holder is dead — one file per
    obligation id; empty while another update is live (it adopts them itself). Surveyed under the
    mutex (a mutator replaces the retired list meanwhile); raises when the list cannot be read."""
    path = path or record_path()
    if not path.exists() and not _claims(path):
        return []
    with _mutex():
        found, held, redundant = _survey(path)
    found = [(src, body) for src, body in found if src not in held and src not in redundant]
    if not found or _live_update_elsewhere():
        return []
    return found


def retire_redundant(path: Path | None = None) -> None:
    """Delete copies of obligations another file already carries (crash leftovers of a transfer)."""
    path = path or record_path()
    if not path.exists() and not _claims(path) and not _retired_path(path).exists():
        return
    with suppress(RecordBusy), _mutex():
        _found, _held, redundant = _survey(path)
        for src in redundant:
            with suppress(OSError):
                src.unlink()
        _prune_retired(path)


def _prune_retired(path: Path) -> None:
    """Forget retired ids no file carries any more (a carrier that cannot be read raises first)."""
    retired = _retired(path)
    bodies = [body for body in map(read, (path, *_claims(path))) if body is not None]
    if not retired:
        return
    carried = {str(b["token"].get("pause_id")) for b in bodies}
    keep = retired & carried
    with suppress(OSError):
        if keep:
            _atomic_write(_retired_path(path), {"schema": 1, "ids": sorted(keep)})
        else:
            _retired_path(path).unlink()


def claim(src: Path) -> tuple[Path, dict] | None:
    """Take *src* (an orphaned record or a dead claimer's claim) for this process; ``None`` when a
    concurrent launch won it or holds the mutex. The claim is published with our identity inside,
    then the source is retired; a crash between leaves a same-id copy :func:`_redundant` drops."""
    path = record_path()
    mine = path.with_name(f"{path.name}.{os.getpid()}.{uuid.uuid4().hex[:8]}.claim")
    try:
        with _mutex():
            body = read(src)
            if body is None or body.get("install_root") != str(install_root()):
                return None
            _found, held, redundant = _survey(path)
            if src in held or src in redundant or src not in {s for s, _ in _found}:
                return None
            body["claimer"] = identity()
            body["rev"] = int(body.get("rev") or 0) + 1  # a copy left behind is never the newer one
            _atomic_write(mine, body)
            with suppress(OSError):
                src.unlink()
    except OSError:
        return None
    return mine, body


def adopt_orphans() -> tuple[dict | None, list[Path]]:
    """For ``hermes update``: claim every orphaned pause and merge their sets. The caller records the
    merged set durably (its ``absorbed`` ids name the claims), then :func:`release_claims`."""
    retire_redundant()
    adopted, claims = None, []
    for src, _body in orphans():
        won = claim(src)
        if won is None:
            continue
        claims.append(won[0])
        with _mutex():
            body = read(won[0]) or won[1]
            adopted = merge_into(adopted, drop_never_stopped(dict(body["token"])))
    return adopted, claims


def release_claims(claims: list[Path]) -> None:
    with suppress(RecordBusy), _mutex():
        for path in claims:
            with suppress(OSError):
                path.unlink()


def record_pause(token: dict, adopted: dict | None, claims: list[Path]) -> dict:
    """Merge the adopted set, stamp the tree and persist under this process BEFORE the first stop.
    Publish then retire: a crash between leaves claims whose ids the record lists as absorbed."""
    if adopted is not None:
        merge_into(token, adopted)
    stamp_tree(token)
    token.setdefault("stop_requested", [])
    write(token, owner=identity())
    release_claims(claims)
    return token


_CARRIED = ("pause_id", "baselines", "identities", "stop_requested", "stop_sent", "stopper_pid",
            "stop_markers", "absorbed")


def finish_pause(token: dict, intended: dict, adopted: dict | None) -> dict:
    """Carry the recorded pause identity onto the final token and mirror it to disk."""
    if adopted is not None:
        merge_into(token, adopted)
    token.update({key: intended[key] for key in _CARRIED if key in intended})
    sync(token)
    return token


def abandon_pause(intended: dict, adopted: dict | None) -> None:
    """This run's own stops were rolled back in-line: only an adopted orphan set is still owed,
    and it goes back on disk unowned for the next launch."""
    if adopted is None:
        discharge(intended)
        return
    carried = {key: intended[key] for key in ("pause_id", "baselines", "absorbed") if key in intended}
    write({**adopted, **carried}, owner=UNOWNED)


def _live_pids(token: dict) -> set[str]:
    return {str(pid) for pid, ct in (token.get("identities") or {}).items()
            if identity_is_live({"pid": pid, "ct": ct})}


def _without(token: dict, pids: set[str]) -> dict:
    token["profiles"] = {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) not in pids}
    token["unmapped"] = [u for u in (token.get("unmapped") or []) if str(u.get("pid")) not in pids]
    if "posix_units" in token:  # supervised POSIX entries (update_cmd_posix_pause) are judged like the rest
        token["posix_units"] = [u for u in token["posix_units"] or [] if str(u.get("pid")) not in pids]
    return token


def _request_on_disk(token: dict, pid: str) -> bool | None:
    """The planned-stop marker this update's stopper wrote for *pid* is still on disk (the gateway's
    watcher has not consumed it yet): the request was issued even if the updater died before
    :func:`mark_stop_sent` — or the gateway's receipt of accepting it (:func:`mark_stop_accepted`).
    A marker naming another stopper (a user's ``hermes gateway stop``) is not. ``None``: one of them
    exists but cannot be read or parsed — the stop may have been accepted, so it is unknown."""
    path = (token.get("stop_markers") or {}).get(str(pid))
    if not path:
        return False
    from gateway.status import _PLANNED_STOP_MARKER_TTL_S
    # An unconsumed request expires; the receipt of an accepted one is evidence for the whole drain
    # (only the accepting incarnation is ever judged: a later one is not this entry's live process).
    verdicts = (_names_request(token, pid, Path(path), _PLANNED_STOP_MARKER_TTL_S),
                _names_request(token, pid, _accepted_path(Path(path)), None))
    return True if True in verdicts else None if None in verdicts else False


def _names_request(token: dict, pid: str, path: Path, ttl_s: int | None) -> bool | None:
    """Does *path* hold this stopper's request for *pid*'s incarnation? Only an absent file or a
    valid one naming something else is ``False``; refused (a sharing violation, an AV scanner) or
    malformed is ``None``, unknown."""
    from gateway.status import _marker_is_stale, get_process_start_time
    try:
        marker = json.loads(path.read_text(encoding="utf-8-sig"))
        target, stopper = int(marker["target_pid"]), int(marker["stopper_pid"])
    except FileNotFoundError:
        return False
    except (OSError, ValueError, TypeError, KeyError):
        return None
    if (target != int(pid) or stopper != int(token.get("stopper_pid") or 0)
            or (ttl_s is not None and not marker.get("accepted")
                and _marker_is_stale(marker.get("written_at") or "", ttl_s))):
        return False
    expected, actual = marker.get("target_start_time"), get_process_start_time(int(pid))
    # Match the consumer's optional birth fingerprint, including unavailable clocks.
    return None in (expected, actual) or expected == actual


def _asked(token: dict) -> tuple[set[str], set[str]]:
    """``(asked, unknown)``: pids this update actually asked to stop (recorded as sent, or whose
    request is still on disk), and pids whose request evidence exists but cannot be read."""
    sent = {str(p) for p in token.get("stop_sent") or []}
    verdicts = {str(p): _request_on_disk(token, str(p)) for p in token.get("stop_requested") or [] if str(p) not in sent}
    return sent | {p for p, v in verdicts.items() if v}, {p for p, v in verdicts.items() if v is None}


def drop_never_stopped(token: dict) -> dict:
    """Drop entries whose recorded process is the same live incarnation AND was never actually asked
    to stop (the update died before issuing its request, even with the intent on disk): it is still
    serving, and a later exit — a user's ``hermes gateway stop`` included — is not this update's to
    undo. One it asked may be draining: it keeps its restart debt (:func:`split_draining`). The
    evidence is resolved here, once, into ``stop_sent`` (a merge carries it under another stopper).
    Evidence that cannot be read keeps the entry too (it may be draining) but resolves nothing: it is
    judged again next time. No stop record at all (a set from before stop tracking): nothing can be
    told apart."""
    if token.get("stop_requested") is None:
        return token
    asked, unknown = _asked(token)
    token["stop_sent"] = sorted(asked)
    return _without(token, _live_pids(token) - asked - unknown)


def split_draining(token: dict) -> dict:
    """Remove and return ``{"profiles", "unmapped", "posix_units"}`` whose recorded process is still
    running: it was asked to stop and has not exited yet, so it can only be restarted once it has."""
    live = _live_pids(token)
    draining = {"profiles": {p: pid for p, pid in (token.get("profiles") or {}).items() if str(pid) in live},
                "unmapped": [u for u in (token.get("unmapped") or []) if str(u.get("pid")) in live],
                "posix_units": [u for u in (token.get("posix_units") or []) if str(u.get("pid")) in live]}
    _without(token, live)
    return draining


def merge_into(token: dict | None, adopted: dict) -> dict:
    """Fold an orphaned pause into this update's token so its resume brings both sets back; its
    obligation id (and every id it absorbed) is recorded so a leftover copy is never resumed twice."""
    token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
    token["resume_needed"] = True
    # Every pause keeps its own tree evidence: the adopted baseline stops this run's stamp from
    # certifying what a killed update left half-written as "dirty before the pause", and this
    # run's own baseline still guards the HEAD a killed update moved to.
    baselines = token.setdefault("baselines", [])
    baselines.extend(b for b in adopted.get("baselines") or [] if b not in baselines)
    absorbed = token.setdefault("absorbed", [])
    absorbed.extend(i for i in [adopted.get("pause_id"), *(adopted.get("absorbed") or [])] if i and i not in absorbed)
    profiles = token.setdefault("profiles", {})
    for name, pid in (adopted.get("profiles") or {}).items():
        profiles.setdefault(name, pid)
    unmapped = token.setdefault("unmapped", [])
    unmapped.extend(u for u in adopted.get("unmapped") or [] if u not in unmapped)
    identities = token.setdefault("identities", {})
    for pid, ct in (adopted.get("identities") or {}).items():
        identities.setdefault(pid, ct)
    if "stop_requested" in adopted or "stop_requested" in token:
        requested, sent = adopted.get("stop_requested"), adopted.get("stop_sent")
        if requested is None:  # a set with no stop record keeps every live entry: all count as asked
            requested = sent = list(adopted.get("identities") or {})
        token["stop_requested"] = sorted({*map(str, token.get("stop_requested") or []), *map(str, requested)})
        token["stop_sent"] = sorted({*map(str, token.get("stop_sent") or []), *map(str, sent or [])})
    cold = token.setdefault("cold_start_profiles", {})
    for name, generation in (adopted.get("cold_start_profiles") or {}).items():
        cold.setdefault(name, generation)
    if adopted.get("cold_start_if_installed"):
        token["cold_start_if_installed"] = True
        if "attested_generation" in adopted:
            token.setdefault("attested_generation", adopted["attested_generation"])
    if adopted.get("platform"):  # a POSIX set (update_cmd_posix_pause): its supervised units ride along
        token["platform"] = adopted["platform"]
        units = token.setdefault("posix_units", [])
        units.extend(u for u in adopted.get("posix_units") or [] if u not in units)
    services = token.setdefault("services", [])
    services.extend(s for s in adopted.get("services") or [] if s not in services)
    if services:
        token.setdefault("expected_services", []).extend(s for s in services if s not in token["expected_services"])
        token.setdefault("restarted_services", [])
        token.setdefault("service_profiles", {}).update(adopted.get("service_profiles") or {})
    return token


def _has_work(token: dict) -> bool:
    return bool(token.get("profiles") or any(u.get("argv") for u in token.get("unmapped") or [])
                or token.get("services") or token.get("posix_units") or token.get("cold_start_if_installed") or token.get("cold_start_profiles"))


def _resume_claimed(claim_path: Path, body: dict) -> None:
    """Resume under the checkout lock, so no update mutates the tree between the gate and the
    verified start; an update that took the checkout after our claim gets the set back untouched."""
    from hermes_cli.update_lock import UpdateLock
    fence = UpdateLock()
    if not fence.acquire_checkout(install_root()):
        with suppress(OSError), _mutex():
            _atomic_write(claim_path, {**(read(claim_path) or body), "claimer": UNOWNED})
        return
    try:
        _resume_fenced(claim_path, body)
    finally:
        fence.release()


def _resume_fenced(claim_path: Path, body: dict) -> None:
    with _mutex():
        # A live consumer may have checkpointed after claim() returned its snapshot.
        body = read(claim_path) or body
        token = drop_never_stopped(dict(body["token"]))
        draining = split_draining(token)
    _hold_backing_off(token, draining)
    attempted = _relaunch_keys(token)
    token.update(resume_needed=True, recovery=True)
    try:
        if _has_work(token):
            print("→ Restarting gateway(s) paused by an interrupted `hermes update`...", file=sys.stderr)
            from hermes_cli.update_cmd_windows import _resume_windows_gateways_after_update
            # Whatever command this launch runs owns stdout (``--json`` output, a Desktop pipe).
            with redirect_stdout(sys.stderr):
                _resume_windows_gateways_after_update(token)
        else:
            token["resume_needed"] = False
    except Exception as exc:  # health: allow BLE001 -- recovery boundary: any resume error is reported and the claim handed back (finally), never raised into the launch
        print(f"  ⚠ Could not restart every paused gateway: {exc}. Run `hermes update` or "
              "`hermes gateway start`.", file=sys.stderr)
    finally:
        _back_off_unready(token, attempted, draining)
        _hand_back(claim_path, body, token, draining)


# A recovering launch runs before every command (chat, doctor, a Desktop-spawned serve) and waits
# for each relaunched gateway to become ready. One that does not is re-launched only after this
# backoff (``relaunch_retry``), so an interactive command pays that wait at most once per window;
# the debt itself stays owed, and ``hermes update`` (which drops the backoff) retries at once.
_RELAUNCH_RETRY_BASE_S = 60.0
_RELAUNCH_RETRY_MAX_S = 3600.0


def _relaunch_keys(token: dict) -> set[str]:
    return ({f"profile:{name}" for name in token.get("profiles") or {}}
            | {f"unmapped:{u.get('pid')}" for u in token.get("unmapped") or [] if u.get("argv")}
            | {f"unit:{u.get('unit') or u.get('label')}" for u in token.get("posix_units") or []})


def _hold_backing_off(token: dict, held: dict) -> None:
    """Move the profile/unmapped/supervised entries still inside their relaunch backoff from *token*
    into *held* (beside the draining ones): owed, not relaunched by this launch."""
    now = time.time()
    waiting = {key for key, state in (token.get("relaunch_retry") or {}).items() if now < float(state.get("next_at") or 0)}
    held["profiles"].update({n: p for n, p in token["profiles"].items() if f"profile:{n}" in waiting})
    held["unmapped"].extend(u for u in token["unmapped"] if u.get("argv") and f"unmapped:{u.get('pid')}" in waiting)
    token["profiles"] = {n: p for n, p in token["profiles"].items() if f"profile:{n}" not in waiting}
    token["unmapped"] = [u for u in token["unmapped"] if u not in held["unmapped"]]
    if units := token.get("posix_units"):
        held["posix_units"].extend(u for u in units if f"unit:{u.get('unit') or u.get('label')}" in waiting)
        token["posix_units"] = [u for u in units if u not in held["posix_units"]]


def _back_off_unready(token: dict, attempted: set[str], held: dict) -> None:
    """Start or extend the backoff of each attempted entry the resume left owed (not ready, not
    launched); forget it for one that came back. A tree-gate deferral attempted nothing."""
    retry = dict(token.get("relaunch_retry") or {})
    if not token.get("resume_deferred"):
        unready = _relaunch_keys(token)
        for key in attempted:
            if key not in unready:
                retry.pop(key, None)
                continue
            attempts = int((retry.get(key) or {}).get("attempts") or 0) + 1
            retry[key] = {"attempts": attempts, "next_at": time.time() + min(
                _RELAUNCH_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _RELAUNCH_RETRY_MAX_S)}
    owed = _relaunch_keys(token) | _relaunch_keys(held)
    token["relaunch_retry"] = {key: state for key, state in retry.items() if key in owed}


def _hand_back(claim_path: Path, body: dict, token: dict, draining: dict) -> None:
    """Retire the claim when nothing is owed; else hand it back unowned for the next launch — this
    launch may live for hours (a chat) — which retries at once, except a relaunch that did not
    become ready (its backoff, :func:`_back_off_unready`). A draining process stays owed until it
    has exited and been restarted. A failed rewrite leaves our claimer line, dead once we exit."""
    token["profiles"] = {**(token.get("profiles") or {}), **draining["profiles"]}
    token["unmapped"] = [*(token.get("unmapped") or []), *draining["unmapped"]]
    if draining.get("posix_units"):
        token["posix_units"] = [*(token.get("posix_units") or []), *draining["posix_units"]]
    owed = bool(token.get("resume_needed") or token.get("resume_deferred") or draining["profiles"]
                or draining["unmapped"] or draining.get("posix_units"))
    with suppress(OSError), _mutex():
        if owed:
            kept = {key: value for key, value in token.items() if key not in ("recovery", "resume_deferred")}
            _atomic_write(claim_path, {**body, "token": {**kept, "resume_needed": True}, "claimer": UNOWNED})
        else:
            _retire(record_path(), [(claim_path, read(claim_path) or body)])


def recover(argv: list[str] | None = None) -> None:
    """Start-of-run recovery: resume every orphaned pause. Never raises."""
    from hermes_cli._parser import command_argv
    command = command_argv(list(sys.argv[1:] if argv is None else argv))
    if command[:1] == ["update"] or command[:2] == ["gateway", "run"]:
        return  # the update adopts it itself; a booting gateway must not block on its siblings
    try:
        retire_redundant()
        for src, _body in orphans():
            won = claim(src)
            if won is not None:
                _resume_claimed(*won)
    except Exception as exc:  # health: allow BLE001 -- never brick a launch on recovery; the record stays for the next one
        print(f"  ⚠ Paused-gateway recovery skipped: {exc}", file=sys.stderr)