fix: a paused unmapped gateway keeps the home discovery proved
Discovery owns an unmapped gateway whose environment is unreadable when the exact-birth spawn ledger names its home, but the pause snapshot re-read only the environment and wrote `home: null`. The readiness predicate read that as a pre-home record and let another home's gateway with the same selectorless argv retire the debt. The snapshot, discovery and readiness now share one resolver (update_fleet_scope.gateway_pid_home: environment, else ledger witness). A home that no longer proves refuses the pause before any stop, and only an entry with no `home` key (an older updater's token) keeps argv-only matching; a recorded empty home is never matched.
This commit is contained in:
@@ -949,15 +949,22 @@ def _pause_windows_gateways_for_update() -> dict | None:
|
||||
return _cold_start_pause_token(adopted, claims)
|
||||
unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids]
|
||||
from gateway.status import get_process_start_time
|
||||
from hermes_cli.dashboard_procs import _hermes_home_for_pid
|
||||
from hermes_cli.update_fleet_scope import gateway_pid_home
|
||||
born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered
|
||||
# Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it.
|
||||
# Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). Two homes can
|
||||
# run identical selectorless argv, so the home and birth identify the runtime the replay must restore.
|
||||
# run identical selectorless argv, so the home and birth identify the runtime the replay must restore:
|
||||
# read with discovery's own resolver (environment, else the exact-birth ledger witness), and a home
|
||||
# that no longer proves refuses the pause before any stop. ``home: null`` would read as a pre-home
|
||||
# record, whose argv-only match lets another home's gateway retire this debt.
|
||||
homes = {int(pid): gateway_pid_home(int(pid)) for pid in unmapped_pids}
|
||||
if None in homes.values():
|
||||
raise RuntimeError("Could not prove the Hermes home of unmapped gateway PID(s) "
|
||||
+ ", ".join(str(p) for p, h in homes.items() if h is None) + "; nothing was stopped")
|
||||
unmapped = [
|
||||
{"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p),
|
||||
"Could not capture argv for unmapped gateway %s: %s", int(pid)),
|
||||
"home": _hermes_home_for_pid(int(pid)), "ct": born[int(pid)]}
|
||||
"home": homes[int(pid)], "ct": born[int(pid)]}
|
||||
for pid in unmapped_pids
|
||||
]
|
||||
intended = {
|
||||
@@ -1400,18 +1407,21 @@ def _unmapped_ready_filter(entry: dict, taken: set):
|
||||
same arguments after the (normalized) interpreter on the entry's home, born after the process it
|
||||
replaces, not one another entry already vouched for. Two homes can run identical selectorless argv,
|
||||
so argv alone would let home B's gateway retire home A's debt (a service gateway on another home
|
||||
included; on A's home only one gateway can run). An entry recorded before ``home`` was captured
|
||||
keeps the argv-only match: nothing else identifies it."""
|
||||
included; on A's home only one gateway can run). Only an entry recorded before ``home`` was
|
||||
captured (no key: a pre-durable-pause updater's token) keeps the argv-only match; a recorded
|
||||
``home`` that is empty proves no runtime and is never matched."""
|
||||
import psutil
|
||||
from gateway.status import _looks_like_gateway_process, _same_hermes_home, get_process_start_time
|
||||
from hermes_cli.dashboard_procs import _hermes_home_for_pid
|
||||
from hermes_cli.update_fleet_scope import gateway_pid_home
|
||||
tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0)
|
||||
home, old_ct = entry.get("home"), entry.get("ct")
|
||||
|
||||
def same_runtime(pid: int) -> bool:
|
||||
if not home:
|
||||
if "home" not in entry:
|
||||
return True
|
||||
live_home, ct = _hermes_home_for_pid(pid), get_process_start_time(pid)
|
||||
if not home:
|
||||
return False
|
||||
live_home, ct = gateway_pid_home(pid), get_process_start_time(pid)
|
||||
return live_home is not None and _same_hermes_home(live_home, home) and (old_ct is None or ct is None or ct > old_ct)
|
||||
|
||||
def matches(pid: int) -> bool:
|
||||
|
||||
@@ -65,14 +65,21 @@ def _ledger_gateway_home(pid: int) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def gateway_pid_in_update_scope(pid: int, scope: set[Path] | None = None) -> bool | None:
|
||||
"""Does gateway *pid* run on a home this update owns? ``None`` when its home cannot be read."""
|
||||
def gateway_pid_home(pid: int) -> str | None:
|
||||
"""The home gateway *pid* runs on: its live environment, else the exact-birth ledger witness;
|
||||
``None`` when neither proves one. Discovery, the Windows pause record and its readiness check
|
||||
share it, so the home discovery proved is the home the restart debt keeps."""
|
||||
from hermes_cli.dashboard_procs import _hermes_home_for_pid
|
||||
try:
|
||||
home = _hermes_home_for_pid(pid)
|
||||
except Exception:
|
||||
except Exception: # health: allow BLE001 -- tri-state resolver: any environment-read failure is "unknown" and the ledger witness decides
|
||||
home = None
|
||||
home = home or _ledger_gateway_home(pid)
|
||||
return home or _ledger_gateway_home(pid)
|
||||
|
||||
|
||||
def gateway_pid_in_update_scope(pid: int, scope: set[Path] | None = None) -> bool | None:
|
||||
"""Does gateway *pid* run on a home this update owns? ``None`` when its home cannot be read."""
|
||||
home = gateway_pid_home(pid)
|
||||
if home is None:
|
||||
return None
|
||||
return home_in_update_scope(home, scope)
|
||||
|
||||
101
tests/hermes_cli/test_update_pause_unmapped_home_proof.py
Normal file
101
tests/hermes_cli/test_update_pause_unmapped_home_proof.py
Normal file
@@ -0,0 +1,101 @@
|
||||
"""A paused unmapped gateway keeps the home discovery proved for it (#132338 R2 / review thread 7).
|
||||
|
||||
Discovery owns a gateway whose environment is unreadable (elevated, another account) when this
|
||||
install's spawn ledger holds an exact-birth witness of its home. The pause record must keep THAT
|
||||
home: written as ``home: null`` it read as a pre-home record, whose argv-only readiness match let
|
||||
another home's gateway with the same selectorless argv retire the paused runtime's debt.
|
||||
|
||||
Real processes, each under its own ``HERMES_HOME``, a real ledger entry, the real pause producer and
|
||||
durable record, and the real readiness predicate. Seams: the OS denying A's environment
|
||||
(``_pid_environ``), the Windows-only gateway enumeration and the stop itself (never signalled here).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
import pytest
|
||||
|
||||
from gateway.status import get_process_start_time
|
||||
from hermes_cli import dashboard_procs, gateway, process_identity, update_cmd_windows
|
||||
from hermes_cli import update_pause_record as pause_record
|
||||
from hermes_cli.update_cmd_windows import _pause_windows_gateways_for_update, _unmapped_ready_filter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def homes_and_spawn(tmp_path):
|
||||
entry = tmp_path / "bin" / "hermes" # a ``hermes`` entry token: the cmdline reads as a gateway
|
||||
entry.parent.mkdir()
|
||||
entry.write_text("import time\ntime.sleep(60)\n", encoding="utf-8")
|
||||
argv = [sys.executable, str(entry), "gateway", "run"]
|
||||
root = os.environ["HERMES_HOME"] # the sandboxed home this update owns, with two profiles
|
||||
home_a, home_b = (os.path.join(root, "profiles", name) for name in ("a", "b"))
|
||||
os.makedirs(home_a)
|
||||
os.makedirs(home_b)
|
||||
procs: list[subprocess.Popen] = []
|
||||
|
||||
def spawn(home) -> subprocess.Popen:
|
||||
proc = subprocess.Popen(argv, env={**os.environ, "HERMES_HOME": str(home)})
|
||||
procs.append(proc)
|
||||
return proc
|
||||
|
||||
yield home_a, home_b, spawn
|
||||
for proc in procs:
|
||||
proc.kill()
|
||||
proc.wait(timeout=10)
|
||||
|
||||
|
||||
@pytest.mark.platforms("linux")
|
||||
@pytest.mark.spawns_gateway_lookalike # stub children that sleep; the fixture reaps them
|
||||
def test_a_ledger_proven_unmapped_gateway_is_recorded_with_its_home(homes_and_spawn, monkeypatch):
|
||||
home_a, home_b, spawn = homes_and_spawn
|
||||
a = spawn(home_a)
|
||||
time.sleep(0.05)
|
||||
ct = process_identity._process_create_time(a.pid)
|
||||
assert process_identity._append_entry(process_identity.LedgerEntry(
|
||||
a.pid, ct, "gateway", process_identity.install_id(), None, None, time.time(), "", hermes_home=home_a))
|
||||
real_environ = dashboard_procs._pid_environ
|
||||
monkeypatch.setattr(dashboard_procs, "_pid_environ", lambda pid: None if pid == a.pid else real_environ(pid))
|
||||
from hermes_cli import main
|
||||
monkeypatch.setattr(main, "_is_windows", lambda: True)
|
||||
monkeypatch.setattr(gateway, "find_gateway_pids", lambda all_profiles=False: [a.pid])
|
||||
monkeypatch.setattr(gateway, "find_profile_gateway_processes", lambda strict=False: [])
|
||||
monkeypatch.setattr(gateway, "find_windows_gateway_services", lambda profile_processes=(): [])
|
||||
stopped = []
|
||||
monkeypatch.setattr(update_cmd_windows, "_stop_windows_gateways", lambda *a, **k: stopped.append(a[0]) or {})
|
||||
monkeypatch.setattr(update_cmd_windows, "_record_attested_cold_start_profiles", lambda *a: None)
|
||||
|
||||
_pause_windows_gateways_for_update()
|
||||
assert stopped == [[a.pid]]
|
||||
entry = pause_record.read(pause_record.record_path())["token"]["unmapped"][0]
|
||||
assert entry["home"] == home_a, "the pause dropped the home discovery proved"
|
||||
|
||||
a.kill()
|
||||
a.wait(timeout=10)
|
||||
time.sleep(0.05) # a later clock tick: a replacement is born after the process it replaces
|
||||
sibling = spawn(home_b)
|
||||
assert _unmapped_ready_filter(entry, set())([sibling.pid]) == [], \
|
||||
"home B's gateway with A's argv retired A's restart debt"
|
||||
replacement = spawn(home_a)
|
||||
assert _unmapped_ready_filter(entry, set())([sibling.pid, replacement.pid]) == [replacement.pid]
|
||||
|
||||
|
||||
@pytest.mark.platforms("linux")
|
||||
@pytest.mark.spawns_gateway_lookalike
|
||||
def test_a_recorded_but_empty_home_is_never_matched_only_a_pre_home_entry_is(homes_and_spawn):
|
||||
home_a, home_b, spawn = homes_and_spawn
|
||||
old = spawn(home_a)
|
||||
argv = list(psutil.Process(old.pid).cmdline())
|
||||
born = get_process_start_time(old.pid)
|
||||
old.kill()
|
||||
old.wait(timeout=10)
|
||||
time.sleep(0.05)
|
||||
sibling = spawn(home_b)
|
||||
assert _unmapped_ready_filter({"pid": old.pid, "argv": argv, "home": None, "ct": born}, set())([sibling.pid]) == [], \
|
||||
"a newly incomplete record took the legacy argv-only match"
|
||||
# A token from an updater that predates the home field has nothing else to match on.
|
||||
assert _unmapped_ready_filter({"pid": old.pid, "argv": argv}, set())([sibling.pid]) == [sibling.pid]
|
||||
Reference in New Issue
Block a user