fix(estop): honor canonical ~/.hermes/ESTOP from profile gateways
Profile processes launch with HERMES_HOME=~/.hermes/profiles/<name>, so `hermes pause` at the fleet root did not bind fleet-analyst dispatch (t_7b65ff88). Check/resume both the process home and the fleet root.
This commit is contained in:
119
agent/estop.py
119
agent/estop.py
@@ -51,24 +51,62 @@ def _hermes_home() -> Path:
|
||||
return Path(os.path.expanduser("~/.hermes"))
|
||||
|
||||
|
||||
def _canonical_root() -> Path:
|
||||
"""Fleet-wide Hermes root, even when this process is a profile gateway.
|
||||
|
||||
Profile gateways launch with HERMES_HOME=~/.hermes/profiles/<name>.
|
||||
``hermes pause`` from an operator seat writes ~/.hermes/ESTOP. If we
|
||||
only inspect the profile home, the emergency stop does not bind
|
||||
(jarvis-os/t_7b65ff88: fleet-analyst kept dispatching through pause).
|
||||
"""
|
||||
try:
|
||||
from hermes_constants import get_default_hermes_root
|
||||
return Path(get_default_hermes_root())
|
||||
except Exception:
|
||||
return Path(os.path.expanduser("~/.hermes"))
|
||||
|
||||
|
||||
def sentinel_path() -> Path:
|
||||
"""Path of the ESTOP sentinel under the active HERMES_HOME."""
|
||||
"""Path of the ESTOP sentinel this process would write on `hermes pause`."""
|
||||
return _hermes_home() / SENTINEL_NAME
|
||||
|
||||
|
||||
def is_engaged() -> bool:
|
||||
"""Cheap check (one stat): is the global emergency stop engaged?
|
||||
|
||||
Fail SAFE on stat errors: if we cannot determine whether the sentinel
|
||||
exists (permission error, transient I/O failure on HERMES_HOME), report
|
||||
engaged. The module contract is that the pause must hold even when the
|
||||
sentinel is unreadable — a fail-open here would silently lift an
|
||||
operator's emergency stop exactly when the filesystem is misbehaving.
|
||||
"""
|
||||
def _candidate_sentinel_paths() -> list:
|
||||
"""Profile home first, then the fleet root if it is a different directory."""
|
||||
primary = sentinel_path()
|
||||
paths = [primary]
|
||||
try:
|
||||
return sentinel_path().exists()
|
||||
except OSError:
|
||||
return True
|
||||
root = _canonical_root() / SENTINEL_NAME
|
||||
except Exception:
|
||||
return paths
|
||||
if not isinstance(primary, Path):
|
||||
# Test doubles (e.g. fail-safe stat fixture) are not Path objects.
|
||||
paths.append(root)
|
||||
return paths
|
||||
try:
|
||||
if root.resolve() != primary.resolve():
|
||||
paths.append(root)
|
||||
except Exception:
|
||||
if root != primary:
|
||||
paths.append(root)
|
||||
return paths
|
||||
|
||||
|
||||
def is_engaged() -> bool:
|
||||
"""Cheap check: is the global emergency stop engaged?
|
||||
|
||||
Engaged if ANY candidate sentinel exists: the process HERMES_HOME
|
||||
(profile-local) or the fleet canonical root (~/.hermes). Fail SAFE on
|
||||
stat errors so an unreadable sentinel still holds the pause.
|
||||
"""
|
||||
saw_stat_error = False
|
||||
for path in _candidate_sentinel_paths():
|
||||
try:
|
||||
if path.exists():
|
||||
return True
|
||||
except OSError:
|
||||
saw_stat_error = True
|
||||
return saw_stat_error
|
||||
|
||||
|
||||
def engage(reason: Optional[str] = None) -> Path:
|
||||
@@ -91,14 +129,22 @@ def engage(reason: Optional[str] = None) -> Path:
|
||||
|
||||
|
||||
def disengage() -> bool:
|
||||
"""Remove the ESTOP sentinel. Returns True if a pause was lifted."""
|
||||
try:
|
||||
sentinel_path().unlink()
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
"""Remove ESTOP sentinels this process can see.
|
||||
|
||||
Lifts both the process-local sentinel and the fleet-root sentinel so
|
||||
``hermes resume`` from a profile gateway still clears an operator pause
|
||||
written at ~/.hermes/ESTOP.
|
||||
"""
|
||||
lifted = False
|
||||
for path in _candidate_sentinel_paths():
|
||||
try:
|
||||
path.unlink()
|
||||
lifted = True
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except (OSError, AttributeError):
|
||||
continue
|
||||
return lifted
|
||||
|
||||
|
||||
def get_state() -> Optional[dict]:
|
||||
@@ -107,18 +153,31 @@ def get_state() -> Optional[dict]:
|
||||
A sentinel with an unreadable/corrupt body still reports engaged, with
|
||||
both fields None — the pause is authoritative, the metadata is not.
|
||||
"""
|
||||
path = sentinel_path()
|
||||
if not path.exists():
|
||||
if not is_engaged():
|
||||
return None
|
||||
reason = None
|
||||
engaged_at = None
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(raw, dict):
|
||||
reason = raw.get("reason") or None
|
||||
engaged_at = raw.get("engaged_at") or None
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
found = False
|
||||
for path in _candidate_sentinel_paths():
|
||||
try:
|
||||
exists = path.exists()
|
||||
except OSError:
|
||||
return {"reason": None, "engaged_at": None}
|
||||
except AttributeError:
|
||||
continue
|
||||
if not exists:
|
||||
continue
|
||||
found = True
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
if isinstance(raw, dict):
|
||||
reason = raw.get("reason") or None
|
||||
engaged_at = raw.get("engaged_at") or None
|
||||
break
|
||||
except (OSError, ValueError, AttributeError):
|
||||
continue
|
||||
if not found:
|
||||
return None
|
||||
return {"reason": reason, "engaged_at": engaged_at}
|
||||
|
||||
|
||||
|
||||
@@ -376,3 +376,31 @@ def test_pause_command_registered_for_gateway():
|
||||
assert "pause" in GATEWAY_KNOWN_COMMANDS
|
||||
# Must be dispatchable while an agent is running (in-band emergency stop).
|
||||
assert cmd.busy_policy == "dispatch"
|
||||
|
||||
|
||||
def test_profile_gateway_honors_canonical_root_estop(tmp_path, monkeypatch):
|
||||
"""fleet-analyst-class: HERMES_HOME is a profile dir; pause lives at root.
|
||||
|
||||
A process launched with HERMES_HOME=~/.hermes/profiles/fleet-analyst must
|
||||
still treat ~/.hermes/ESTOP as engaged. Otherwise `hermes pause` is not
|
||||
a global emergency stop (t_7b65ff88).
|
||||
"""
|
||||
root = tmp_path / "hermes-root"
|
||||
profile = root / "profiles" / "fleet-analyst"
|
||||
profile.mkdir(parents=True)
|
||||
monkeypatch.setenv("HERMES_HOME", str(profile))
|
||||
estop._reset_log_state_for_tests()
|
||||
|
||||
assert estop.is_engaged() is False
|
||||
(root / "ESTOP").write_text("{\"reason\": \"thundering herd\"}\n", encoding="utf-8")
|
||||
assert estop.is_engaged() is True
|
||||
assert estop.paused_reply() is not None
|
||||
assert "paused" in estop.paused_reply().lower()
|
||||
# Profile-local engage still works and is independent.
|
||||
estop.engage(reason="local")
|
||||
assert (profile / "ESTOP").exists()
|
||||
assert estop.is_engaged() is True
|
||||
(root / "ESTOP").unlink()
|
||||
assert estop.is_engaged() is True # still held by profile sentinel
|
||||
estop.disengage()
|
||||
assert estop.is_engaged() is False
|
||||
|
||||
Reference in New Issue
Block a user