feat(cron): import-error cron failures now name gateway code skew and the one-command fix (#95294 part 3)

When an agent cron job dies with an import-class error (cannot import
name / ModuleNotFoundError / ImportError), the failure summarizer — which
runs inside the gateway process — now consults gateway.code_skew: if the
process booted on a different revision than disk HEAD, the delivered
message appends 'gateway is running stale code (booted on X, disk is at
Y) — run hermes gateway restart'. Turns the reported two-day mystery
(15 missed jobs, identical ImportError, no explanation) into a one-line
fix instruction on the first failure.

Fail-safe by construction: skew detection returns None on non-git
installs and processes without a boot fingerprint, the probe seam
swallows every exception, and no_agent script jobs (fresh subprocess,
consistent imports) fall through to the generic cleaner — their
ImportErrors are the script's own problem, and blaming gateway skew
there would send the reader to the wrong place (same mode-gating as the
provider branches).

Reuses gateway/code_skew.py (the /model-switch skew detector) rather
than adding a second fingerprint reader.
This commit is contained in:
Teknium
2026-08-26 00:58:06 -07:00
parent f0c0c986c4
commit 1fe0f2f3ac
2 changed files with 159 additions and 1 deletions

View File

@@ -191,6 +191,21 @@ def _failure_streak_nudge(job: dict) -> str:
)
def _detect_gateway_code_skew() -> tuple[str, str] | None:
"""Boot-vs-disk revision skew for THIS process, or None.
Thin wrapper over ``gateway.code_skew.detect_code_skew`` so the failure
summarizer stays a pure function under test (monkeypatch this seam) and
a broken import can never take the delivery path down with it.
"""
try:
from gateway.code_skew import detect_code_skew
return detect_code_skew()
except Exception:
return None
def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
"""Return a compact one-line failure message for chat delivery.
@@ -339,7 +354,43 @@ def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
cleaned = re.sub(r"\s+", " ", cleaned).strip()
if len(cleaned) > 180:
cleaned = cleaned[:177].rstrip() + "..."
return f"⚠️ Cron '{job_name}' failed: {cleaned}"
message = f"⚠️ Cron '{job_name}' failed: {cleaned}"
# Import-class failures (#95294 part 3): a long-lived gateway whose
# checkout was updated underneath it (interrupted `hermes update`, manual
# git pull) serves MIXED modules — old entries frozen in sys.modules,
# new files loaded by lazy imports — and every agent cron job then dies
# with `cannot import name X` / ModuleNotFoundError. The error itself
# reads like a code bug, so operators debug the wrong thing (2 days on
# the reporting incident, 15 missed jobs). This process knows its own
# boot fingerprint: when boot SHA differs from disk HEAD, APPEND the
# cause and the one-command fix — never replacing the raw error text,
# which carries the failing symbol name.
#
# Fail-safe by construction: skew detection returns None on non-git
# installs and in processes without a boot fingerprint (no false
# accusations — message delivered unchanged), the probe seam swallows
# every exception, and no_agent script jobs are excluded via the same
# mode-gate as the provider branches (a fresh subprocess resolves
# imports consistently against disk; its ImportError is the script's
# own problem, and blaming gateway skew would send the reader to the
# wrong place).
if provider_reachable and re.search(
r"cannot import name|modulenotfounderror|importerror", lower
):
try:
skew = _detect_gateway_code_skew()
except Exception:
skew = None # delivery must never die on a diagnostics probe
if skew is not None:
boot_rev, disk_rev = skew
message += (
f" Likely cause: the gateway is running stale code (booted "
f"on {boot_rev}, disk is at {disk_rev}) — run "
"`hermes gateway restart` to fix it."
)
return message
def _upsert_incident_for_failure(

View File

@@ -0,0 +1,107 @@
"""Import-class cron failures name gateway code skew when it exists (#95294).
Field incident: an interrupted `hermes update` (pull done, restart never ran)
left the gateway on stale code for two days; every agent cron job failed with
`ImportError: cannot import name 'user_originated_turn_view'` and the operator
had no way to know the fix was one `hermes gateway restart`. The failure
summarizer runs inside the gateway process, which knows its own boot
fingerprint — when boot SHA != disk HEAD, the delivered error must say so and
name the command.
"""
import cron.scheduler as scheduler
from cron.scheduler import _summarize_cron_failure_for_delivery
IMPORT_ERROR = (
"ImportError: cannot import name 'user_originated_turn_view' "
"from 'agent.context_compressor'"
)
def test_import_error_with_skew_names_shas_and_the_restart_command(monkeypatch):
monkeypatch.setattr(
scheduler,
"_detect_gateway_code_skew",
lambda: ("7e67f64fce", "ec5e369fe6"),
)
job = {"name": "morning-brief", "id": "aaa111"}
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
# The raw error text (with the failing symbol) must survive — the hint
# is APPENDED, never a replacement.
assert "cannot import name 'user_originated_turn_view'" in msg
assert "stale code" in msg
assert "booted on 7e67f64fce" in msg
assert "disk is at ec5e369fe6" in msg
assert "hermes gateway restart" in msg
def test_import_error_without_skew_stays_a_plain_import_message(monkeypatch):
"""No skew (or non-git install): message is byte-identical to today's."""
monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", lambda: None)
job = {"name": "morning-brief", "id": "aaa111"}
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
assert "cannot import name 'user_originated_turn_view'" in msg
assert "stale code" not in msg
assert "hermes gateway restart" not in msg
def test_modulenotfound_matches_the_import_class(monkeypatch):
monkeypatch.setattr(
scheduler,
"_detect_gateway_code_skew",
lambda: ("aaaa111111", "bbbb222222"),
)
job = {"name": "nightly-digest", "id": "ccc333"}
msg = _summarize_cron_failure_for_delivery(
job, "ModuleNotFoundError: No module named 'agent.turn_context'"
)
assert "hermes gateway restart" in msg
def test_no_agent_script_import_error_never_blames_gateway_skew(monkeypatch):
"""A no_agent script runs in a fresh subprocess — its ImportError is the
script's own problem and must reach the generic cleaner untouched."""
monkeypatch.setattr(
scheduler,
"_detect_gateway_code_skew",
lambda: ("aaaa111111", "bbbb222222"),
)
job = {"name": "disk-watchdog", "id": "ddd444", "no_agent": True}
msg = _summarize_cron_failure_for_delivery(
job, "ImportError: cannot import name 'requests'"
)
assert "stale code" not in msg
assert "hermes gateway restart" not in msg
def test_skew_probe_failure_degrades_to_the_plain_message(monkeypatch):
"""The seam swallowing an exception must behave exactly like no-skew."""
def boom():
raise RuntimeError("git exploded")
monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", boom)
job = {"name": "morning-brief", "id": "aaa111"}
try:
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
except RuntimeError:
raise AssertionError(
"summarizer must not propagate a skew-probe failure"
) from None
assert "cannot import name" in msg
def test_wrapper_seam_swallows_detector_import_failure(monkeypatch):
"""_detect_gateway_code_skew itself never raises when the gateway module
is unimportable (e.g. stripped install)."""
import builtins
real_import = builtins.__import__
def failing_import(name, *args, **kwargs):
if name.startswith("gateway"):
raise ImportError("gateway package missing")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", failing_import)
assert scheduler._detect_gateway_code_skew() is None