feat(cron): import-error cron failures now name gateway code skew and the one-command fix (#95294 part 3)
When an agent cron job dies with an import-class error (cannot import name / ModuleNotFoundError / ImportError), the failure summarizer — which runs inside the gateway process — now consults gateway.code_skew: if the process booted on a different revision than disk HEAD, the delivered message appends 'gateway is running stale code (booted on X, disk is at Y) — run hermes gateway restart'. Turns the reported two-day mystery (15 missed jobs, identical ImportError, no explanation) into a one-line fix instruction on the first failure. Fail-safe by construction: skew detection returns None on non-git installs and processes without a boot fingerprint, the probe seam swallows every exception, and no_agent script jobs (fresh subprocess, consistent imports) fall through to the generic cleaner — their ImportErrors are the script's own problem, and blaming gateway skew there would send the reader to the wrong place (same mode-gating as the provider branches). Reuses gateway/code_skew.py (the /model-switch skew detector) rather than adding a second fingerprint reader.
This commit is contained in:
@@ -191,6 +191,21 @@ def _failure_streak_nudge(job: dict) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _detect_gateway_code_skew() -> tuple[str, str] | None:
|
||||
"""Boot-vs-disk revision skew for THIS process, or None.
|
||||
|
||||
Thin wrapper over ``gateway.code_skew.detect_code_skew`` so the failure
|
||||
summarizer stays a pure function under test (monkeypatch this seam) and
|
||||
a broken import can never take the delivery path down with it.
|
||||
"""
|
||||
try:
|
||||
from gateway.code_skew import detect_code_skew
|
||||
|
||||
return detect_code_skew()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
|
||||
"""Return a compact one-line failure message for chat delivery.
|
||||
|
||||
@@ -339,7 +354,43 @@ def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
|
||||
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
||||
if len(cleaned) > 180:
|
||||
cleaned = cleaned[:177].rstrip() + "..."
|
||||
return f"⚠️ Cron '{job_name}' failed: {cleaned}"
|
||||
message = f"⚠️ Cron '{job_name}' failed: {cleaned}"
|
||||
|
||||
# Import-class failures (#95294 part 3): a long-lived gateway whose
|
||||
# checkout was updated underneath it (interrupted `hermes update`, manual
|
||||
# git pull) serves MIXED modules — old entries frozen in sys.modules,
|
||||
# new files loaded by lazy imports — and every agent cron job then dies
|
||||
# with `cannot import name X` / ModuleNotFoundError. The error itself
|
||||
# reads like a code bug, so operators debug the wrong thing (2 days on
|
||||
# the reporting incident, 15 missed jobs). This process knows its own
|
||||
# boot fingerprint: when boot SHA differs from disk HEAD, APPEND the
|
||||
# cause and the one-command fix — never replacing the raw error text,
|
||||
# which carries the failing symbol name.
|
||||
#
|
||||
# Fail-safe by construction: skew detection returns None on non-git
|
||||
# installs and in processes without a boot fingerprint (no false
|
||||
# accusations — message delivered unchanged), the probe seam swallows
|
||||
# every exception, and no_agent script jobs are excluded via the same
|
||||
# mode-gate as the provider branches (a fresh subprocess resolves
|
||||
# imports consistently against disk; its ImportError is the script's
|
||||
# own problem, and blaming gateway skew would send the reader to the
|
||||
# wrong place).
|
||||
if provider_reachable and re.search(
|
||||
r"cannot import name|modulenotfounderror|importerror", lower
|
||||
):
|
||||
try:
|
||||
skew = _detect_gateway_code_skew()
|
||||
except Exception:
|
||||
skew = None # delivery must never die on a diagnostics probe
|
||||
if skew is not None:
|
||||
boot_rev, disk_rev = skew
|
||||
message += (
|
||||
f" Likely cause: the gateway is running stale code (booted "
|
||||
f"on {boot_rev}, disk is at {disk_rev}) — run "
|
||||
"`hermes gateway restart` to fix it."
|
||||
)
|
||||
|
||||
return message
|
||||
|
||||
|
||||
def _upsert_incident_for_failure(
|
||||
|
||||
107
tests/cron/test_cron_import_error_skew_hint.py
Normal file
107
tests/cron/test_cron_import_error_skew_hint.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""Import-class cron failures name gateway code skew when it exists (#95294).
|
||||
|
||||
Field incident: an interrupted `hermes update` (pull done, restart never ran)
|
||||
left the gateway on stale code for two days; every agent cron job failed with
|
||||
`ImportError: cannot import name 'user_originated_turn_view'` and the operator
|
||||
had no way to know the fix was one `hermes gateway restart`. The failure
|
||||
summarizer runs inside the gateway process, which knows its own boot
|
||||
fingerprint — when boot SHA != disk HEAD, the delivered error must say so and
|
||||
name the command.
|
||||
"""
|
||||
|
||||
import cron.scheduler as scheduler
|
||||
from cron.scheduler import _summarize_cron_failure_for_delivery
|
||||
|
||||
IMPORT_ERROR = (
|
||||
"ImportError: cannot import name 'user_originated_turn_view' "
|
||||
"from 'agent.context_compressor'"
|
||||
)
|
||||
|
||||
|
||||
def test_import_error_with_skew_names_shas_and_the_restart_command(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
scheduler,
|
||||
"_detect_gateway_code_skew",
|
||||
lambda: ("7e67f64fce", "ec5e369fe6"),
|
||||
)
|
||||
job = {"name": "morning-brief", "id": "aaa111"}
|
||||
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
|
||||
# The raw error text (with the failing symbol) must survive — the hint
|
||||
# is APPENDED, never a replacement.
|
||||
assert "cannot import name 'user_originated_turn_view'" in msg
|
||||
assert "stale code" in msg
|
||||
assert "booted on 7e67f64fce" in msg
|
||||
assert "disk is at ec5e369fe6" in msg
|
||||
assert "hermes gateway restart" in msg
|
||||
|
||||
|
||||
def test_import_error_without_skew_stays_a_plain_import_message(monkeypatch):
|
||||
"""No skew (or non-git install): message is byte-identical to today's."""
|
||||
monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", lambda: None)
|
||||
job = {"name": "morning-brief", "id": "aaa111"}
|
||||
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
|
||||
assert "cannot import name 'user_originated_turn_view'" in msg
|
||||
assert "stale code" not in msg
|
||||
assert "hermes gateway restart" not in msg
|
||||
|
||||
|
||||
def test_modulenotfound_matches_the_import_class(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
scheduler,
|
||||
"_detect_gateway_code_skew",
|
||||
lambda: ("aaaa111111", "bbbb222222"),
|
||||
)
|
||||
job = {"name": "nightly-digest", "id": "ccc333"}
|
||||
msg = _summarize_cron_failure_for_delivery(
|
||||
job, "ModuleNotFoundError: No module named 'agent.turn_context'"
|
||||
)
|
||||
assert "hermes gateway restart" in msg
|
||||
|
||||
|
||||
def test_no_agent_script_import_error_never_blames_gateway_skew(monkeypatch):
|
||||
"""A no_agent script runs in a fresh subprocess — its ImportError is the
|
||||
script's own problem and must reach the generic cleaner untouched."""
|
||||
monkeypatch.setattr(
|
||||
scheduler,
|
||||
"_detect_gateway_code_skew",
|
||||
lambda: ("aaaa111111", "bbbb222222"),
|
||||
)
|
||||
job = {"name": "disk-watchdog", "id": "ddd444", "no_agent": True}
|
||||
msg = _summarize_cron_failure_for_delivery(
|
||||
job, "ImportError: cannot import name 'requests'"
|
||||
)
|
||||
assert "stale code" not in msg
|
||||
assert "hermes gateway restart" not in msg
|
||||
|
||||
|
||||
def test_skew_probe_failure_degrades_to_the_plain_message(monkeypatch):
|
||||
"""The seam swallowing an exception must behave exactly like no-skew."""
|
||||
|
||||
def boom():
|
||||
raise RuntimeError("git exploded")
|
||||
|
||||
monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", boom)
|
||||
job = {"name": "morning-brief", "id": "aaa111"}
|
||||
try:
|
||||
msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR)
|
||||
except RuntimeError:
|
||||
raise AssertionError(
|
||||
"summarizer must not propagate a skew-probe failure"
|
||||
) from None
|
||||
assert "cannot import name" in msg
|
||||
|
||||
|
||||
def test_wrapper_seam_swallows_detector_import_failure(monkeypatch):
|
||||
"""_detect_gateway_code_skew itself never raises when the gateway module
|
||||
is unimportable (e.g. stripped install)."""
|
||||
import builtins
|
||||
|
||||
real_import = builtins.__import__
|
||||
|
||||
def failing_import(name, *args, **kwargs):
|
||||
if name.startswith("gateway"):
|
||||
raise ImportError("gateway package missing")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", failing_import)
|
||||
assert scheduler._detect_gateway_code_skew() is None
|
||||
Reference in New Issue
Block a user