From 1fe0f2f3ac9748ce799272eb93bee2937b5ab802 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:58:06 -0700 Subject: [PATCH] feat(cron): import-error cron failures now name gateway code skew and the one-command fix (#95294 part 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an agent cron job dies with an import-class error (cannot import name / ModuleNotFoundError / ImportError), the failure summarizer — which runs inside the gateway process — now consults gateway.code_skew: if the process booted on a different revision than disk HEAD, the delivered message appends 'gateway is running stale code (booted on X, disk is at Y) — run hermes gateway restart'. Turns the reported two-day mystery (15 missed jobs, identical ImportError, no explanation) into a one-line fix instruction on the first failure. Fail-safe by construction: skew detection returns None on non-git installs and processes without a boot fingerprint, the probe seam swallows every exception, and no_agent script jobs (fresh subprocess, consistent imports) fall through to the generic cleaner — their ImportErrors are the script's own problem, and blaming gateway skew there would send the reader to the wrong place (same mode-gating as the provider branches). Reuses gateway/code_skew.py (the /model-switch skew detector) rather than adding a second fingerprint reader. --- cron/scheduler.py | 53 ++++++++- .../cron/test_cron_import_error_skew_hint.py | 107 ++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 tests/cron/test_cron_import_error_skew_hint.py diff --git a/cron/scheduler.py b/cron/scheduler.py index 11b3e91314..60aadc082f 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -191,6 +191,21 @@ def _failure_streak_nudge(job: dict) -> str: ) +def _detect_gateway_code_skew() -> tuple[str, str] | None: + """Boot-vs-disk revision skew for THIS process, or None. + + Thin wrapper over ``gateway.code_skew.detect_code_skew`` so the failure + summarizer stays a pure function under test (monkeypatch this seam) and + a broken import can never take the delivery path down with it. + """ + try: + from gateway.code_skew import detect_code_skew + + return detect_code_skew() + except Exception: + return None + + def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str: """Return a compact one-line failure message for chat delivery. @@ -339,7 +354,43 @@ def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str: cleaned = re.sub(r"\s+", " ", cleaned).strip() if len(cleaned) > 180: cleaned = cleaned[:177].rstrip() + "..." - return f"⚠️ Cron '{job_name}' failed: {cleaned}" + message = f"⚠️ Cron '{job_name}' failed: {cleaned}" + + # Import-class failures (#95294 part 3): a long-lived gateway whose + # checkout was updated underneath it (interrupted `hermes update`, manual + # git pull) serves MIXED modules — old entries frozen in sys.modules, + # new files loaded by lazy imports — and every agent cron job then dies + # with `cannot import name X` / ModuleNotFoundError. The error itself + # reads like a code bug, so operators debug the wrong thing (2 days on + # the reporting incident, 15 missed jobs). This process knows its own + # boot fingerprint: when boot SHA differs from disk HEAD, APPEND the + # cause and the one-command fix — never replacing the raw error text, + # which carries the failing symbol name. + # + # Fail-safe by construction: skew detection returns None on non-git + # installs and in processes without a boot fingerprint (no false + # accusations — message delivered unchanged), the probe seam swallows + # every exception, and no_agent script jobs are excluded via the same + # mode-gate as the provider branches (a fresh subprocess resolves + # imports consistently against disk; its ImportError is the script's + # own problem, and blaming gateway skew would send the reader to the + # wrong place). + if provider_reachable and re.search( + r"cannot import name|modulenotfounderror|importerror", lower + ): + try: + skew = _detect_gateway_code_skew() + except Exception: + skew = None # delivery must never die on a diagnostics probe + if skew is not None: + boot_rev, disk_rev = skew + message += ( + f" Likely cause: the gateway is running stale code (booted " + f"on {boot_rev}, disk is at {disk_rev}) — run " + "`hermes gateway restart` to fix it." + ) + + return message def _upsert_incident_for_failure( diff --git a/tests/cron/test_cron_import_error_skew_hint.py b/tests/cron/test_cron_import_error_skew_hint.py new file mode 100644 index 0000000000..cf7b4c4d80 --- /dev/null +++ b/tests/cron/test_cron_import_error_skew_hint.py @@ -0,0 +1,107 @@ +"""Import-class cron failures name gateway code skew when it exists (#95294). + +Field incident: an interrupted `hermes update` (pull done, restart never ran) +left the gateway on stale code for two days; every agent cron job failed with +`ImportError: cannot import name 'user_originated_turn_view'` and the operator +had no way to know the fix was one `hermes gateway restart`. The failure +summarizer runs inside the gateway process, which knows its own boot +fingerprint — when boot SHA != disk HEAD, the delivered error must say so and +name the command. +""" + +import cron.scheduler as scheduler +from cron.scheduler import _summarize_cron_failure_for_delivery + +IMPORT_ERROR = ( + "ImportError: cannot import name 'user_originated_turn_view' " + "from 'agent.context_compressor'" +) + + +def test_import_error_with_skew_names_shas_and_the_restart_command(monkeypatch): + monkeypatch.setattr( + scheduler, + "_detect_gateway_code_skew", + lambda: ("7e67f64fce", "ec5e369fe6"), + ) + job = {"name": "morning-brief", "id": "aaa111"} + msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR) + # The raw error text (with the failing symbol) must survive — the hint + # is APPENDED, never a replacement. + assert "cannot import name 'user_originated_turn_view'" in msg + assert "stale code" in msg + assert "booted on 7e67f64fce" in msg + assert "disk is at ec5e369fe6" in msg + assert "hermes gateway restart" in msg + + +def test_import_error_without_skew_stays_a_plain_import_message(monkeypatch): + """No skew (or non-git install): message is byte-identical to today's.""" + monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", lambda: None) + job = {"name": "morning-brief", "id": "aaa111"} + msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR) + assert "cannot import name 'user_originated_turn_view'" in msg + assert "stale code" not in msg + assert "hermes gateway restart" not in msg + + +def test_modulenotfound_matches_the_import_class(monkeypatch): + monkeypatch.setattr( + scheduler, + "_detect_gateway_code_skew", + lambda: ("aaaa111111", "bbbb222222"), + ) + job = {"name": "nightly-digest", "id": "ccc333"} + msg = _summarize_cron_failure_for_delivery( + job, "ModuleNotFoundError: No module named 'agent.turn_context'" + ) + assert "hermes gateway restart" in msg + + +def test_no_agent_script_import_error_never_blames_gateway_skew(monkeypatch): + """A no_agent script runs in a fresh subprocess — its ImportError is the + script's own problem and must reach the generic cleaner untouched.""" + monkeypatch.setattr( + scheduler, + "_detect_gateway_code_skew", + lambda: ("aaaa111111", "bbbb222222"), + ) + job = {"name": "disk-watchdog", "id": "ddd444", "no_agent": True} + msg = _summarize_cron_failure_for_delivery( + job, "ImportError: cannot import name 'requests'" + ) + assert "stale code" not in msg + assert "hermes gateway restart" not in msg + + +def test_skew_probe_failure_degrades_to_the_plain_message(monkeypatch): + """The seam swallowing an exception must behave exactly like no-skew.""" + + def boom(): + raise RuntimeError("git exploded") + + monkeypatch.setattr(scheduler, "_detect_gateway_code_skew", boom) + job = {"name": "morning-brief", "id": "aaa111"} + try: + msg = _summarize_cron_failure_for_delivery(job, IMPORT_ERROR) + except RuntimeError: + raise AssertionError( + "summarizer must not propagate a skew-probe failure" + ) from None + assert "cannot import name" in msg + + +def test_wrapper_seam_swallows_detector_import_failure(monkeypatch): + """_detect_gateway_code_skew itself never raises when the gateway module + is unimportable (e.g. stripped install).""" + import builtins + + real_import = builtins.__import__ + + def failing_import(name, *args, **kwargs): + if name.startswith("gateway"): + raise ImportError("gateway package missing") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", failing_import) + assert scheduler._detect_gateway_code_skew() is None