hermes-agent/cron/scheduler.py

"""Cron job scheduler: tick() runs due jobs (gateway calls it every 60s from a background thread).
A file lock (~/.hermes/cron/.tick.lock) keeps overlapping processes to one tick at a time.
"""

import atexit
import concurrent.futures
import contextlib
import contextvars
import errno
import json
import logging
import os
import re
import subprocess
import sys
import threading
import time
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone

# fcntl is Unix-only; Windows uses msvcrt
try:
    import fcntl
except ImportError:
    fcntl = None
    try:
        import msvcrt
    except ImportError:
        msvcrt = None
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Protocol, Union

# Must precede repo-level imports: standalone invocations (e.g. module reload after
# `hermes update`) otherwise fail with ModuleNotFoundError for hermes_time et al.
sys.path.insert(0, str(Path(__file__).parent.parent))

from cron.worker_bootstrap import WORKER_MARKER
from hermes_constants import get_hermes_home, hermes_home_key
from hermes_cli.observability.shared_metrics_gateway import note_cron_execution, note_cron_skipped
from cron.env_settings import cron_env_setting
from hermes_cli._subprocess_compat import windows_hide_flags
from hermes_cli.config import (
    load_config, load_config_readonly)
from hermes_cli.fallback_config import get_fallback_chain, scoped_fallback_chain
from hermes_time import now as _hermes_now, safe_strftime
from agent.interrupt_compat import request_hard_interrupt
from agent.delegation_context import (
    enter_non_dispatcher_owned_context, exit_non_dispatcher_owned_context)
from agent.memory_provider import ctx_bound
from agent.session_activity import AwakeIdleMeter
from agent.turn_failure_copy import is_max_iteration_handoff

logger = logging.getLogger(__name__)


def _close_late_session_db_result(future: "concurrent.futures.Future") -> None:
    """Done-callback: close a SessionDB whose constructor finished after run_job's init timeout
    (worker abandoned via ``shutdown(wait=False)``), else its .db/WAL/SHM handles leak to EMFILE.

    If the constructor later completes inside that abandoned worker, the Future's result — an open SessionDB
    holding .db / WAL / SHM file handles — would be orphaned and never closed, leaking descriptors until
    EMFILE (#72782). This callback retrieves and closes that eventual late result.
    """
    with contextlib.suppress(Exception):
        db = future.result()
        if db is not None:
            from hermes_state_registry import release_or_close
            release_or_close(db)


def _set_cron_session_title(session_db, session_id, base_title):
    """Persist a non-blank, unique title for a finished cron session; returns it (None if unset).
    Runs BEFORE end_session()/close() so no write races the close. Duplicate title (unique-index
    ValueError) -> get_next_title_in_lineage(); if unavailable, raise rather than end up untitled.

    Centralizes the title write so the cron finally block can guarantee a non-blank, unique title is
    persisted before end_session()/close() tear the connection down (issues #50535, #50536, #50537):
    - #50535: never leaves the session blank. base_title already carries a cron-id fallback for nameless
    jobs; this also guards a failed write. Recover by appending a #N suffix via get_next_title_in_lineage()
    when supported, instead of swallowing the error and ending up untitled. - #50536: this runs
    synchronously in the cron finally block ahead of the session close, so no in-flight title write can race
    the close.
    """
    if not session_db or not session_id:
        return None
    title = (base_title or "").strip()
    if not title:
        return None
    try:
        session_db.set_session_title(session_id, title)
        return title
    except ValueError:
        # Unique-title collision: fall back to the next lineage title (base #2, #3, ...).
        next_title_fn = getattr(session_db, "get_next_title_in_lineage", None)
        if next_title_fn is None:
            raise
        deduped = next_title_fn(title)
        if not deduped or deduped == title:
            raise
        session_db.set_session_title(session_id, deduped)
        return deduped


def _job_route_pinned(job: dict) -> bool:
    """True when the job carries its own provider, model or endpoint. Unpinned jobs store none of
    these (they follow the main model at fire time), so any value is an explicit operator pin."""
    return any(isinstance(job.get(k), str) and job[k].strip() for k in ("provider", "model", "base_url"))


def _job_fallback_chain(job: dict, cfg: Any) -> Optional[list]:
    """The fallback chain this job may walk, at credential resolution AND mid-run (#100437).

    A pinned job never borrows the global ``fallback_providers`` chain, the same rule a pinned
    ``delegate_task`` child follows (``scoped_fallback_chain``): a chain entry is a different
    provider and usually a different model, which is exactly what the pin ruled out. Same-provider
    credential-pool rotation is not the chain and still applies. Unpinned jobs inherit the chain.
    """
    return scoped_fallback_chain(
        get_fallback_chain(cfg), None, pinned=_job_route_pinned(job), owner="cron job")


def _fallback_chain_phrase(job: Optional[dict] = None) -> str:
    """Backup-provider clause for a provider-failure notice: "pinned, no fallback" vs "the backups
    failed too" vs "none configured" (most installs). Fails open to "the backups failed too" if
    config can't be read — never crash delivery.
    """
    if job is not None and _job_route_pinned(job):
        return (
            "This job is pinned to its own provider/model, so it does not fall back to "
            f"`fallback_providers`; `hermes cron edit {job.get('id')} --unpin` lets it follow the "
            "main model and its fallback chain."
        )
    try:
        cfg = load_config() or {}
        chain = get_fallback_chain(cfg)
    except Exception:
        return "No backup provider succeeded either."
    if chain:
        return "No backup provider succeeded either."
    return (
        "No backup provider is configured — add one with `hermes fallback add`, "
        "or set a cron-wide default via `cron.model` + `cron.model_provider` in config.yaml."
    )


def _failure_streak_nudge(job: dict) -> str:
    """Review nudge when a recurring job keeps failing, else "". The failure message is delivered
    BEFORE mark_job_run records this run, hence stored ``failure_streak`` + 1. Threshold:
    ``cron.failure_nudge_threshold`` (default 3, 0 disables)."""
    schedule_kind = (job.get("schedule") or {}).get("kind")
    if schedule_kind not in {"cron", "interval"}:
        return ""
    try:
        cfg = load_config() or {}
        threshold = int(
            ((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get(
                "failure_nudge_threshold", 3
            )
        )
    except Exception:
        threshold = 3
    if threshold <= 0:
        return ""
    streak = int(job.get("failure_streak") or 0) + 1  # +1 = this run
    if streak < threshold:
        return ""
    job_ref = job.get("name") or job.get("id") or "this job"
    return (
        f"\nThis job has failed {streak} runs in a row — worth a review. "
        f"Fix its prompt/config, or pause it with `hermes cron pause {job_ref}` "
        "(resume/remove also available) to stop the noise."
    )


def _detect_gateway_code_skew() -> tuple[str, str] | None:
    """Boot-vs-disk revision skew for THIS process, or None. Test seam over
    ``gateway.code_skew.detect_code_skew``; a broken import must never take delivery down."""
    try:
        from gateway.code_skew import detect_code_skew

        return detect_code_skew()
    except Exception:
        return None


def _current_gateway_code_sha() -> str | None:
    """Full revision currently on disk; kept separate from display-shortened skew labels."""
    try:
        from gateway.code_skew import current_code_sha

        return current_code_sha()
    except Exception:
        return None


class CronTickYielded(RuntimeError):
    """A stale-code ticker yielded this tick to a fresh gateway.

    Raised by ``tick()`` BEFORE the tick lock when boot fingerprint ≠ disk, this process does NOT
    own the runtime lock and its live holder reports the disk revision — the stale process must
    stay out of the dispatch race (contention would starve the fresh ticker). Skew ``None`` never
    yields (fail open). Raised, not returned, so ``record_ticker_error`` sees it and ``hermes cron
    status`` isn't green.
    """

    def __init__(self, boot_rev: str, disk_rev: str) -> None:
        self.boot_rev = boot_rev
        self.disk_rev = disk_rev
        super().__init__(
            f"Cron tick yielded to a fresh gateway process (stale code: "
            f"booted on {boot_rev}, disk is at {disk_rev})"
        )


_STALE_YIELD_RE = re.compile(r"stale code: booted on (\S+), disk is at (\S+)\)")


def stale_code_yield_labels(recorded_error: str | None) -> tuple[str, str] | None:
    """``(boot_rev, disk_rev)`` when a persisted ``ticker_last_error`` is a ``CronTickYielded``.

    ``hermes cron status`` runs in another process and only sees the marker text; a yielding
    ticker still refreshes its heartbeat, so this is the one signal that separates "stale code,
    firing nothing" from a healthy loop (#117275).
    """
    if not recorded_error or not recorded_error.startswith(CronTickYielded.__name__):
        return None
    match = _STALE_YIELD_RE.search(recorded_error)
    return (match.group(1), match.group(2)) if match else None


# Log the yield at most once per episode (reset when the skew changes) to avoid per-interval spam.
_YIELD_LOG_INTERVAL_SECONDS = 3600.0
_last_yield_log: dict[str, object] = {}


def _should_yield_tick_to_fresh_gateway() -> tuple[str, str] | None:
    """``(boot_rev, disk_rev)`` when THIS profile's tick must yield to a fresher gateway, else None.

    Yields only when ALL hold: code skew, this process is not the cron owner for the profile being
    ticked, and another live host gateway whose served set covers that profile reports a fresh
    heartbeat on the disk revision. Every probe failure returns None — yielding is a certainty
    claim, never a guess.

    The question is asked PER PROFILE. One process multiplexes every profile, so "do I own the
    runtime lock" (process-global, stamped at boot for the launch home) cannot answer it: a
    stale multiplexer that holds the lock must still keep ticking the profiles nobody else ticks,
    and must still yield the ones a fresher gateway serves.

    ``gateway_state.json`` is per-HOME and last-writer-wins, not per-process: during a
    ``--replace`` takeover both the stale and the fresh gateway stamp it, so which one this
    predicate reads is write-order dependent. The pid equality in ``live_gateway_ticking`` is what
    binds the record to the current lock holder; the takeover window itself fails open (no yield).
    """
    skew = _detect_gateway_code_skew()
    if skew is None:
        return None
    disk_sha = _current_gateway_code_sha()
    if disk_sha is None:
        return None
    try:
        from cron.scheduler_ownership import live_gateway_ticking, owns_cron_tick_for

        home = _get_hermes_home()
        if owns_cron_tick_for(home):
            return None
        holder_status = live_gateway_ticking(home)
        if holder_status is None or holder_status.get("code_sha") != disk_sha:
            return None
    except Exception:
        return None
    return skew


def _log_tick_yield_once(reason: str) -> None:
    """Log the yield at error level once per episode (skew signature)."""
    global _last_yield_log
    now = time.monotonic()
    last_reason = _last_yield_log.get("reason")
    last_at = _last_yield_log.get("at", 0.0)
    if last_reason != reason or (now - float(last_at)) >= _YIELD_LOG_INTERVAL_SECONDS:
        logger.error(
            "Cron tick yielded: this process is running stale code (%s) and a "
            "fresher gateway owns the runtime lock — jobs will fire from that "
            "process. Restart this one to reclaim its ticks.",
            reason)
    _last_yield_log = {"reason": reason, "at": now}


def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str:
    """One-line failure notice for chat delivery (full details stay in the run output).

    Deterministic scheduler/script shapes are matched first (their text can contain "timed out"
    and would otherwise be blamed on the model service); everything else goes through the shared
    ``classify_api_error`` verdict and the copy table in ``scheduler_failure_copy``."""
    from cron.scheduler_failure_copy import (
        classify_cron_failure_reason, generic_failure_notice, inactivity_notice,
        provider_failure_notice, script_timeout_notice)

    job_name = job.get("name") or job.get("id") or "cron job"
    job_id = job.get("id") or job_name
    text = (error or "unknown error").strip()
    lower = text.lower()

    # Script runner contract ("Script timed out after {n}s: {path}") — also for agent jobs with a
    # context script. Must precede provider classification so it never claims a model failure.
    # See #78503, #82460.
    if lower.startswith("script timed out"):
        return script_timeout_notice(job_name, job_id)

    # Scheduler inactivity watchdog ("idle for {n}s (limit {m}s)"): the job's OWN tool call went
    # quiet, no model service involved. Its text may still contain "timed out", so it must be
    # recognised before the classifier (field-reported: a stuck `terminal` call was blamed on the
    # provider and the operator debugged the wrong system).
    if re.search(r"idle for \d+s\s*\(limit \d+s\)", lower):
        return inactivity_notice(job_name, job_id)

    # no_agent jobs never reach a model, so provider errors are structurally impossible for them:
    # gate on job MODE before classifying, or a script's own wording ("429", "timed out") would
    # blame the wrong subsystem.
    if not job.get("no_agent"):
        notice = provider_failure_notice(
            job_name, job_id, classify_cron_failure_reason(text),
            backup_provider_phrase=_fallback_chain_phrase(job), provider=job.get("provider"))
        if notice is not None:
            return notice

    # Strip exception wrappers; bound input first so a multi-KB blob can't slow the regexes.
    cleaned = re.sub(r"^(RuntimeError|Exception|ValueError|HTTPStatusError):\s*", "", text[:2000])
    cleaned = re.sub(r"\s+", " ", cleaned).strip().rstrip(".")
    if len(cleaned) > 180:
        cleaned = cleaned[:177].rstrip() + "..."
    message = generic_failure_notice(job_name, job_id, cleaned)

    # Import-class failures (#95294 part 3): a long-lived gateway whose checkout was updated
    # underneath it (interrupted `hermes update`, manual git pull) serves MIXED modules and every
    # agent cron job dies with `cannot import name X`. The error reads like a code bug, so APPEND
    # cause + fix — never replace the raw error, which carries the failing symbol. Fail-safe: skew
    # is None on non-git/no-fingerprint; no_agent jobs excluded (a fresh subprocess resolves
    # imports against disk, so its ImportError is the script's own problem).
    if not job.get("no_agent") and re.search(
        r"cannot import name|modulenotfounderror|importerror", lower
    ):
        try:
            skew = _detect_gateway_code_skew()
        except Exception:
            skew = None  # delivery must never die on a diagnostics probe
        if skew is not None:
            boot_rev, disk_rev = skew
            message += (
                f" Likely cause: the gateway is running stale code (booted "
                f"on {boot_rev}, disk is at {disk_rev}) — run "
                "`hermes gateway restart` to fix it."
            )

    return message


DEFAULT_FAILURE_REPEAT_ALERT_HOURS = 6.0


def _failure_repeat_alert_hours() -> float:
    """``cron.failure_repeat_alert_hours``: how long an ``alerted`` incident stays silent before one
    reminder ping. ``0`` (or negative) re-alerts on every failing run (the pre-gate behaviour)."""
    from cron.jobs import _cron_config_number

    return _cron_config_number("failure_repeat_alert_hours", DEFAULT_FAILURE_REPEAT_ALERT_HOURS, float)


def _repeat_alert_withheld(incident: dict) -> bool:
    """An ``alerted`` incident withholds the per-run ping until the reminder cooldown has elapsed
    since its last ping. A missing/unparseable ``alerted_at`` (ledger written before the column
    existed) delivers rather than swallowing an alert."""
    hours = _failure_repeat_alert_hours()
    if hours <= 0:
        return False
    alerted_at = incident.get("alerted_at")
    if not alerted_at:
        return False
    try:
        from cron.jobs import _elapsed_seconds, _ensure_aware

        last = _ensure_aware(datetime.fromisoformat(str(alerted_at)))
    except (TypeError, ValueError):
        return False
    return _elapsed_seconds(_hermes_now(), last) < hours * 3600


def _upsert_incident_for_failure(
    job: dict, error: str, *, output_file: Optional[Any] = None
) -> tuple[bool, Optional[str]]:
    """Record a durable failure incident (grouped by job + error signature). Returns
    ``(withheld, incident_id)``; withheld=True when the signature's incident is already ``closed``
    (operator ack) or ``alerted`` inside the ``cron.failure_repeat_alert_hours`` cooldown (a ping
    already went out) -> suppress the per-run ping. Store errors log at debug; the caller delivers
    as if none existed."""
    try:
        from cron.incidents import get_incident, upsert_incident

        incident_id, _is_new = upsert_incident(
            job["id"], str(error or ""), job_name=job.get("name"), output_file=output_file)
        incident = get_incident(incident_id)
        state = incident.get("state") if incident else None
        withheld = state == "closed" or (state == "alerted" and _repeat_alert_withheld(incident))
        return withheld, incident_id
    except Exception as exc:
        logger.debug(
            "Incident store unavailable for job %s (delivery unaffected): %s",
            job["id"], exc)
        return False, None


def _resolve_incidents_for_recovered_job(job: dict) -> None:
    """Best-effort: a successful run marks the job's open incidents ``resolved`` (never touches an
    operator ``closed`` ack). Store errors log at debug; delivery is unaffected."""
    try:
        from cron.incidents import close_incidents_for_recovered_job

        close_incidents_for_recovered_job(job["id"])
    except Exception as exc:
        logger.debug("Incident store unavailable for job %s (delivery unaffected): %s", job["id"], exc)


def _mark_incident_alerted(incident_id: Optional[str]) -> None:
    """Best-effort: mark incident ``alerted`` (no-op for closed; never resurrects an acked one)."""
    if not incident_id:
        return
    try:
        from cron.incidents import set_incident_state

        set_incident_state(incident_id, "alerted")
    except Exception as exc:
        logger.debug("Failed marking incident %s alerted: %s", incident_id, exc)


class CronPromptInjectionBlocked(Exception):
    """Raised by _build_job_prompt when the assembled prompt (incl. runtime-loaded skill content,
    unseen by create-time scanning) trips the injection scanner; run_job turns it into a clean
    "job blocked" delivery.

    Assembled-prompt scanning (including loaded skill content) plugs the gap from #3968: create-time
    scanning only covers the user-supplied prompt field; skill content loaded at runtime was never scanned,
    so a malicious skill could carry an injection payload that reached the non-interactive (auto-approve)
    cron agent.
    """


def _resolve_cron_disabled_toolsets(cfg: dict) -> list[str]:
    """Toolsets a cron-spawned agent must never receive: ``messaging``/``clarify`` always
    (interactive); ``cronjob`` by default (loop prevention, not a security boundary —
    ``cron.allow_agent_scheduling: true`` lifts only that); ``agent.disabled_toolsets`` layered on
    top so per-job ``enabled_toolsets`` cannot widen past config.yaml's denylist.

    See #25752.
    """
    cron_cfg = (cfg or {}).get("cron") or {}
    if cron_cfg.get("allow_agent_scheduling"):
        disabled = ["messaging", "clarify"]
    else:
        disabled = ["cronjob", "messaging", "clarify"]
    agent_cfg = (cfg or {}).get("agent") or {}
    from agent.skill_utils import parse_config_string_list

    user_disabled = parse_config_string_list(agent_cfg.get("disabled_toolsets"))
    for name in user_disabled:
        name = str(name).strip()
        if name and name not in disabled:
            disabled.append(name)
    return disabled


def _merge_mcp_into_per_job_toolsets(per_job: list[str], cfg: dict) -> list[str]:
    """Layer enabled MCP servers onto a per-job ``enabled_toolsets`` allowlist (else a per-job list
    silently drops every MCP server). Mirrors ``_get_platform_tools``: ``no_mcp`` sentinel -> none
    (stripped); any MCP server already listed -> allowlist, add nothing; else union all enabled."""
    result = [t for t in per_job if t != "no_mcp"]
    if "no_mcp" in per_job:
        return result
    # lazy: avoid heavy hermes_cli import at module load; shares MCP-membership with gateway/CLI
    from hermes_cli.tools_config import enabled_mcp_server_names
    enabled_mcp = enabled_mcp_server_names(cfg)
    if set(result) & enabled_mcp:
        return result
    for name in sorted(enabled_mcp):
        if name not in result:
            result.append(name)
    return result


def _resolve_cron_enabled_toolsets(job: dict, cfg: dict) -> list[str]:
    """Toolset list for a cron job. Precedence: per-job ``enabled_toolsets`` (+ MCP merge) >
    ``cron`` platform config (``_get_platform_tools``, which strips _DEFAULT_OFF_TOOLSETS so fresh
    installs run without ``moa``). A lookup failure fails CLOSED: the run errors out.

    1. Per-job ``enabled_toolsets`` (set via ``cronjob`` tool on create/update). Keeps the agent's
    job-scoped toolset override intact — #6130. Enabled MCP servers are layered on per
    ``_merge_mcp_into_per_job_toolsets`` so a native-toolset allowlist does not silently strip MCP tools. 2.
    Mirrors gateway behavior (``_get_platform_tools(cfg, platform_key)``) so users can gate cron toolsets
    globally without recreating every job. 3. Never ``None``: AIAgent reads ``None`` as "every
    toolset", so an unreadable ``platform_toolsets.cron`` restriction would hand an unattended job
    the full default set (#111380). The raise reaches ``run_job``'s failure path, which records the
    error on the job and opens an incident, so the operator sees it instead of a widened run.
    """
    per_job = job.get("enabled_toolsets")
    if per_job:
        return _merge_mcp_into_per_job_toolsets(list(per_job), cfg or {})
    try:
        from hermes_cli.tools_config import _get_platform_tools  # lazy: avoid heavy import at cron module load
        return sorted(_get_platform_tools(cfg or {}, "cron"))
    except Exception as exc:
        raise RuntimeError(
            "Cron toolset resolution failed, so this run was refused rather than given every "
            f"tool. Check `platform_toolsets.cron` in config.yaml (`hermes cron doctor`): {exc}"
        ) from exc


def _resolve_job_reasoning_config(job: dict, cfg: dict, model: str) -> dict | None:
    """Effective reasoning config for a cron run. A per-job ``reasoning_effort`` pin beats global
    and per-model config and is model-independent by design (also governs an auth-fallback swap);
    clamping stays with provider transports. An unparseable pin warns and falls back to config."""
    from hermes_constants import parse_reasoning_effort, resolve_reasoning_config

    pinned = job.get("reasoning_effort")
    if pinned is not None:
        parsed = parse_reasoning_effort(pinned)
        if parsed is not None:
            logger.info("Job '%s': using per-job reasoning_effort '%s'", job.get("id", "?"), pinned)
            return parsed
        logger.warning(
            "Job '%s': invalid stored reasoning_effort %r — ignoring the pin "
            "and falling back to config resolution. Fix with `cronjob "
            "action=update job_id=%s reasoning_effort=<level>` (valid: none, "
            "minimal, low, medium, high, xhigh, max, ultra).",
            job.get("id", "?"),
            pinned,
            job.get("id", "?"))
    return resolve_reasoning_config(cfg if isinstance(cfg, dict) else {}, str(model))


from cron.jobs import (
    _ensure_cron_dir, advance_next_runs, claim_dispatch, claim_job_for_fire, fire_claim_fence,
    clear_run_claim, get_due_jobs, heartbeat_fire_claim, heartbeat_run_claim, mark_job_run,
    save_job_output, self_removal_delivery_allowed, self_removal_delivery_scope, use_cron_store)
from cron.execution_identity import enter_cron_execution, exit_cron_execution
from cron.executions import (
    _TERMINAL_STATES, HANDOFF_ADOPTION_GRACE_SECONDS, create_execution, finish_execution,
    get_execution, mark_execution_handoff_pending, mark_execution_running,
    recover_interrupted_executions, terminalize_dead_owner)

# Response marker that suppresses delivery (output is still saved locally for audit).
SILENT_MARKER = "[SILENT]"

# Agent-declared failure marker for cron runs. Unlike SILENT, it is deliberately strict so a
# report that merely quotes the token cannot turn a healthy run into a failed one.
CRON_FAILURE_MARKER = "[CRON_FAILURE]"


def _cron_failure_marker_error(text: str) -> Optional[str]:
    """Return failure evidence when an agent response declares a cron failure.

    Only the exact, standalone first line is control text. The caller keeps the complete response
    in the saved run output while routing this evidence through normal failure bookkeeping.
    """
    lines = (text or "").splitlines()
    if not lines or lines[0].rstrip() != CRON_FAILURE_MARKER:
        return None
    evidence = "\n".join(lines[1:]).strip()
    return evidence or "Cron agent reported failure."


def _is_cron_silence_response(text: str) -> bool:
    """True when a cron final response should suppress delivery: ``[SILENT]`` (or SILENT /
    NO_REPLY / NO REPLY) as the whole response OR its own first/last line — NOT mid-sentence.
    Shares the webhook-lane matcher in :mod:`gateway.response_filters` so the two cannot drift.

    Recognizes the bracketed ``[SILENT]`` sentinel (whole-response, first line, or last line) plus the
    bracketless ``SILENT`` / ``NO_REPLY`` / ``NO REPLY`` variants the model emits when it drops the brackets
    (#51438, #46917). Whitespace-trimmed and case-insensitive. A token buried mid-sentence is treated as
    real content and delivered.
    """
    from gateway.response_filters import is_autonomous_silence_response

    return is_autonomous_silence_response(text)

# Persistent pool for parallel cron jobs: tick() submits and returns; long jobs never block it.
# Keyed by profile home: one host gateway multiplexes every profile, and ``max_parallel_jobs`` is a
# per-profile config key — a single process-global pool is sized by whichever profile ticked first
# and then imposes that limit on all the others.
_parallel_pools: Dict[str, concurrent.futures.ThreadPoolExecutor] = {}
_parallel_pool_max_workers: Dict[str, Optional[int]] = {}


def _inflight_key(job_id: str, home: Optional[Union[Path, str]] = None) -> tuple:
    """``(home key, job id)`` — the identity of one in-flight cron run.

    ONE gateway process ticks every profile, so a job id alone is not unique: two profiles
    routinely carry identically named jobs (``daily-brief``) and sharing a key made the second
    profile's run read as a duplicate of the first and get skipped. ``home`` defaults to the
    active cron scope's home, which the ticker binds per profile for the whole tick.
    """
    return (hermes_home_key(home) if home is not None else hermes_home_key(_get_hermes_home()), job_id)


# Home key -> the real home Path that produced it. ``hermes_home_key`` normcases (it lower-cases on
# Windows), so ``Path(key[0])`` is a case-folded path that matches nothing else on disk; bookkeeping
# that needs the profile home reads it here instead of reconstructing it from the key.
_inflight_home_paths: Dict[str, Path] = {}


def _remember_inflight_home(home: Path) -> Path:
    """Record ``home`` under its key so a claim can be mapped back to a usable path."""
    _inflight_home_paths[hermes_home_key(home)] = home
    return home


def _inflight_home_path(home_key: str) -> Path:
    """Real home Path for an in-flight key's home half (falls back to the key itself)."""
    return _inflight_home_paths.get(home_key) or Path(home_key)


# In-flight state below is keyed by ``_inflight_key``; the public accessors still report bare job
# ids so host-wide consumers (shutdown drain, idle-exit, metrics) see the union across profiles.
_running_job_ids: set = set()
_running_fire_owners: dict[tuple, dict[object, tuple[Optional[str], Path]]] = {}
# Parent gateway threads synchronously waiting on restart-safe scope workers.
# Shutdown must not misclassify these as ownerless in-process runs: the tool
# process sweep cannot reach the worker's transient scope.
_restart_safe_waiter_job_ids: set = set()
# in-flight key -> pid of the restart-safe external worker executing it (absent for in-process
# runs), so a drain observer can name the process holding the gateway open.
_running_worker_pids: dict[tuple, int] = {}
# In-flight keys whose external worker runs in its OWN transient systemd scope
# (``GatewayChildDispatch.mode == "scoped"``). Only that worker outlives a gateway restart: a
# systemd stop kills the service cgroup, so a ``degraded`` direct subprocess (no reachable user
# bus) dies with the gateway even though it is external. The gateway's restart after-turn wait
# skips exactly this set — see ``get_restart_wait_cron_counts``.
_scope_isolated_job_ids: set = set()
_running_lock = threading.Lock()

# Per in-flight id: time.time() claim instant + the future owning its release (``_FUTURE_PENDING``
# until pool.submit returns). Past-allowance with no live future = leak; the sweep force-releases.
_running_since: dict = {}
# in-flight key -> stale-inflight allowance (s), resolved once per run by get_wedged_job_ids.
_running_allowance_s: dict = {}
_running_futures: dict = {}

# Installed in ``_running_futures`` at claim time so a sweep landing before ``pool.submit`` returns
# never sees ``missing`` and releases a claim about to get its future.
_FUTURE_PENDING = object()
# Identity tokens fence cleanup and late future attachment after recovery.
_running_registration_owners: dict = {}

# Forced-release count/history for ``get_inflight_guard_stats()``; mirrored to JSONL for probes.
_forced_release_count: int = 0
_forced_releases: list = []
_FORCED_RELEASE_HISTORY = 20

# Stale-allowance floor (minutes); per-job allowance is max(2 * interval, this).
_INFLIGHT_MIN_ALLOWANCE_MINUTES = 30.0


# Execution tokens (``_running_fire_owners`` identity keys) force-interrupted at shutdown; see
# ``mark_running_jobs_interrupted``. ``run_one_job`` checks its OWN token before writing
# ``last_status`` so a still-running agent thread can't overwrite "interrupted" with a false "ok".
# Token keying scopes the flag to one execution (recurring jobs reuse IDs); legacy paths without a
# fire owner fall back to the bare job ID.
# ``run_one_job``'s own completion path checks its OWN token before writing ``last_status`` so a cron agent
# thread that keeps running in-process after its tool was killed out from under it — and produces a
# plausible-looking final response from truncated output — can never overwrite the interrupted status with a
# false "ok" (#60432). Token keying keeps an interruption scoped to that exact execution: a later run of the
# same job ID (recurring jobs reuse the ID every fire) must not inherit the stale flag.
_interrupted_job_ids: set = set()


class _CancelEventLike(Protocol):
    """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""

    def is_set(self) -> bool: ...
    def set(self) -> None: ...


class _CombinedCancelEvent:
    """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat
    ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.
    """

    def __init__(self, *events: Optional["_CancelEventLike"]) -> None:
        self._events = [event for event in events if event is not None]

    def is_set(self) -> bool:
        return any(event.is_set() for event in self._events)

    def set(self) -> None:
        for event in self._events:
            event.set()


def get_running_job_ids() -> "frozenset[str]":
    """Thread-safe snapshot of executing job IDs (dispatch until ``_process_job`` returns). Read by
    the gateway shutdown drain, otherwise blind to cron work (runs outside ``_running_agents``).

    _drain_active_agents``) reads this to treat in-flight cron work as active the same way it already treats
    in-flight chat sessions via ``_running_agents`` — cron jobs run through their own thread pool here,
    entirely outside that dict, so without this the drain is structurally blind to them (#60432).
    """
    with _running_lock:
        return frozenset(key[1] for key in _running_job_ids | _running_fire_owners.keys())


def get_running_job_details() -> list[dict]:
    """Per in-flight job: ``{"job_id", "elapsed_s", "worker_pid", "restart_safe"}`` (``worker_pid``
    None for in-process runs; ``restart_safe`` True when the worker owns its own systemd scope). The
    drain wait publishes this so ``hermes update`` can say WHICH job it is waiting on."""
    now = time.time()
    with _running_lock:
        return [
            {"job_id": key[1],
             "elapsed_s": round(now - _running_since[key], 1) if key in _running_since else None,
             "worker_pid": _running_worker_pids.get(key),
             "restart_safe": key in _scope_isolated_job_ids}
            for key in sorted(_running_job_ids | _running_fire_owners.keys())
        ]


def get_wedged_job_ids() -> "frozenset[str]":
    """In-flight job IDs older than their stale-inflight allowance (``max(2 * interval,
    cron.inflight_max_minutes)``) — the scheduler's own definition of a claim that can no longer be
    making progress. ``sweep_stale_inflight`` cannot release these while the worker thread is still
    alive (a delivery blocked on a dead transport, #115469), so the gateway restart drain reads this to
    skip them the way it skips wedged chat turns; restart is their remedy.
    """
    return frozenset(key[1] for key in _wedged_inflight_keys())


def _wedged_inflight_keys() -> "frozenset[tuple]":
    """In-flight keys behind :func:`get_wedged_job_ids`, before the projection to bare job IDs."""
    now = time.time()
    with _running_lock:
        ages = {key: now - started for key, started in _running_since.items() if key in _running_job_ids}
        allowances = {key: _running_allowance_s[key] for key in ages if key in _running_allowance_s}
    if not ages:
        return frozenset()
    floor_seconds = _inflight_min_allowance_minutes() * 60.0
    unresolved = [key for key in ages if key not in allowances]
    if unresolved:
        # One jobs.json parse per run, not per tick per job: the restart drain polls this every
        # 0.1 s on the event loop for the whole wait, and get_job() re-reads the file each call.
        # ``load_jobs`` reads the ACTIVE scope's store, so only claims from that home can have
        # their interval resolved; a claim from another profile falls back to the floor.
        by_id: dict = {}
        with contextlib.suppress(Exception):
            from cron.jobs import load_jobs
            by_id = {j.get("id"): j for j in load_jobs()}
        local_home = hermes_home_key(_get_hermes_home())
        with _running_lock:
            for key in unresolved:
                allowance = floor_seconds
                local_job = by_id.get(key[1]) if key[0] == local_home else None
                interval_minutes = _job_interval_minutes(local_job or {})
                if interval_minutes:
                    allowance = max(allowance, 2.0 * interval_minutes * 60.0)
                allowances[key] = allowance
                if key in _running_job_ids:  # released meanwhile -> don't resurrect the entry
                    _running_allowance_s[key] = allowance
    return frozenset(
        key for key, age in ages.items() if age >= max(allowances[key], floor_seconds))


def get_restart_wait_cron_counts() -> dict:
    """In-flight cron runs split for the gateway restart wait: ``awaitable``, ``wedged`` and
    ``restart_safe`` (scoped external worker, not wedged). The three are disjoint and sum to the
    number of in-flight runs.

    A ``scoped`` worker runs in a transient user scope outside the gateway cgroup, so a systemd
    stop/restart cannot reach it: the shutdown drain deliberately leaves it unmarked
    (``mark_running_jobs_interrupted``) and its final send rides the durable delivery queue, drained
    by whichever gateway is live next. Holding the restart wait for it only keeps the gateway in
    ``draining`` (refusing new turns) for up to ``agent.restart_after_turn_timeout``. A ``degraded``
    worker (external subprocess, no reachable user bus) stays in the gateway cgroup and dies with a
    systemd stop, so it stays awaitable.

    Counted per in-flight KEY, not per bare job ID: one multiplexed gateway can run the same job ID
    in two profiles, and projecting to bare IDs first lets one profile's wedged or scoped run hide
    the other profile's live run, which the restart would then kill (e.g. a scoped ``daily-brief``
    in profile A next to a degraded ``daily-brief`` in profile B).
    """
    wedged = _wedged_inflight_keys()
    with _running_lock:
        inflight = _running_job_ids | _running_fire_owners.keys()
        restart_safe = (_scope_isolated_job_ids & inflight) - wedged
    wedged = wedged & inflight
    return {
        "awaitable": len(inflight - wedged - restart_safe),
        "wedged": len(wedged),
        "restart_safe": len(restart_safe),
    }


def is_job_running(job_id: str, home: Optional[Union[Path, str]] = None) -> bool:
    """True when THIS process has an in-flight run of ``job_id`` FOR ``home`` (default: the active
    cron scope's home).

    The bare-id accessors above report the host-wide union so the shutdown drain and metrics see
    every profile's work. Liveness consumers must not: one process ticks every profile, so a
    ``daily-brief`` running in profile A would otherwise report profile B's idle ``daily-brief``
    as running and keep B's stale one-shot alive as a "possibly live run".
    """
    key = _inflight_key(job_id, home)
    with _running_lock:
        return key in _running_job_ids or key in _running_fire_owners


def _record_external_cron_worker(job_id: str, pid: Any, *, scope_isolated: bool) -> None:
    """Record the external worker holding *job_id* so a drain observer can name it, and whether
    its scope isolates it from this process.

    ``scope_isolated`` is the strong form of restart-safety: only a ``scoped`` dispatch
    (``GatewayChildDispatch``) puts the worker in its own transient systemd scope, outside the
    gateway cgroup. A ``degraded`` worker is still external but shares the cgroup and dies with a
    systemd stop, so it must NOT be listed as restart-safe. Called once per accepted handoff;
    both maps are cleared by ``release_running_job`` and by the worker waiter's ``finally``.
    """
    key = _inflight_key(job_id)
    with _running_lock:
        with contextlib.suppress(TypeError, ValueError):
            _running_worker_pids[key] = int(pid)
        if scope_isolated:
            _scope_isolated_job_ids.add(key)


def try_register_running_job(job_id: str, *, owner=None, future=_FUTURE_PENDING) -> bool:
    """Atomically add ``job_id`` to the in-flight set; False (caller must skip) if already mid-run.
    Single dedupe owner for ticker + manual runs (the fire claim's 300s TTL is outlived by real
    jobs). Callers MUST pair success with ``release_running_job`` in a ``finally``.
    Dispatchers pass a unique ``owner`` back on release. Direct workers register a running
    ``future`` atomically because no executor submission will replace the pending sentinel.

    This is the single dedupe owner shared by the ticker's ``_submit_with_guard`` and manual runs
    (``tools/cronjob_tools``): the fire claim alone cannot prevent a double-fire because its TTL (300s) is
    routinely outlived by real jobs, after which a manual ``cronjob(action='run')`` would claim successfully
    and run the same job concurrently (idea from #53395 by @izumi0uu).
    Registration also makes the run visible to ``get_running_job_ids`` (the gateway shutdown drain, #60432)
    and ``mark_running_jobs_interrupted``. Dedupe is per PROFILE: the key carries the active cron
    scope's home, so one multiplexing process never treats two profiles' same-named jobs as one.
    """
    from hermes_cli.backend_retirement import retirement

    key = _inflight_key(job_id, _remember_inflight_home(_get_hermes_home()))
    with retirement.work() as admitted, _running_lock:
        if not admitted or key in _running_job_ids:
            return False
        _running_job_ids.add(key)
        # Same critical section as the add: no window where an in-flight id lacks an age the sweep
        # can bound. Sentinel is replaced by the real future once ``pool.submit`` returns.
        _running_since[key] = time.time()
        _running_futures[key] = future
        _running_registration_owners[key] = owner
        return True


def release_running_job(
    job_id: str, home: Optional[Union[Path, str]] = None, *, owner=None,
) -> None:
    """Remove the registration unless an explicit ``owner`` has been replaced.

    ``home`` MUST be passed by any caller that does not run inside the same cron scope the claim
    was registered under. The scope is a ContextVar the ticker binds per profile: a pool worker
    releasing in a ``finally`` outside ``ctx.run`` resolves the default (launch) home instead, so
    the discard misses the real key and every secondary profile's claim leaks.
    """
    key = _inflight_key(job_id, home)
    with _running_lock:
        if owner is not None and _running_registration_owners.get(key) is not owner:
            return
        _running_registration_owners.pop(key, None)
        _running_job_ids.discard(key)
        _running_since.pop(key, None)
        _running_allowance_s.pop(key, None)
        _running_futures.pop(key, None)
        _running_worker_pids.pop(key, None)
        _scope_isolated_job_ids.discard(key)


def _inflight_min_allowance_minutes() -> float:
    """Stale allowance floor (min): ``cron.inflight_max_minutes``, else env escape hatch/default."""
    with contextlib.suppress(Exception):
        _ucfg = load_config() or {}
        _cfg_val = (
            _ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}
        ).get("inflight_max_minutes")
        if _cfg_val is not None:
            val = float(_cfg_val)
            if val > 0:
                return val
    raw = cron_env_setting("HERMES_CRON_INFLIGHT_MAX_MINUTES").strip()
    if raw:
        try:
            val = float(raw)
            if val > 0:
                return val
        except (ValueError, TypeError):
            logger.warning(
                "Invalid HERMES_CRON_INFLIGHT_MAX_MINUTES=%r; using default %s",
                raw,
                _INFLIGHT_MIN_ALLOWANCE_MINUTES)
    return _INFLIGHT_MIN_ALLOWANCE_MINUTES


# expr -> minutes; cadence never changes, so avoid re-evaluating croniter every tick.
_cron_interval_cache: dict = {}


def _cron_interval_minutes(expr: str) -> Optional[float]:
    """Cron expression cadence (gap between next two fires) in minutes; None -> floor allowance."""
    if expr in _cron_interval_cache:
        return _cron_interval_cache[expr]
    result = None
    ok = False
    with contextlib.suppress(Exception):
        from cron.jobs import _ensure_croniter

        ok = _ensure_croniter()
        if ok:
            from cron.jobs import croniter as _croniter
            from datetime import datetime

            base = datetime.now()
            it = _croniter(expr, base)
            first = it.get_next(datetime)
            second = it.get_next(datetime)
            gap = (second - first).total_seconds() / 60.0
            result = gap if gap > 0 else None
    # Cache a real cadence or a bad-expr None (croniter loaded, expr invalid: stable). Skip the
    # None from a transient croniter ImportError so it can't pin the floor allowance for the
    # process lifetime once the import recovers.
    if result is not None or ok:
        _cron_interval_cache[expr] = result
    return result


def _job_interval_minutes(job: dict) -> Optional[float]:
    """Best-effort job interval in minutes (None if unknown / one-shot -> floor). ``schedule`` is
    persisted as a parsed dict; the string path is only a fallback for programmatic callers."""
    with contextlib.suppress(Exception):
        schedule = job.get("schedule")
        if isinstance(schedule, str) and schedule.strip():
            from cron.jobs import parse_schedule

            schedule = parse_schedule(schedule) or {}
        if isinstance(schedule, dict):
            kind = schedule.get("kind")
            if kind == "interval":
                minutes = schedule.get("minutes")
                return float(minutes) if minutes else None
            if kind == "cron":
                return _cron_interval_minutes(str(schedule.get("expr") or ""))
    return None


def get_inflight_guard_stats() -> dict:
    """Probe-visible snapshot; non-zero ``forced_releases`` means a job wedged and was recovered."""
    now = time.time()
    with _running_lock:
        return {
            "running": sorted(key[1] for key in _running_job_ids),
            "running_ages_seconds": {
                key[1]: round(now - started, 1)
                for key, started in _running_since.items()
            },
            "forced_releases": _forced_release_count,
            "recent_forced_releases": list(_forced_releases)}


def _record_forced_release(job_id: str, name: str, age_seconds: float, allowance_seconds: float) -> None:
    """Persist a countable signal for one forced release (best-effort)."""
    entry = {
        "job_id": job_id,
        "name": name,
        "age_seconds": round(age_seconds, 1),
        "allowance_seconds": round(allowance_seconds, 1),
        "at": _hermes_now().isoformat()}
    with _running_lock:
        _forced_releases.append(entry)
        del _forced_releases[:-_FORCED_RELEASE_HISTORY]
    try:
        path = _get_hermes_home() / "cron" / "inflight_forced_releases.jsonl"
        _ensure_cron_dir(path.parent)
        with open(path, "a", encoding="utf-8") as fh:
            fh.write(json.dumps(entry) + "\n")
    except Exception as e:  # never let telemetry break a tick
        logger.debug("Could not append forced-release record: %s", e)


def _latest_executions_for_releasable_claims() -> dict:
    """Latest durable execution per releasable-looking claim (missing/pending/done future), one
    indexed query so the healthy path pays no DB work. Snapshot under _running_lock — iterating
    the set while try_register/release mutate it raises RuntimeError.

    Scoped to the ACTIVE profile's claims: the executions ledger it queries is that home's.
    """
    local_home = hermes_home_key(_get_hermes_home())
    with _running_lock:
        claim_futures = {
            key[1]: _running_futures.get(key)
            for key in _running_job_ids if key[0] == local_home
        }
    candidates = [
        job_id for job_id, fut in claim_futures.items()
        if fut is None or fut is _FUTURE_PENDING or fut.done()
    ]
    if not candidates:
        return {}
    try:
        from cron.executions import latest_executions as _latest_execs
        return _latest_execs(candidates)
    except Exception:
        return {}


def _row_belongs_to_claim(row: dict, claim_started: float) -> bool:
    """True when the ledger row was claimed at/after this in-memory claim. An older terminal row is
    the PREVIOUS run's (try_register->create_execution window); releasing on it would
    double-dispatch. Unparseable timestamps fail closed (the age path still bounds the claim)."""
    claimed_at = row.get("claimed_at")
    if not claimed_at:
        return False
    try:
        from cron.jobs import _ensure_aware as _ensure_aware_ts
        row_ts = _ensure_aware_ts(datetime.fromisoformat(claimed_at))
        return row_ts.timestamp() >= claim_started
    except (ValueError, TypeError, OSError):
        return False


def _record_stale_release(job: dict, job_id: str, age: float, allowance: float, fut, reason: str) -> None:
    """WARNING log + probe record for one forced release, then ``last_error`` unless the ledger
    already holds the run's real outcome or the job has a finite repeat budget."""
    name = job.get("name") or job_id
    future_state = "pending" if fut is _FUTURE_PENDING else "missing" if fut is None else "finished"
    logger.warning(
        "cron.inflight.forced_release event=forced_release reason=%s job='%s' "
        "id=%s age=%.0fs allowance=%.0fs future=%s — stale in-flight claim "
        "released; the job was skipping every fire with 'already running'",
        reason, name, job_id, age, allowance, future_state)
    _record_forced_release(job_id, name, age, allowance)
    # Ledger already records how the run ended: mark_job_run here would clobber an honest
    # ok status with a synthetic failure or double-write a failure.
    if reason == "ledger-terminal":
        return
    # Age release may lack a ledger row, so last_error is how it surfaces. But a forced release
    # is NOT a real run: never consume a finite repeat budget or let mark_job_run auto-delete.
    repeat = job.get("repeat") or {}
    if isinstance(repeat, dict) and repeat.get("times") is not None:
        logger.warning(
            "cron.inflight.forced_release.job_untouched job='%s' id=%s — "
            "finite-repeat job released without mark_job_run (repeat budget "
            "preserved); row left in place so it re-fires normally",
            name, job_id)
        return
    try:
        mark_job_run(
            job_id, False,
            f"Stale in-flight claim force-released after {age / 60:.1f}m "
            f"(allowance {allowance / 60:.1f}m); previous run never released "
            f"the scheduler in-flight guard")
    except Exception as e:
        logger.warning("Could not record forced release for job %s: %s", job_id, e)


def sweep_stale_inflight(due_jobs: Optional[list] = None) -> list:
    """Force-release in-flight claims that can no longer be making progress; returns released ids.

    Stale = older than ``max(2 * interval, floor)`` AND (no live future — submit path hung before
    ``pool.submit`` returned — or finished without discarding the id) — or the claim's OWN ledger
    row is terminal regardless of age. Each release logs WARNING ``event=forced_release``, bumps
    the probe counter, mirrors JSONL, and writes ``last_error``.
    """
    global _forced_release_count

    by_id = {j.get("id"): j for j in (due_jobs or []) if isinstance(j, dict)}
    floor_seconds = _inflight_min_allowance_minutes() * 60.0
    now = time.time()
    stale: list = []
    from cron.executions import _TERMINAL_STATES as _terminal_states

    _latest = _latest_executions_for_releasable_claims()
    _local_home = hermes_home_key(_get_hermes_home())
    # Compute intervals OUTSIDE _running_lock so croniter doesn't block try_register/release.
    _intervals = {jid: _job_interval_minutes(j) for jid, j in by_id.items()}

    with _running_lock:
        for key in [k for k in _running_job_ids if k[0] == _local_home]:
            job_id = key[1]
            started = _running_since.get(key)
            if started is None:
                # Claim predates this guard — adopt it; sweepable one allowance from now.
                _running_since[key] = now
                continue
            age = now - started
            interval_minutes = _intervals.get(job_id)
            allowance = floor_seconds
            if interval_minutes:
                allowance = max(allowance, 2.0 * interval_minutes * 60.0)
            fut = _running_futures.get(key)
            if fut is _FUTURE_PENDING:
                # Submit path hung before ``pool.submit`` returned — the wedge class; release it.
                pass
            elif fut is not None and not fut.done():
                continue  # genuinely still executing
            # Ledger reconciliation: a terminal row belonging to THIS claim proves it stale even
            # inside its age allowance. Row must be this claim's, else a recurring job's previous
            # run would double-dispatch a fresh claim.
            latest = _latest.get(job_id)
            if (
                latest is not None
                and latest.get("status") in _terminal_states
                and _row_belongs_to_claim(latest, started)
            ):
                reason = "ledger-terminal"
            elif age >= allowance:
                reason = "age"
            else:
                continue
            _running_registration_owners.pop(key, None)
            _running_job_ids.discard(key)
            _running_since.pop(key, None)
            _running_allowance_s.pop(key, None)
            _running_futures.pop(key, None)
            _forced_release_count += 1
            stale.append((job_id, age, allowance, fut, reason))

    for job_id, age, allowance, fut, _reason in stale:
        _record_stale_release(by_id.get(job_id) or {}, job_id, age, allowance, fut, _reason)
    return [s[0] for s in stale]


def mark_running_jobs_interrupted(
    reason: str, *, only_owners: Optional[set] = None,
) -> list:
    """Best-effort: mark every in-flight cron job interrupted; returns the job IDs marked.

    Called by gateway shutdown right after ``process_registry.kill_all()``: a job whose tool was
    killed must never report success. ``only_owners`` (``(job_id, fire_owner)`` pairs) restricts
    marking. Tokens go into ``_interrupted_job_ids`` BEFORE ``last_status`` is written so
    ``run_one_job`` sees them.
    """
    with _running_lock:
        restart_safe_waiters = set(_restart_safe_waiter_job_ids)
        active_fires = [
            (token, key, owner, profile_home)
            for key, executions in _running_fire_owners.items()
            if key not in restart_safe_waiters
            for token, (owner, profile_home) in executions.items()
        ]
        if only_owners is not None:
            active_fires = [fire for fire in active_fires if (fire[1][1], fire[2]) in only_owners]
        registered_keys = {key for _t, key, _o, _p in active_fires}
        if only_owners is None:
            # The key's home half IS the profile home this claim belongs to — the only record of
            # it for a claim that never reached ``_running_fire_owners``. Read the real Path back
            # from the key, never ``Path(key[0])``: the key is normcased.
            active_fires.extend(
                (None, key, None, _inflight_home_path(key[0]))
                for key in (
                    _running_job_ids - registered_keys - restart_safe_waiters
                )
            )
        _interrupted_job_ids.update(
            token if token is not None else key
            for token, key, _owner, _profile_home in active_fires
        )
    marked = []
    for _token, key, fire_owner, profile_home in active_fires:
        job_id = key[1]
        if not fire_owner:
            logger.warning(
                "Job '%s' interrupted before its durable fire owner was registered; "
                "leaving persisted state untouched",
                job_id)
            # Still report it: shutdown uses the returned IDs for the interrupted-cron notice. The
            # in-memory flag WAS recorded above; only the persisted last_status write is skipped.
            # See #82232.
            marked.append(job_id)
            continue
        try:
            with use_cron_store(profile_home):
                if mark_job_run(
                    job_id, False, reason, expected_fire_owner=fire_owner):
                    marked.append(job_id)
        except Exception as e:
            logger.warning("Failed to mark job %s interrupted: %s", job_id, e)
    return marked


def _is_interrupted(job_id: str, token: Optional[object] = None) -> bool:
    """Non-destructive peek: has shutdown marked THIS execution interrupted? Used before deciding
    what to deliver; does not clear the flag (the authoritative pre-``last_status`` check needs it).
    ``token`` scopes to one execution so a fresh run reusing the job ID isn't poisoned."""
    key = _inflight_key(job_id)
    with _running_lock:
        if token is not None and token in _interrupted_job_ids:
            return True
        return key in _interrupted_job_ids


def _consume_interrupted_flag(job_id: str, token: Optional[object] = None) -> bool:
    """Return True and clear the flag if shutdown marked THIS execution interrupted. Called right
    before ``last_status`` is written; consuming stops the flag leaking into a later run."""
    key = _inflight_key(job_id)
    with _running_lock:
        hit = False
        if token is not None and token in _interrupted_job_ids:
            _interrupted_job_ids.discard(token)
            hit = True
        if key in _interrupted_job_ids:
            _interrupted_job_ids.discard(key)
            hit = True
        return hit


def _inactivity_watchdog_loop(
    *, get_idle_seconds: Callable[[], float], limit_s: float, poll_s: float, stop: threading.Event,
    future_done: Callable[[], bool], meter: Optional[AwakeIdleMeter] = None,
) -> bool:
    """Poll idle time until limit (-> True), stop, or the future completes (-> False). Uses
    ``threading.Event.wait``, not asyncio, so a blocked event loop cannot disable the watchdog.

    Driven by ``threading.Event.wait`` (a kernel timeout), not asyncio, so a blocked event-loop /
    ``run_job`` thread cannot disable this watchdog the way ``asyncio.sleep`` / ``wait_for`` would (family A
    of #94285 — the 4118s-idle-on-a-600s-limit cron hang). Returns True when *limit_s* of inactivity was
    observed.
    """
    # A sleeping host freezes the job with it, so time asleep never counts as inactivity.
    if meter is None:
        meter = AwakeIdleMeter()
    while not stop.wait(poll_s):
        if future_done():
            return False
        try:
            idle = float(get_idle_seconds() or 0.0)
        except Exception:
            idle = 0.0
        if meter.measure(idle) >= limit_s:
            return True
    return False


def _cron_inactivity_seconds() -> float:
    """Parse HERMES_CRON_TIMEOUT (seconds). 0 = unlimited; bad input = 600. Shared by the
    inactivity monitor and the cwd-lock bound so they can't drift: the lock bound must stay >= the
    inactivity limit or waiters fail while a healthy holder runs."""
    raw = cron_env_setting("HERMES_CRON_TIMEOUT").strip()
    if not raw:
        return 600.0
    try:
        return float(raw)
    except (ValueError, TypeError):
        logger.warning("Invalid HERMES_CRON_TIMEOUT=%r; using default 600s", raw)
        return 600.0


def _get_parallel_pool(max_workers: Optional[int]) -> concurrent.futures.ThreadPoolExecutor:
    """Return (or create) the persistent parallel pool for the ACTIVE profile.

    Pools are per profile home because ``cron.max_parallel_jobs`` is a per-profile config key:
    a single process-global pool is created by whichever profile the multiplexing gateway ticks
    first and then silently imposes that profile's limit on every other profile.
    """
    home = hermes_home_key(_remember_inflight_home(_get_hermes_home()))
    pool = _parallel_pools.get(home)
    if pool is None or _parallel_pool_max_workers.get(home) != max_workers:
        if pool is not None:
            pool.shutdown(wait=False, cancel_futures=False)
        pool = concurrent.futures.ThreadPoolExecutor(
            max_workers=max_workers, thread_name_prefix="cron-parallel")
        _parallel_pools[home] = pool
        _parallel_pool_max_workers[home] = max_workers
    return pool


def discard_parallel_pools(home_keys) -> None:
    """Drop the parallel pools of homes this process no longer ticks.

    Pools are per home and used to live until ``atexit``: a host that serves many profiles — or
    churns them — accumulated one ThreadPoolExecutor and its live ``cron-parallel`` threads per
    home ever ticked, and a deleted profile's pool was never reclaimed. ``wait=False`` so the
    ticker is never blocked; queued work still drains before the executor dies.
    """
    for key in list(home_keys):
        pool = _parallel_pools.pop(key, None)
        _parallel_pool_max_workers.pop(key, None)
        if pool is not None:
            pool.shutdown(wait=False, cancel_futures=False)


def _shutdown_parallel_pool() -> None:
    """Shut down every profile's persistent pool on process exit."""
    for pool in list(_parallel_pools.values()):
        pool.shutdown(wait=True, cancel_futures=False)
    _parallel_pools.clear()
    _parallel_pool_max_workers.clear()


atexit.register(_shutdown_parallel_pool)
# Per-fire usage audit log; resolves via _get_hermes_home() so profile-scoped paths work.
def _usage_audit_path() -> Path:
    return _get_hermes_home() / "cron" / "usage_audit.jsonl"


def _utcnow_iso_ms() -> str:
    """RFC3339 UTC timestamp with millisecond precision and 'Z' suffix."""
    now = datetime.now(timezone.utc)
    return now.strftime("%Y-%m-%dT%H:%M:%S.") + f"{now.microsecond // 1000:03d}Z"


def _write_usage_audit(record: dict) -> None:
    """Append one JSONL line to cron/usage_audit.jsonl. NEVER raises — a logger bug must not
    break cron jobs (the whole write is inside one try)."""
    try:
        path = _usage_audit_path()
        _ensure_cron_dir(path.parent)
        line = json.dumps(record, ensure_ascii=False)
        with open(path, "a", encoding="utf-8") as f:
            f.write(line + "\n")
    except Exception as e:
        logger.warning("usage_audit write failed: %s", e)


def _interpreter_shutting_down(exc: Optional[BaseException] = None) -> bool:
    """True when the interpreter is finalizing (tick fired during gateway teardown): concurrent.
    futures/asyncio refuse new work, so delivery attempts only pollute errors.log — callers skip
    with a warning. ``exc`` lets an already-raised scheduling error count as a shutdown signal.

    A cron tick can fire while the gateway is tearing down — SIGTERM from ``hermes update`` / ``hermes
    gateway stop`` / systemd restart, or an OOM-kill. Once finalization starts, ``concurrent.futures``
    refuses new work with ``RuntimeError: cannot schedule new futures after interpreter shutdown`` and
    asyncio's default executor is gone, so *any* attempt to schedule delivery (live-adapter,
    ``asyncio.run``, or a fresh pool) is doomed and only pollutes ``errors.log`` with a traceback. See
    #55924, #58720.
    """
    from tools.interpreter_shutdown import interpreter_shutting_down

    return interpreter_shutting_down(exc)


# Module override hook for tests / emergency monkeypatches.
_hermes_home: Path | None = None


def _get_hermes_home() -> Path:
    """Hermes home at call time (honouring the test override). Cron is per-profile: never freeze
    this at import or anchor it at the shared default root — either breaks profile isolation.

    Cron is per-profile by design (#4707): the in-process ticker runs inside a profile-scoped gateway, so
    resolving the active HERMES_HOME at call time means a profile's jobs are stored AND executed under that
    profile's home (its .env, config.yaml, scripts, skills).
    """
    return _hermes_home or get_hermes_home()


def _get_lock_paths() -> tuple[Path, Path]:
    """Resolve cron lock paths at call time so profile/env changes are honored."""
    hermes_home = _get_hermes_home()
    lock_dir = hermes_home / "cron"
    return lock_dir, lock_dir / ".tick.lock"


# Errnos that mean "another ticker (or manual tick) holds the tick lock", as opposed to a real failure
# opening/locking the file. Everything else — most importantly EMFILE/ENFILE (fd exhaustion, #87644) and
# EACCES on open() — must be surfaced, never swallowed as lock contention.
def _is_lock_contention_errno(err: OSError) -> bool:
    """True when *err* from the lock syscall means another ticker holds the lock (POSIX flock:
    EWOULDBLOCK/EAGAIN, EACCES on some NFS; msvcrt.locking: EACCES/EDEADLK). Everything else —
    notably EMFILE/ENFILE and EACCES on open() — must be surfaced, never swallowed as contention."""
    if err.errno is None:
        return False
    if fcntl is not None:
        return err.errno in (errno.EWOULDBLOCK, errno.EAGAIN, errno.EACCES)
    if msvcrt is not None:
        return err.errno in (errno.EACCES, errno.EDEADLK)
    return False


def _is_fd_exhaustion_text(text: str) -> bool:
    """Text half of _is_fd_exhaustion (shared with the CLI hint)."""
    lowered = text.lower()
    return "too many open files" in lowered or "emfile" in lowered


def _is_fd_exhaustion(exc: BaseException) -> bool:
    """True when *exc* indicates fd exhaustion: EMFILE/ENFILE errno, or the "Too many open files"
    wording for wrapped exceptions (load_jobs wraps the OSError in a RuntimeError).

    See #87644.
    """
    if isinstance(exc, OSError) and exc.errno in (errno.EMFILE, errno.ENFILE):
        return True
    return _is_fd_exhaustion_text(str(exc))


def _reclaim_fds_best_effort() -> None:
    """Best-effort fd reclamation: gc.collect() closes file objects stuck in reference cycles;
    apply_nofile_soft_limit() raises the RLIMIT_NOFILE soft limit for headroom. Never raises.

    The cron FD-leak family (#60859, #79742, #80792) leaks descriptors from abandoned workers/sessions. Two
    safe, idempotent levers:
    """
    with contextlib.suppress(Exception):
        import gc

        gc.collect()
    with contextlib.suppress(Exception):
        from hermes_cli.resource_limits import apply_nofile_soft_limit

        apply_nofile_soft_limit(None)


def drain_delivery_queue(adapters, loop) -> int:
    """Send queued worker results through this gateway's live adapters."""
    from cron.delivery_queue import _path, drain

    # Only restart-safe workers create the queue file.  Every gateway (macOS,
    # Windows, launchd, Docker) runs this housekeeping tick, so skip the sqlite
    # open/create entirely until a worker has actually queued something.
    if not _path().exists():
        return 0
    return drain(
        lambda queued_job, queued_content, queued_for_failure: _deliver_result(
            queued_job,
            queued_content,
            adapters=adapters,
            loop=loop,
            for_failure=queued_for_failure,
        )
    )


_DEFAULT_SCRIPT_TIMEOUT = 3600  # seconds (1 hour)
# Backward-compatible module override used by tests and emergency monkeypatches.
_SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT
_RUN_CLAIM_HEARTBEAT_SECONDS = 60.0
_FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS = _RUN_CLAIM_HEARTBEAT_SECONDS * 3
# Pause before re-sampling a fire-claim heartbeat miss; a genuinely re-owned claim misses twice.
_FIRE_CLAIM_MISS_CONFIRM_SECONDS = 1.0


def _cron_cleanup_timeout_seconds() -> float:
    """Return the wall-clock bound for cron post-run cleanup."""
    default = 10.0
    try:
        from hermes_cli.config import load_config

        cfg = load_config() or {}
        cron_cfg = cfg.get("cron", {}) if isinstance(cfg, dict) else {}
        configured = cron_cfg.get("cleanup_timeout_seconds")
        if configured is not None:
            timeout = float(configured)
            if timeout >= 0:
                return timeout
    except Exception as exc:
        logger.debug("Failed to load cron cleanup timeout from config: %s", exc)
    return default


def _run_cron_cleanup_with_timeout(
    cleanup, *, job_id: str, label: str, timeout_seconds: Optional[float] = None,
) -> bool:
    """Run fallible post-run cleanup without permanently wedging a cron ID."""
    timeout = (_cron_cleanup_timeout_seconds() if timeout_seconds is None else float(timeout_seconds))
    if timeout <= 0:
        try:
            cleanup()
            return True
        except (Exception, KeyboardInterrupt) as exc:
            logger.debug("Job '%s': %s failed: %s", job_id, label, exc)
            return False

    done = threading.Event()
    error: list[BaseException] = []

    def _runner() -> None:
        try:
            cleanup()
        except BaseException as exc:
            error.append(exc)
        finally:
            done.set()

    # Daemon thread is deliberate: unlike ThreadPoolExecutor workers it is not joined at interpreter
    # exit if cleanup never returns, so the gateway can still shut down.
    worker = threading.Thread(
        target=ctx_bound(_runner), name=f"cron-cleanup-{job_id}", daemon=True)
    worker.start()
    if not done.wait(timeout):
        logger.error(
            "Job '%s': %s exceeded %.1fs; abandoning cleanup so future runs remain dispatchable",
            job_id,
            label,
            timeout)
        return False
    if error:
        logger.debug("Job '%s': %s failed: %s", job_id, label, error[0])
        return False
    return True


class _BoundedCronSessionDB:
    """Proxy SessionDB cleanup calls through the cron cleanup timeout; after the first failure or
    timeout all later calls fail immediately (a damaged connection leaks at most one worker)."""

    def __init__(self, session_db, job_id: str):
        self._session_db = session_db
        self._job_id = job_id
        self._disabled = False

    def __getattr__(self, name):
        target = getattr(self._session_db, name)
        if not callable(target):
            return target

        def _bounded(*args, **kwargs):
            if self._disabled:
                raise RuntimeError("session finalization disabled after prior cleanup failure")

            result = {}

            def _call():
                try:
                    result["value"] = target(*args, **kwargs)
                except BaseException as exc:
                    result["error"] = exc
                    raise

            ok = _run_cron_cleanup_with_timeout(
                _call, job_id=self._job_id, label=f"session finalization ({name})")
            if not ok:
                error = result.get("error")
                if error is not None:
                    raise error
                # No error yet not complete == timeout: disable so later steps fail fast.
                self._disabled = True
                raise TimeoutError(f"session finalization method {name} timed out")
            return result.get("value")

        return _bounded


def _job_doc_header(job_name: str, job_id: str, now_iso: str, mode: str) -> str:
    """Common markdown header for the short-circuit run docs (no_agent / monitor)."""
    return (
        f"# Cron Job: {job_name}\n\n"
        f"**Job ID:** {job_id}\n"
        f"**Run Time:** {now_iso}\n"
        f"**Mode:** {mode}\n"
    )


def _resolve_job_workdir(job: dict, job_id: str) -> Optional[str]:
    """Configured job workdir, or None when unset / no longer a directory (logged)."""
    workdir = (job.get("workdir") or "").strip() or None
    if workdir and not Path(workdir).is_dir():
        logger.warning(
            "Job '%s': configured workdir %r no longer exists — running without it",
            job_id, workdir)
        return None
    return workdir


def _run_no_agent_job(
    job: dict, job_id: str, job_name: str, cancel_event,
) -> tuple[bool, str, str, Optional[str]]:
    """no_agent short-circuit — the script IS the job (no AIAgent, no tokens). stdout → delivered
    verbatim; empty stdout or wakeAgent=false → silent success; non-zero exit/timeout → error alert.
    """
    # Load .env first so auto-delivery can resolve *_HOME_CHANNEL: the agent path's per-run dotenv
    # reload never runs for no_agent jobs. Does not override existing values.
    try:
        from hermes_cli.env_loader import load_hermes_dotenv

        load_hermes_dotenv(hermes_home=_get_hermes_home())
    except Exception:
        logger.debug("Job '%s': no_agent .env reload failed", job_id, exc_info=True)

    script_path = job.get("script")
    # Legacy/hand-edited no_agent job without a script: pause it, or it re-fires every tick.
    if not str(script_path or "").strip():
        from cron.jobs import NO_AGENT_WITHOUT_SCRIPT_ERROR

        return _block_and_pause_job(job_id, job_name, NO_AGENT_WITHOUT_SCRIPT_ERROR)

    # Pass workdir as subprocess cwd; never os.chdir() (leaks into concurrent gateway sessions).
    _job_workdir = _resolve_job_workdir(job, job_id)
    try:
        ok, output = _run_job_script_with_claim_heartbeat(
            job, script_path, workdir=_job_workdir, cancel_event=cancel_event)
    except Exception as exc:
        logger.exception("Job '%s': script execution raised unexpectedly", job_id)
        ok, output = False, f"Script execution failed: {exc}"

    now_iso = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
    header = _job_doc_header(job_name, job_id, now_iso, "no_agent (script)")

    if not ok:
        # Deliver the error: a silently broken watchdog is the worst-case outcome.
        alert = (
            f"⚠ Cron watchdog '{job_name}' script failed\n\n"
            f"{output}\n\n"
            f"Time: {now_iso}"
        )
        return False, f"{header}**Status:** script failed\n\n{output}\n", alert, output

    # wakeAgent=false is a silent signal, same as empty stdout.
    if not _parse_wake_gate(output):
        logger.info("Job '%s' (no_agent): wakeAgent=false gate — silent run", job_id)
        return True, f"{header}**Status:** silent (wakeAgent=false)\n", SILENT_MARKER, None

    if not output.strip():
        logger.info("Job '%s' (no_agent): empty stdout — silent run", job_id)
        return True, f"{header}**Status:** silent (empty output)\n", SILENT_MARKER, None

    return True, f"{header}\n---\n\n{output}\n", output, None


def _apply_monitor_gate(
    job: dict, job_id: str, job_name: str, extra_prompt: Optional[str],
) -> tuple[Optional[tuple], Optional[str], Optional[str]]:
    """Monitor gate (hash-suppressed change detection). Must run BEFORE any agent machinery so an
    unchanged tick costs no LLM/delivery. Returns ``(early_result | None, extra_prompt,
    monitor_context)``. Monitor context is runtime data and must remain distinct from a
    user-authored ``extra_prompt`` so the prompt scanner keeps its strict user-input boundary.
    """
    from cron.monitor import check_monitor, job_has_monitor

    if not job_has_monitor(job):
        return None, extra_prompt, None
    _mon = check_monitor(job)
    _mon_now = _hermes_now().strftime("%Y-%m-%d %H:%M:%S")
    header = _job_doc_header(job_name, job_id, _mon_now, "monitor")
    if not _mon.ok:
        # Source failure is an ERROR, never a change: alert so a broken monitor can't silently
        # stop watching. Stored hash untouched.
        logger.error("Job '%s': monitor source failed: %s", job_id, _mon.error)
        _mon_alert = (
            f"⚠ Cron monitor '{job_name}' source failed\n\n"
            f"{_mon.error}\n\n"
            f"Time: {_mon_now}"
        )
        return (
            False, f"{header}**Status:** monitor source failed\n\n{_mon.error}\n", _mon_alert, _mon.error,
        ), extra_prompt, None
    if not _mon.changed:
        # Unchanged: silent no_change tick (ledger doc kept; SILENT_MARKER blocks delivery).
        logger.info("Job '%s': monitor output unchanged — suppressing agent run", job_id)
        return (
            True, f"{header}**Status:** no_change (agent run suppressed)\n", SILENT_MARKER, None,
        ), extra_prompt, None
    # Changed (or first run): pass monitor output through the runtime-data seam. Keep any manual
    # per-run prompt separate: it remains user input and is therefore still strict-scanned.
    return None, extra_prompt, _mon.context_block


@dataclass
class _CronJobConfig:
    """Config-derived inputs for one agent-backed cron run."""

    cfg: dict
    model: str
    model_cfg: Any
    cron_default_provider: str


def _load_cron_job_config(job: dict, job_id: str, job_name: str) -> _CronJobConfig:
    """Load config.yaml and resolve the run's model: per-job pin > cron.model (fleet default) >
    the main agent model (config ``model:``, then HERMES_MODEL). Re-read every tick (no cache) so
    ``hermes cron edit --model`` and ``hermes model`` both apply next tick."""
    model = job.get("model") or cron_env_setting("HERMES_MODEL") or ""
    _cron_default_provider = ""
    _cfg: dict = {}
    _model_cfg: Any = {}
    try:
        from hermes_cli.config_effective import load_user_config_effective
        _cfg_path = str(_get_hermes_home() / "config.yaml")
        if os.path.exists(_cfg_path):
            _cfg = load_user_config_effective(Path(_cfg_path))
            # Coerce null to {} so a falsy default never clobbers a resolved env value.
            _model_cfg = _cfg.get("model") or {}
            _cron_cfg_for_model = _cfg.get("cron") or {}
            _cron_default_model = ""
            if isinstance(_cron_cfg_for_model, dict):
                _cron_default_model = str(_cron_cfg_for_model.get("model") or "").strip()
                _cron_default_provider = str(_cron_cfg_for_model.get("model_provider") or "").strip()
            if not job.get("model"):
                if _cron_default_model:
                    model = _cron_default_model
                else:
                    # The main agent model: ``model: <name>`` shorthand or ``model.default``.
                    _main = _model_cfg if isinstance(_model_cfg, str) else (
                        _model_cfg.get("default") or _model_cfg.get("model") or _model_cfg.get("name")
                        if isinstance(_model_cfg, dict) else "")
                    model = str(_main or "").strip() or model
    except Exception as e:
        logger.warning("Job '%s': failed to load config.yaml, using defaults: %s", job_id, e)

    # Fail fast: an empty model otherwise reaches the provider as an opaque 400.
    # See #23979.
    if not (isinstance(model, str) and model.strip()):
        raise RuntimeError(
            f"Cron job '{job_name}' has no model configured "
            f"(job.model={job.get('model')!r}, "
            f"HERMES_MODEL={cron_env_setting('HERMES_MODEL')!r}, "
            "config.yaml model.default missing or empty). "
            f"Set a per-job model via "
            f"`hermes cron edit {job_id} --model <name>` or set a "
            "default with `hermes model <name>`."
        )

    with contextlib.suppress(Exception):
        from hermes_constants import apply_ipv4_preference
        _net_cfg = _cfg.get("network", {})
        if isinstance(_net_cfg, dict) and _net_cfg.get("force_ipv4"):
            apply_ipv4_preference(force=True)
    return _CronJobConfig(_cfg, model, _model_cfg, _cron_default_provider)


def _load_prefill_messages(cfg: dict, job_id: str) -> Optional[list]:
    """Prefill messages from env or config.yaml (top-level key canonical; agent.* is legacy)."""
    agent_cfg = cfg.get("agent", {}) if isinstance(cfg.get("agent", {}), dict) else {}
    prefill_file = (
        cron_env_setting("HERMES_PREFILL_MESSAGES_FILE")
        or cfg.get("prefill_messages_file", "")
        or agent_cfg.get("prefill_messages_file", "")
    )
    if not prefill_file:
        return None
    pfpath = Path(prefill_file).expanduser()
    if not pfpath.is_absolute():
        pfpath = _get_hermes_home() / pfpath
    if not pfpath.exists():
        return None
    try:
        with open(pfpath, "r", encoding="utf-8-sig") as _pf:
            prefill_messages = json.load(_pf)
        return prefill_messages if isinstance(prefill_messages, list) else None
    except Exception as e:
        logger.warning("Job '%s': failed to parse prefill messages file '%s': %s", job_id, pfpath, e)
        return None


def _preflight_or_block(job: dict, job_id: str, job_name: str, cfg: dict) -> Optional[tuple]:
    """Pre-dispatch config validation: refuse unrunnable jobs (missing key, unready skill,
    unconfigured delivery) BEFORE AIAgent is built. run_one_job keys off BLOCKED_CONFIG_MARKER to
    record blocked_config and alert once (`preflight_alerted` bit). Must run after the wake gate so
    silent ticks stay silent. Opt-out: `cron.preflight: false`. Returns failure tuple or None.
    """
    # --------------------------------------------------------------- Pre-dispatch configuration validation
    # (T1-26). A job whose configuration cannot possibly produce a successful run — missing provider API key
    # (no fallback chain), unready attached skill, unconfigured delivery platform — is refused HERE, before
    # AIAgent is constructed and before the resolution below can feed a doomed runtime into it, so a
    # misconfigured job never burns an LLM call. run_one_job keys off the BLOCKED_CONFIG_MARKER in the
    # returned error to record last_status='blocked_config' and alert exactly once (dedup persisted via the
    # job's `preflight_alerted` bit — the #73506 alert-once shape).
    _pf_reason = None
    try:
        if _cron_preflight_enabled(cfg):
            _pf_reason = _preflight_job_config(job, cfg)
            if not _pf_reason and job.get("preflight_alerted"):
                # Config healthy again: clear alert-once marker so a future break re-alerts.
                with contextlib.suppress(Exception):
                    from cron.jobs import clear_preflight_alerted
                    clear_preflight_alerted(job_id)
    except Exception:
        # Fail open: the validator must never take down a runnable job.
        logger.debug("Job '%s': preflight validation errored — failing open", job_id, exc_info=True)
        _pf_reason = None
    if not _pf_reason:
        return None
    return _blocked_config_result(job_id, job_name, _pf_reason)


def _blocked_config_result(job_id: str, job_name: str, _pf_reason: str) -> tuple:
    """The ``blocked_config`` failure tuple for *_pf_reason*, alerting once per job."""
    logger.warning(
        "Job '%s' (ID: %s): BLOCKED by pre-dispatch config validation — %s (no LLM call was made)",
        job_name, job_id, _pf_reason)
    already_alerted = False
    try:
        from cron.jobs import mark_preflight_alerted
        already_alerted = mark_preflight_alerted(job_id)
    except Exception:
        logger.debug("Job '%s': could not persist preflight alert marker", job_id, exc_info=True)
    marker = BLOCKED_CONFIG_SILENT_MARKER if already_alerted else BLOCKED_CONFIG_MARKER
    blocked_doc = (
        f"# Cron Job: {job_name}\n\n"
        f"**Job ID:** {job_id}\n"
        f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
        f"**Status:** BLOCKED (configuration)\n\n"
        "The pre-run configuration check found a problem, so the agent did not run "
        "(nothing was charged).\n\n"
        f"**Reason:** {_pf_reason}\n\n"
        "Hermes tries again at the next scheduled time and clears this state on the first healthy "
        "run; this alert is not repeated. Check with `hermes cron doctor`. Set `cron.preflight: "
        "false` in config.yaml to disable this check."
    )
    return False, blocked_doc, "", f"{marker} {_pf_reason}"


def _resolve_job_runtime(job: dict, job_id: str, jc: _CronJobConfig) -> tuple[dict, str]:
    """Resolve the runtime, walking the fallback chain on auth/transient-network errors. Returns
    ``(runtime, model)``; provider+model swap atomically (never swap only the provider while keeping
    a paid primary model). Provider precedence: per-job pin > cron.model_provider > persisted
    global config (None lets resolve_runtime_provider read it). A pinned job has no chain here
    (``_job_fallback_chain``): its resolve failure is the job's failure."""
    from hermes_cli.runtime_provider import (
        resolve_runtime_provider, format_runtime_provider_error)
    from hermes_cli.auth import AuthError

    model = jc.model
    requested = job.get("provider") or jc.cron_default_provider or None
    try:
        # Do NOT pass HERMES_INFERENCE_PROVIDER as `requested`: it would override persisted config
        # and resurrect stale providers for unpinned jobs.
        runtime_kwargs = {
            "requested": requested,
            # api_mode must derive from the model actually run, not the stale persisted default.
            "target_model": model,
        }
        if job.get("base_url"):
            runtime_kwargs["explicit_base_url"] = job.get("base_url")
        return resolve_runtime_provider(**runtime_kwargs), model
    except Exception as resolve_exc:
        # Walk the fallback chain on AuthError AND transient network/DNS failures (e.g. during
        # OAuth refresh); anything else re-raises.
        is_auth = isinstance(resolve_exc, AuthError)
        is_transient_net = _is_transient_provider_resolve_error(resolve_exc)
        if not (is_auth or is_transient_net):
            raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc

        chain = _job_fallback_chain(job, jc.cfg) or []
        logger.warning(
            "Job '%s': primary provider resolve failed (%s: %s), %s",
            job_id, "auth" if is_auth else "transient network", resolve_exc,
            "trying fallback" if chain else (
                "not falling back: the job is pinned" if _job_route_pinned(job) else "no fallback configured"))
        for entry in chain:
            if not isinstance(entry, dict):
                continue
            fb_provider = str(entry.get("provider") or "").strip()
            fb_model = str(entry.get("model") or "").strip()
            if not fb_provider or not fb_model:
                continue
            try:
                from hermes_cli.fallback_config import effective_runtime_provider, resolve_entry_api_key

                fb_kwargs = {"requested": fb_provider, "target_model": fb_model}
                if entry.get("base_url"):
                    fb_kwargs["explicit_base_url"] = entry["base_url"]
                fb_api_key = resolve_entry_api_key(entry)
                if fb_api_key:
                    fb_kwargs["explicit_api_key"] = fb_api_key
                runtime = resolve_runtime_provider(**fb_kwargs)
                # Named custom entries resolve to the bare "custom" billing class; keep the configured
                # identity so job sessions record the provider name (#98739).
                runtime["provider"] = effective_runtime_provider(entry, runtime)
                logger.info(
                    "Job '%s': fallback resolved to %s model %s",
                    job_id, runtime.get("provider"), fb_model)
                # Delivered with the job output (#74349): a cron agent has no status rail, so the
                # switch would otherwise stay in the scheduler log only. run_job pops it.
                from hermes_cli.fallback_config import pre_agent_fallback_notice
                runtime["_fallback_notice"] = pre_agent_fallback_notice(
                    requested or (jc.model_cfg.get("provider") if isinstance(jc.model_cfg, dict) else ""),
                    model, runtime.get("provider"), fb_model)
                return runtime, fb_model
            except Exception as fb_exc:
                logger.debug("Job '%s': fallback %s failed: %s", job_id, fb_provider, fb_exc)
        raise RuntimeError(format_runtime_provider_error(resolve_exc)) from resolve_exc


def _load_credential_pool(runtime: dict, job_id: str):
    runtime_provider = str(runtime.get("provider") or "").strip().lower()
    if not runtime_provider:
        return None
    try:
        from agent.credential_pool import load_pool
        pool = load_pool(runtime_provider)
        if pool.has_credentials():
            logger.info(
                "Job '%s': loaded credential pool for provider %s with %d entries",
                job_id, runtime_provider, len(pool.entries()))
            return pool
    except Exception as e:
        logger.debug("Job '%s': failed to load credential pool for %s: %s", job_id, runtime_provider, e)
    return None


def _init_cron_mcp_tools(job_id: str) -> None:
    """Register MCP servers for the agent's tool registry. Idempotent across ticks; non-fatal so a
    broken MCP server never kills a working job."""
    try:
        # Initialize MCP servers so configured mcp_servers are available to the agent's tool registry before
        # AIAgent is constructed. Without this, cron jobs never saw any MCP tools — only the gateway / CLI
        # paths called discover_mcp_tools() at startup. Idempotent: subsequent ticks short-circuit on
        # already-connected servers inside register_mcp_servers(). Non-fatal on failure: a broken MCP server
        # shouldn't kill an otherwise-working cron job. See #4219.
        from tools.mcp_tool_discovery import discover_mcp_tools
        _mcp_tools = discover_mcp_tools()
        if _mcp_tools:
            logger.info("Job '%s': %d MCP tool(s) available", job_id, len(_mcp_tools))
    except Exception as _mcp_exc:
        logger.warning("Job '%s': MCP initialization failed (non-fatal): %s", job_id, _mcp_exc)


def _open_cron_session_db(job: dict):
    """Open the SQLite session store under its own timeout (HERMES_CRON_TIMEOUT only watches
    run_conversation). A wedged sqlite3.connect returns None (no session store) instead of
    wedging the worker thread."""
    # Initialize the SQLite session store so cron job messages are persisted and discoverable via
    # session_search (same pattern as gateway/run.py) — only now, after every early-return path (wake-gate,
    # prompt validation, drift skip) has passed, so a gated run never opens state.db just to abandon the
    # handle (#96290). Bounded with its own timeout (separate from HERMES_CRON_TIMEOUT, which only watches
    # the agent's run_conversation below): SessionDB.__init__ opens/migrates state.db synchronously and has
    # no timeout of its own against a wedged sqlite3.connect (e.g. a stale flock left by a crashed sibling
    # process). An unbounded hang here would wedge the job's worker thread, so the init is bounded and a
    # timeout proceeds without a session store instead of blocking the run forever.
    _session_db_timeout = _get_session_db_timeout()
    try:
        from hermes_state_registry import acquire

        if _session_db_timeout <= 0:
            return acquire()
        _session_db_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
        # Copy the context so a profile run resolves ITS OWN home/state.db on the worker thread
        # instead of the process-global default.
        _session_db_context = contextvars.copy_context()
        _session_db_future = _session_db_pool.submit(_session_db_context.run, acquire)
        try:
            return _session_db_future.result(timeout=_session_db_timeout)
        except concurrent.futures.TimeoutError:
            # The abandoned worker may still finish; close its late result or its SQLite FDs leak.
            # The worker is abandoned (shutdown below doesn't wait for it). If SessionDB() later completes
            # inside it, the future's result would be orphaned and its SQLite FDs (.db, WAL, SHM) leak until
            # process exit. Register a done-callback that retrieves and closes any eventual late result
            # (#72782).
            _session_db_future.add_done_callback(_close_late_session_db_result)
            raise
        finally:
            # Abandon a wedged connect() rather than blocking shutdown on it.
            _session_db_pool.shutdown(wait=False)
    except concurrent.futures.TimeoutError:
        logger.error(
            "Job '%s': SessionDB init did not return within %.0fs — proceeding "
            "without a session store for this run instead of blocking it forever",
            job.get("id", "?"), _session_db_timeout)
    except Exception as e:
        logger.debug("Job '%s': SQLite session store not available: %s", job.get("id", "?"), e)
    return None


def _raise_inactivity_timeout(agent, job_name: str, limit_s: float) -> None:
    """Log the agent's last activity, hard-interrupt it and raise TimeoutError."""
    _activity = {}
    if hasattr(agent, "get_activity_summary"):
        with contextlib.suppress(Exception):
            _activity = agent.get_activity_summary()
    _last_desc = _activity.get("last_activity_desc", "unknown")
    _secs_ago = _activity.get("seconds_since_activity", 0)
    logger.error(
        "Job '%s' idle for %.0fs (inactivity limit %.0fs) "
        "| last_activity=%s | iteration=%s/%s | tool=%s",
        job_name, _secs_ago, limit_s,
        _last_desc, _activity.get("api_call_count", 0), _activity.get("max_iterations", 0),
        _activity.get("current_tool") or "none")
    request_hard_interrupt(agent, "Cron job timed out (inactivity)", tool_reason="cron inactivity watchdog")
    raise TimeoutError(
        f"Cron job '{job_name}' idle for "
        f"{int(_secs_ago)}s (limit {int(limit_s)}s) "
        f"— last activity: {_last_desc}")


def _run_agent_with_watchdog(
    agent, prompt: str, job: dict, job_id: str, job_name: str, task_id: str, cancel_event,
    worker_state: Optional[dict] = None,
) -> dict:
    """Run ``agent.run_conversation`` on a worker thread under the inactivity (not wall-clock)
    watchdog: default 600s, override HERMES_CRON_TIMEOUT, 0 = unlimited."""
    _cron_timeout = _cron_inactivity_seconds()
    _cron_inactivity_limit = _cron_timeout if _cron_timeout > 0 else None
    _POLL_INTERVAL = 5.0
    # Heartbeat the one-shot run_claim while alive: without it a long run looks like a dead owner
    # and gets re-dispatched / stale-removed out from under the live run.
    # Keep the one-shot run_claim fresh while the run is alive (#62002): the claim TTL is a dead-owner
    # detector, but without a heartbeat a run that legitimately outlives it (stream stall, laptop asleep
    # mid-run) is indistinguishable from a dead tick — another process re-dispatches it and get_due_jobs
    # stale-removes the job record out from under the live run. Refreshing the claim from this monitor keeps
    # "expired claim" meaning "owner died".
    _job_schedule = job.get("schedule")
    _is_oneshot = isinstance(_job_schedule, dict) and _job_schedule.get("kind") == "once"
    _run_claim = job.get("run_claim")
    _run_claim_owner = str(_run_claim.get("by") or "") if isinstance(_run_claim, dict) else ""
    _last_claim_heartbeat = time.monotonic()

    def _abort_if_fire_claim_lost() -> None:
        if cancel_event is None or not cancel_event.is_set():
            return
        if agent is not None and hasattr(agent, "interrupt"):
            agent.interrupt("Cron fire claim ownership was lost")
        raise RuntimeError(f"Cron job '{job_name}' lost its durable fire claim ownership")

    def _heartbeat_run_claim_if_due():
        nonlocal _last_claim_heartbeat
        if not _is_oneshot or not _run_claim_owner:
            return
        _mono = time.monotonic()
        if _mono - _last_claim_heartbeat < _RUN_CLAIM_HEARTBEAT_SECONDS:
            return
        _last_claim_heartbeat = _mono
        try:
            heartbeat_run_claim(job_id, expected_owner=_run_claim_owner)
        except Exception:
            logger.debug("Job '%s': run_claim heartbeat failed", job_name, exc_info=True)

    # Establish suspend accounting before the worker can stamp its first activity.
    _awake_idle = AwakeIdleMeter()
    _cron_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
    # Carry scheduler-scoped ContextVar state (e.g. env passthrough) into the worker thread.
    _cron_context = contextvars.copy_context()
    _cron_future = _cron_pool.submit(
        _cron_context.run, agent.run_conversation, prompt, task_id=task_id)
    if worker_state is not None:
        worker_state["future"] = _cron_future
    _inactivity_timeout = False
    _watch_stop = threading.Event()

    def _idle_seconds() -> float:
        if not hasattr(agent, "get_activity_summary"):
            return 0.0
        try:
            _act = agent.get_activity_summary()
            return float(_act.get("seconds_since_activity", 0.0) or 0.0)
        except Exception:
            return 0.0

    def _watch_inactivity() -> None:
        nonlocal _inactivity_timeout
        if _cron_inactivity_limit is None:
            return
        if _inactivity_watchdog_loop(
            get_idle_seconds=_idle_seconds, limit_s=_cron_inactivity_limit, poll_s=_POLL_INTERVAL,
            stop=_watch_stop, future_done=_cron_future.done, meter=_awake_idle):
            _inactivity_timeout = True

    _watch_thread = threading.Thread(
        target=_watch_inactivity, name=f"cron-inactivity-{str(job_id)[:8]}", daemon=True)
    try:
        if _cron_inactivity_limit is not None:
            # Separate daemon thread so a hung get_activity_summary can't stop the limit firing.
            # Daemon thread: kernel ``Event.wait`` timeout, independent of the ``run_job`` thread. A blocked
            # loop / hung ``get_activity_summary`` on this thread can no longer keep the 600s inactivity
            # limit from firing (#94285).
            _watch_thread.start()
        if _cron_inactivity_limit is None and not _is_oneshot and cancel_event is None:
            result = _cron_future.result()
        else:
            result = None
            while True:
                done, _ = concurrent.futures.wait({_cron_future}, timeout=_POLL_INTERVAL)
                if done:
                    _abort_if_fire_claim_lost()
                    result = _cron_future.result()
                    break
                if _inactivity_timeout:
                    break
                _abort_if_fire_claim_lost()
                _heartbeat_run_claim_if_due()
    except Exception:
        _cron_pool.shutdown(wait=False, cancel_futures=True)
        raise
    finally:
        _watch_stop.set()
        _cron_pool.shutdown(wait=False, cancel_futures=True)

    if _inactivity_timeout:
        _raise_inactivity_timeout(agent, job_name, _cron_inactivity_limit)

    if not isinstance(result, dict):
        raise RuntimeError(
            f"agent.run_conversation returned {type(result).__name__} instead of dict: {result!r}"
        )
    return result


def _final_response_from_result(result: dict, job_id: str, job_name: str, AIAgent) -> str:
    """Deliverable final response from a ``run_conversation`` result. Raises RuntimeError on
    `failed=True`/`completed=False`: the error text may sit in `final_response` and would otherwise
    be delivered as the reply with the job marked ok."""
    # If the agent itself reported failure (e.g. all retries exhausted on API errors, model abort, mid-run
    # interrupt), do not silently mark the job as successful. run_agent populates
    # `failed=True`/`completed=False` on these paths and may put the error into `final_response`, which
    # would otherwise be delivered as if it were the agent's reply and the job's `last_status` set to "ok".
    # Raise so the except handler below builds the proper failure tuple. (issue #17855)
    turn_exit_reason = str(result.get("turn_exit_reason") or "")
    final_response_text = (result.get("final_response") or "").strip()
    max_iteration_summary = is_max_iteration_handoff(result)
    if result.get("failed") is True or (result.get("completed") is False and not max_iteration_summary):
        raise RuntimeError(result.get("error") or final_response_text or "agent reported failure")
    if max_iteration_summary:
        logger.warning(
            "Job '%s' reached the iteration limit but produced a final fallback response; "
            "delivering the response instead of failing the cron run",
            job_name)

    final_response = result.get("final_response", "") or ""
    # Repair model-mangled computer_use media paths before delivery (fail-open, as in gateway).
    if final_response:
        from gateway.media_repair import repair_explicit_computer_use_media_paths

        final_response = repair_explicit_computer_use_media_paths(
            final_response, result.get("messages", []))
    if final_response.strip() == "(No response generated)":
        final_response = ""
    # The "⚠️ No reply" turn-completion explainer would be delivered as a cron warning; detect it
    # via the same formatter and treat as empty so cron stays silent on abnormal empty turns.
    if final_response.strip() and turn_exit_reason:
        # Render every persistence-cause variant or cause-refined text slips through.
        _explainer_variants = []
        try:
            from hermes_state_errors import PERSISTENCE_ERROR_CAUSES as _causes
        except Exception:
            _causes = ("locked", "disk", "unknown")
        # The finalizer fills the model name into the explainer; render with the same name (and
        # the bare form) or the comparison below misses and the warning is delivered.
        _model = str(result.get("model") or "")
        for _cause in (None, *_causes):
            for _kwargs in ({"model": _model}, {}):
                try:
                    _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason, _cause, **_kwargs)
                except TypeError:
                    try:
                        _variant = AIAgent._format_turn_completion_explanation(turn_exit_reason)
                    except Exception:
                        _variant = ""
                except Exception:
                    _variant = ""
                if _variant:
                    _explainer_variants.append(_variant.strip())
        if final_response.strip() in _explainer_variants:
            logger.info(
                "Job '%s': abnormal empty turn (%s) — suppressing explainer for cron delivery",
                job_id, turn_exit_reason)
            final_response = ""
    return final_response


def _finalize_cron_session(session_db, agent, job_id: str, job_name: str, cron_session_id: str) -> None:
    """Title, classify, end and release the cron session after the agent turn has returned."""
    # Bound every DB op so storage failure cannot hold the dispatch guard.
    _session_db = _BoundedCronSessionDB(session_db, job_id)
    # Compression may have rotated the run onto a continuation: finalize that, not the stale cron
    # id. SessionDB lineage is authoritative; agent.session_id is only a fail-safe.
    _final_cron_session_id = cron_session_id
    try:
        _compression_tip = _session_db.get_compression_tip(cron_session_id)
        if _compression_tip:
            _final_cron_session_id = _compression_tip
    except (Exception, KeyboardInterrupt) as e:
        with contextlib.suppress((Exception, KeyboardInterrupt)):
            _agent_session_id = getattr(agent, "session_id", None)
            # CLI (single-process) path: the approval contextvar is only bound during gateway/TUI turns and
            # HERMES_SESSION_KEY is not in the CLI environment, so the key resolves empty here. Since #64240
            # the CLI drains completions through a positive-ownership filter keyed on the durable
            # AIAgent.session_id — an empty session_key would fail closed and the CLI could never claim its
            # own completions, while a restored foreign event with an empty key could leak into any
            # unfiltered consumer (#64484). Stamp the parent's durable session id instead; compression
            # rotations are handled on the drain side via resolve_resume_session_id lineage resolution.
            if _agent_session_id:
                _final_cron_session_id = _agent_session_id
        logger.debug("Job '%s': failed to resolve cron compression tip: %s", job_id, e)
    # Title must persist BEFORE end_session()/close(). Run-time suffix keeps it unique against the
    # sessions.title index; the fallbacks below guarantee a non-blank title.
    try:
        # Title the cron session from the job (name -> id) and PERSIST it BEFORE end_session()/close() tear
        # the connection down, so the close can never run over an in-flight title write (#50536).
        _title_base = " ".join(job_name.split())[:60].strip() or f"cron {job_id}"
        _cron_title = f"{_title_base} · {safe_strftime(_hermes_now(), '%b %d %H:%M')}"
        if not _set_cron_session_title(_session_db, _final_cron_session_id, _cron_title):
            _set_cron_session_title(_session_db, _final_cron_session_id, f"cron {job_id}")
    except (Exception, KeyboardInterrupt) as e:
        logger.debug("Job '%s': failed to set cron session title: %s", job_id, e)
        # Never leave the session untitled.
        # Try the next free title in the lineage, then a bare id-stamped title. See #50535.
        for _fallback in (
            getattr(_session_db, "get_next_title_in_lineage", lambda b: b)(f"cron {job_id}"),
            f"cron {job_id} {_final_cron_session_id[-6:]}"):
            try:
                if _set_cron_session_title(_session_db, _final_cron_session_id, _fallback):
                    break
            except (Exception, KeyboardInterrupt):
                continue
    # Book cron_complete only when the last row is a real assistant reply ([SILENT] counts). Only a
    # POSITIVELY recognized bad status downgrades (keep tuple in sync with
    # session_lifecycle_statuses); unknown values / probe failures fail OPEN.
    # Verified completion booking (#93820): the run may only be recorded as cron_complete when the session's
    # LAST message row is a real assistant reply — a plain answer or the [SILENT] sentinel (both are
    # assistant-text rows, so both classify as 'complete'). A turn that died after a tool call,
    # mid-API-wait, or without any assistant text leaves the last row as a tool result / pending call / user
    # prompt and must not surface as a healthy run. session_lifecycle_statuses is the existing cost-bounded
    # classifier for exactly this shape. Only a POSITIVELY recognized pathological status (see the status
    # vocabulary in hermes_state's session_lifecycle_statuses docstring — keep the tuple below in sync when
    # it grows) downgrades the booking: an unknown value (newer classifier shape, test doubles) keeps the
    # historical reason, and so does a failed probe — the booking itself is FAIL-OPEN on probe errors,
    # because classification is best-effort metadata and must not mislabel a healthy run.
    _end_reason = "cron_complete"
    try:
        _statuses = _session_db.session_lifecycle_statuses([_final_cron_session_id])
        _lifecycle = _statuses.get(_final_cron_session_id)
        if _lifecycle in ("interrupted", "error", "empty"):
            _end_reason = "cron_incomplete_no_output"
            logger.warning(
                "Job '%s': session ended without a final assistant "
                "message (lifecycle=%s) — booking run as %s",
                job_id, _lifecycle, _end_reason)
    except (Exception, KeyboardInterrupt) as e:
        logger.debug("Job '%s': session lifecycle classification failed: %s", job_id, e)
    try:
        _session_db.end_session(_final_cron_session_id, _end_reason)
        # The scheduler owns cron-session finalization. AIAgent.close() also
        # finalizes owned session rows by default; once the shared SessionDB is
        # released below, that second end_session() would reopen the just-closed
        # SQLite handle (#94736). The reason is durably booked, so disarm only the
        # agent's redundant row-finalization; its resource teardown still runs in
        # _teardown_cron_agent.
        if agent is not None:
            agent._end_session_on_close = False
    except (Exception, KeyboardInterrupt) as e:
        logger.debug("Job '%s': failed to end session: %s", job_id, e)
    try:
        from hermes_state_registry import release_or_close
        release_or_close(_session_db)
    except (Exception, KeyboardInterrupt) as e:
        logger.debug("Job '%s': failed to close SQLite session store: %s", job_id, e)


def _normalize_newlines(text: str) -> str:
    """CRLF/CR -> LF, matching what text-mode reads of the archive return."""
    return text.replace("\r\n", "\n").replace("\r", "\n")


def _run_doc_header(job: dict, title: str, job_id: str, prompt: str, *, prompt_stamp: str = "") -> str:
    """Header of the persisted run document (title, ids, schedule, prompt)."""
    return (
        f"# Cron Job: {title}\n\n"
        f"**Job ID:** {job_id}\n"
        f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
        f"**Schedule:** {job.get('schedule_display', 'N/A')}\n\n"
        f"{prompt_stamp}{_PROMPT_HEADING}{prompt}{_PROMPT_SEPARATOR}"
    )


_RunResult = tuple[bool, str, str, Optional[str]]


def _prepare_job_prompt(
    job: dict, job_id: str, job_name: str, extra_prompt: Optional[str], cancel_event,
) -> tuple[Optional[_RunResult], Optional[str]]:
    """Run every pre-agent gate and build the prompt. Returns ``(early_result, prompt)``: an early
    result short-circuits ``run_job`` (no_agent job, empty payload, monitor gate, wake gate,
    injection block, empty prompt); otherwise ``prompt`` is set."""
    # Fail closed on a corrupt config.yaml: defaults would let auto-detection bill a provider the
    # user never chose. no_agent jobs are exempt. Escape hatch: HERMES_IGNORE_USER_CONFIG=1.
    if not job.get("no_agent"):
        from hermes_cli.config import InvalidUserConfigError, require_parseable_user_config

        try:
            require_parseable_user_config()
        except InvalidUserConfigError as exc:
            logger.error("Job '%s': refusing to run — %s", job_id, exc)
            return (False, f"# Cron Job: {job_name}\n\nError: {exc}\n", "", str(exc)), None

    # no_agent short-circuits BEFORE importing run_agent / opening SessionDB.
    if job.get("no_agent"):
        return _run_no_agent_job(job, job_id, job_name, cancel_event), None

    # Legacy / hand-edited job with nothing to run: pause it instead of waking the LLM every fire.
    from cron.jobs import EMPTY_PAYLOAD_ERROR, job_payload_is_empty

    if job_payload_is_empty(job):
        return _block_and_pause_job(job_id, job_name, EMPTY_PAYLOAD_ERROR), None

    _early, extra_prompt, monitor_context = _apply_monitor_gate(job, job_id, job_name, extra_prompt)
    if _early is not None:
        return _early, None

    # Wake-gate: run the pre-check script BEFORE building the prompt; its result is passed into
    # _build_job_prompt so the script runs only once.
    # NOTE: the SQLite session store used to be initialized here, BEFORE the wake-gate and prompt-validation
    # early returns below. Every gated run (``wakeAgent: false``, blocked prompt) opened state.db and
    # returned without reaching the finally that closes it, relying on GC to release the handle. Init now
    # happens inside the main try, right before the agent is constructed — after every early-return path
    # (#96290).
    prerun_script = None
    script_path = job.get("script")
    if script_path:
        prerun_script = _run_job_script_with_claim_heartbeat(
            job,
            script_path,
            workdir=_resolve_job_workdir(job, job_id),
            cancel_event=cancel_event,
        )
        _ran_ok, _script_output = prerun_script
        if _ran_ok and not _parse_wake_gate(_script_output):
            logger.info("Job '%s' (ID: %s): wakeAgent=false, skipping agent run", job_name, job_id)
            note_cron_skipped(job)
            silent_doc = (
                f"# Cron Job: {job_name}\n\n"
                f"**Job ID:** {job_id}\n"
                f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n\n"
                "Script gate returned `wakeAgent=false` — agent skipped.\n"
            )
            return (True, silent_doc, SILENT_MARKER, None), None

    try:
        prompt = _build_job_prompt(
            job, prerun_script=prerun_script, extra_prompt=extra_prompt,
            runtime_data_prompt=monitor_context,
        )
    except CronPromptInjectionBlocked as block_exc:
        # Injection scanner tripped: refuse this tick and tell the operator WHY.
        logger.warning(
            "Job '%s' (ID: %s): blocked by prompt-injection scanner — %s", job_name, job_id, block_exc,
        )
        blocked_doc = (
            f"# Cron Job: {job_name}\n\n"
            f"**Job ID:** {job_id}\n"
            f"**Run Time:** {_hermes_now().strftime('%Y-%m-%d %H:%M:%S')}\n"
            f"**Status:** BLOCKED\n\n"
            "The assembled prompt (user prompt + loaded skill content) tripped "
            "the cron injection scanner and the agent was NOT run.\n\n"
            f"**Scanner result:** {block_exc}\n\n"
            "Audit the skill(s) attached to this job for prompt-injection "
            "payloads or invisible-unicode markers. If the skill is legitimate "
            "and the match is a false positive, rephrase the content to avoid "
            "the threat pattern (`tools/cronjob_tools.py::_CRON_THREAT_PATTERNS`)."
        )
        return (False, blocked_doc, "", str(block_exc)), None
    if prompt is None:
        logger.info("Job '%s': script produced no output, skipping AI call.", job_name)
        note_cron_skipped(job)
        return (True, "", SILENT_MARKER, None), None
    return None, prompt


_CRON_DELIVERY_VARS = (
    "HERMES_CRON_AUTO_DELIVER_PLATFORM",
    "HERMES_CRON_AUTO_DELIVER_CHAT_ID",
    "HERMES_CRON_AUTO_DELIVER_THREAD_ID")


class _CronRunScope:
    """Per-run ContextVar / tool-cwd scope for ``run_job`` (ContextVars, not os.environ, so
    parallel jobs don't clobber each other). Construct before the try, ``enter()`` as its first
    statement, ``exit()`` in the finally — every setter here has a matching reset there.

    HERMES_SESSION_* are deliberately NOT seeded from job["origin"]: it is delivery metadata, not
    a sender, and terminal/tts/skills/send_message tools would act as if the origin user were
    driving the agent. Delivery reads job["origin"] / HERMES_CRON_AUTO_DELIVER_* directly.
    """

    def __init__(self, job: dict, job_id: str, execution_id: Optional[str]):
        from gateway.session_context import set_session_vars, _VAR_MAP
        from tools.terminal_tool import record_session_cwd

        self._var_map = _VAR_MAP
        # Resolve workdir BEFORE set_session_vars so it owns the _SESSION_CWD set/clear.
        self.workdir = _resolve_job_workdir(job, job_id)
        self._ctx_tokens = set_session_vars(
            platform="",
            chat_id="",
            chat_name="",
            # Cron can't receive completions after its turn; async delegation output could
            # otherwise route to an unrelated chat via the ambient session key => inline delegation.
            # We clear the HERMES_SESSION_* routing keys just below, so an async delegation's completion
            # event carries session_key="" — _enrich_async_delegation_routing cannot resolve it and
            # _inject_watch_notification drops it ("no routing metadata"). And by the time a child finishes,
            # run_job has already shipped the job's final response via _deliver_result; there is no turn
            # left to re-enter. (Worse, get_current_session_key() can fall back to the ambient os.environ
            # HERMES_SESSION_KEY, which risks routing a cron subagent's output into an unrelated user chat.)
            # Declaring the channel stateless routes delegate_task to its existing inline/synchronous path,
            # so results return within the job's own turn. See declare_stateless_channel(). Upstream:
            # #53027, #63142.
            async_delivery=False,
            cwd=self.workdir or "",
        )
        for name in _CRON_DELIVERY_VARS:
            _VAR_MAP[name].set("")
        # Workdir binds to the per-run task id (tool-layer cwd authority) instead of mutating
        # global TERMINAL_CWD; _SESSION_CWD above remains the prompt/context-file authority.
        self.task_id = f"cron:{job_id}:{execution_id or job.get('execution_id') or uuid.uuid4().hex}"
        if self.workdir:
            record_session_cwd(self.task_id, self.workdir)
        self._cron_session_var = _VAR_MAP["HERMES_CRON_SESSION"]
        self._cron_session_token = None
        self._non_dispatcher_token = None

    def enter(self) -> None:
        # Scope cron approval policy; exit() RESETS via token (pinning "" would suppress the legacy
        # os.environ fallback used by standalone entrypoints/tests).
        self._cron_session_token = self._cron_session_var.set("1")
        # Mark NOT the kanban worker: a worker's cronjob(action="run") lands here with
        # HERMES_KANBAN_TASK in env, and an unrelated job could close the worker's task. Must be a
        # ContextVar, NOT an os.environ clear (env is shared with the worker heartbeat and
        # concurrent jobs); copy_context() carries it into the agent thread.
        self._non_dispatcher_token = enter_non_dispatcher_owned_context()

    def exit(self) -> None:
        from gateway.session_context import clear_session_vars
        from tools.terminal_tool import clear_session_cwd

        clear_session_cwd(self.task_id)
        clear_session_vars(self._ctx_tokens)  # also clears _SESSION_CWD
        if self._cron_session_token is not None:
            self._cron_session_var.reset(self._cron_session_token)
        if self._non_dispatcher_token is not None:
            exit_non_dispatcher_owned_context(self._non_dispatcher_token)
        for name in _CRON_DELIVERY_VARS:
            self._var_map[name].set("")


def _reload_dotenv_and_publish_delivery_target(job: dict) -> None:
    """Re-read .env for this run and publish the auto-deliver target into the session ContextVars."""
    # Reset the secret-source cache FIRST or a Bitwarden/BSM-backed secret is never re-resolved
    # (only the placeholder reloads -> 401s).
    from hermes_cli.env_loader import load_hermes_dotenv, reset_secret_source_cache
    from gateway.session_context import _VAR_MAP

    reset_secret_source_cache(_get_hermes_home())
    load_hermes_dotenv(hermes_home=_get_hermes_home())

    delivery_target = _resolve_delivery_target(job)
    if delivery_target:
        _VAR_MAP["HERMES_CRON_AUTO_DELIVER_PLATFORM"].set(delivery_target["platform"])
        _VAR_MAP["HERMES_CRON_AUTO_DELIVER_CHAT_ID"].set(str(delivery_target["chat_id"]))
        _VAR_MAP["HERMES_CRON_AUTO_DELIVER_THREAD_ID"].set(
            "" if delivery_target.get("thread_id") is None else str(delivery_target["thread_id"])
        )


@dataclass
class _CronAgentSetup:
    """Everything ``AIAgent(...)`` needs that is resolved from job + config (or a preflight block)."""
    blocked: Optional[_RunResult] = None
    model: str = ""
    runtime: dict = None
    prefill_messages: Any = None
    max_iterations: Any = None
    reasoning_config: Any = None
    fallback_model: Any = None
    credential_pool: Any = None
    fallback_notice: Optional[str] = None


def _resolve_cron_agent_setup(job: dict, job_id: str, job_name: str, jc) -> _CronAgentSetup:
    """Resolve model/runtime/reasoning/pool for the run, in the original gate order: exfil guard ->
    preflight (may block) -> runtime (+ fallback chain) -> credential pool -> MCP."""
    _cfg = jc.cfg
    setup = _CronAgentSetup(model=jc.model)
    setup.prefill_messages = _load_prefill_messages(_cfg, job_id)

    # resolve_turn_limit() honors none/unlimited (sys.maxsize) and explicit 0 / null.
    from hermes_cli.config import resolve_turn_limit as _resolve_turn_limit
    _mt = _cfg.get("agent", {}).get("max_turns")
    if _mt is None:
        _mt = _cfg.get("max_turns")
    setup.max_iterations = _resolve_turn_limit(_mt)

    # Runtime backstop (CWE-200/522): fail closed BEFORE resolution on a provider/base_url pair
    # that would ship a stored credential off-host; hand-written jobs bypass create-time checks.
    _guard_job_credential_exfil(job)

    setup.blocked = _preflight_or_block(job, job_id, job_name, _cfg)
    if setup.blocked is not None:
        return setup

    setup.runtime, setup.model = _resolve_job_runtime(job, job_id, jc)
    setup.fallback_notice = setup.runtime.pop("_fallback_notice", None)
    setup.reasoning_config = _resolve_job_reasoning_config(
        job, _cfg if isinstance(_cfg, dict) else {}, str(setup.model)
    )
    # Mid-run provider ladder: same rule as resolution above, so a pinned job cannot be swapped
    # onto the global chain by a 5xx/429 either.
    setup.fallback_model = _job_fallback_chain(job, _cfg)
    setup.credential_pool = _load_credential_pool(setup.runtime, job_id)
    # MCP servers must be registered before AIAgent is constructed.
    _init_cron_mcp_tools(job_id)
    # Only now can a requested MCP toolset be judged: its alias is process-global but its tools live
    # in this profile's registry overlay, and quiet_mode hides the empty resolution (#109050).
    if _cron_preflight_enabled(_cfg):
        _mcp_reason = _empty_requested_mcp_toolsets(job, _cfg)
        if _mcp_reason:
            setup.blocked = _blocked_config_result(job_id, job_name, _mcp_reason)
    return setup


def _construct_cron_agent(AIAgent, job: dict, _cfg: dict, setup: _CronAgentSetup, *, workdir, session_id, session_db):
    runtime = setup.runtime
    pr = _cfg.get("provider_routing") or {}
    return AIAgent(
        model=setup.model,
        api_key=runtime.get("api_key"),
        base_url=runtime.get("base_url"),
        provider=runtime.get("provider"),
        requested_provider=runtime.get("requested_provider"),
        api_mode=runtime.get("api_mode"),
        request_overrides=runtime.get("request_overrides"),
        acp_command=runtime.get("command"),
        acp_args=runtime.get("args"),
        max_iterations=setup.max_iterations,
        reasoning_config=setup.reasoning_config,
        prefill_messages=setup.prefill_messages,
        fallback_model=setup.fallback_model,
        credential_pool=setup.credential_pool,
        providers_allowed=pr.get("only"),
        providers_ignored=pr.get("ignore"),
        providers_order=pr.get("order"),
        provider_sort=pr.get("sort"),
        openrouter_min_coding_score=(_cfg.get("openrouter") or {}).get("min_coding_score"),
        enabled_toolsets=_resolve_cron_enabled_toolsets(job, _cfg),
        disabled_toolsets=_resolve_cron_disabled_toolsets(_cfg),
        quiet_mode=True,
        # Project context files only with a configured workdir; SOUL.md always.
        skip_context_files=not bool(workdir),
        load_soul_identity=True,
        skip_memory=False,
        skip_background_review=True,  # Cron has no human-in-the-loop need for skill/memory review forks (~30K tok/event)
        platform="cron",
        session_id=session_id,
        session_db=session_db,
    )


class _FireAudit:
    """One usage_audit.jsonl line per fire (created once the agent exists; fire id + start clock)."""

    def __init__(self, job: dict, job_id: str, model: str):
        self.job, self.job_id, self.model = job, job_id, model
        self.fire_id = uuid.uuid4().hex
        self.t_start = time.monotonic()

    def write(self, result: dict, error: Optional[str]) -> None:
        _write_usage_audit({
            "ts": _utcnow_iso_ms(),
            "job_id": self.job_id,
            "fire_id": self.fire_id,
            "prompt_tokens": result.get("prompt_tokens"),
            "completion_tokens": result.get("completion_tokens"),
            "total_tokens": result.get("total_tokens"),
            "response_silent": bool(result.get("response_silent")),
            "deliver_target": self.job.get("deliver"),
            "model": self.model or None,
            "duration_ms": int((time.monotonic() - self.t_start) * 1000),
            "error": error})



def run_job(
    job: dict, *, defer_agent_teardown: Optional[list] = None, extra_prompt: Optional[str] = None,
    cancel_event: Optional[_CancelEventLike] = None, execution_id: Optional[str] = None,
) -> tuple[bool, str, str, Optional[str]]:
    """Execute a single cron job. Returns (success, full_output_doc, final_response, error).
    ``defer_agent_teardown``: if a list, the live agent is appended instead of torn down; the caller
    MUST call ``_teardown_cron_agent(agent)`` AFTER delivery (a torn-down async client can't
    deliver). ``extra_prompt``: per-fire context, never persisted.

    ``defer_agent_teardown``: when a caller passes a list, ``run_job`` skips the agent's async-resource
    teardown (``agent.close()`` + ``cleanup_stale_async_clients()``) in its ``finally`` block and instead
    appends the live agent to that list. The caller is then responsible for calling
    ``_teardown_cron_agent(agent)`` AFTER it has delivered the result. This closes the ordering window in
    #58720 where delivery ran against a torn-down async client (defense-in-depth alongside the
    interpreter-shutdown guard). When ``None`` (the default) teardown happens inline as before, so every
    existing caller is unchanged.
    ``extra_prompt``: optional per-run context from ``cronjob(action='run', prompt=...)`` (#57331). Appended
    to the stored prompt for this fire only — never persisted to the job definition.
    """
    job_id = job["id"]
    job_name = str(job.get("name") or job.get("prompt") or job_id or "cron job")

    early, prompt = _prepare_job_prompt(job, job_id, job_name, extra_prompt, cancel_event)
    if early is not None:
        return early
    from run_agent import AIAgent

    _cron_session_id = f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}"
    logger.info("Running job '%s' (ID: %s)", job_name, job_id)
    logger.info("Prompt: %s", prompt[:100])

    agent = None
    model = ""
    _session_db = None
    _audit: Optional[_FireAudit] = None
    _worker_state: dict = {}
    scope = _CronRunScope(job, job_id, execution_id)
    try:
        scope.enter()
        if scope.workdir:
            logger.info("Job '%s': using task-scoped workdir %s", job_id, scope.workdir)
        _reload_dotenv_and_publish_delivery_target(job)

        jc = _load_cron_job_config(job, job_id, job_name)
        _cfg = jc.cfg
        model = jc.model
        setup = _resolve_cron_agent_setup(job, job_id, job_name, jc)
        if setup.blocked is not None:
            return setup.blocked
        model = setup.model

        # Open state.db only after every early-return gate has passed.
        _session_db = _open_cron_session_db(job)
        agent = _construct_cron_agent(
            AIAgent, job, _cfg, setup, workdir=scope.workdir, session_id=_cron_session_id,
            session_db=_session_db)
        _audit = _FireAudit(job, job_id, model)

        result = _run_agent_with_watchdog(
            agent, prompt, job, job_id, job_name, scope.task_id, cancel_event,
            worker_state=_worker_state)
        final_response = _final_response_from_result(result, job_id, job_name, AIAgent)
        if (setup.fallback_notice and final_response.strip() and not _is_cron_silence_response(final_response)
                and _cron_failure_marker_error(final_response) is None):
            # Pre-agent provider switch (#74349) rides with the delivered report; silence and the
            # agent-declared failure marker keep their first-line/whole-response contract.
            final_response = f"{setup.fallback_notice}\n\n{final_response}"
        # Keep final_response clean for delivery logic (empty = no delivery).
        logged_response = final_response if final_response else "(No response generated)"
        # Text-file reads normalize newlines; count the same characters the
        # context_from reader sees so quoted markers can never become boundaries.
        framed_prompt = _normalize_newlines(prompt)
        logged_response = _normalize_newlines(logged_response)
        output = (
            _run_doc_header(job, job_name, job_id, framed_prompt,
                            prompt_stamp=f"{_PROMPT_FRAME}{len(framed_prompt)}\n")
            + f"{_RESPONSE_FRAME}{len(logged_response)}\n{_RESPONSE_HEADING}{logged_response}{_RESPONSE_TERMINATOR}"
        )
        logger.info("Job '%s' completed successfully", job_name)
        _audit.write(dict(result, response_silent=_is_cron_silence_response(final_response or "")), None)
        return True, output, final_response, None

    except Exception as e:
        error_msg = f"{type(e).__name__}: {str(e)}"
        logger.exception("Job '%s' failed: %s", job_name, error_msg)
        # Cowork-style unreachable-model re-run (cron/unreachable_retry.py): flag failures where
        # the model was never reached (transient network/DNS, zero API calls) so the bookkeeping
        # tail can schedule a bounded automatic re-run instead of waiting a full period.
        try:
            from cron.unreachable_retry import is_model_unreachable_failure
            if is_model_unreachable_failure(e, agent):
                job["_model_unreachable"] = True
            # Provider usage window closed for a known duration (cron/quota_hold.py): flag it so the
            # bookkeeping tail parks the job past the window instead of re-firing into it (#89376).
            from cron.quota_hold import hold_seconds_from_failure
            _hold_s = hold_seconds_from_failure(e)
            if _hold_s:
                job["_quota_hold_seconds"] = _hold_s
        except Exception:  # classification must never mask the real failure
            logger.debug("Job '%s': unreachable-failure classification failed", job_id)
        # No audit row when we failed before the agent existed; the audit write must never raise.
        if _audit is not None:
            _audit.write({}, error_msg)
        from cron.scheduler_diagnostics import format_run_error
        output = (
            _run_doc_header(job, f"{job_name} (FAILED)", job_id, prompt)
            + format_run_error(e)
        )
        return False, output, "", error_msg

    finally:
        from cron.scheduler_detached_worker import defer_teardown_to_running_worker
        _worker_teardown_deferred = defer_teardown_to_running_worker(
            _worker_state.get("future"), _session_db, agent, job_id, job_name, _cron_session_id)
        scope.exit()
        if _session_db and not _worker_teardown_deferred:
            _finalize_cron_session(_session_db, agent, job_id, job_name, _cron_session_id)
        # Tear down the ephemeral agent or the gateway leaks fds per tick (EMFILE). With deferred
        # teardown, hand the live agent back: delivery needs a live async client.
        # Release subprocesses, terminal sandboxes, browser daemons, and the main OpenAI/httpx client held
        # by this ephemeral cron agent. Without this, a gateway that ticks cron every N minutes leaks fds
        # per job until it hits EMFILE (#10200 / "too many open files"). When the caller opted to defer
        # teardown (passed a list), hand the live agent back instead of closing it here — delivery must run
        # against a live async client, and the caller tears down afterwards (#58720).
        if not _worker_teardown_deferred:
            if defer_agent_teardown is not None:
                if agent is not None:
                    defer_agent_teardown.append(agent)
            else:
                _teardown_cron_agent(agent, job_id)


def _teardown_cron_agent(
    agent, job_id: str, *, timeout_seconds: Optional[float] = None
) -> None:
    """Release an ephemeral cron agent's async resources within a hard bound (this runs outside the
    inactivity watchdog). Shared by ``run_job``'s finally and deferred post-delivery teardown.

    Split out of ``run_job``'s ``finally`` so a caller that defers teardown (to deliver first — #58720) can
    invoke the identical cleanup AFTER delivery. The timeout matters because this executes after
    ``run_conversation`` has returned, outside the agent inactivity watchdog.
    """
    def _cleanup_agent() -> None:
        try:
            if agent is not None:
                agent.close()
        except (Exception, KeyboardInterrupt) as e:
            logger.debug("Job '%s': failed to close agent resources: %s", job_id, e)
        # Worker-thread event loop dies with the executor; reap httpx clients cached under it.
        try:
            from agent.auxiliary_client import cleanup_stale_async_clients
            cleanup_stale_async_clients()
        except Exception as e:
            logger.debug("Job '%s': failed to reap stale auxiliary clients: %s", job_id, e)

    _run_cron_cleanup_with_timeout(
        _cleanup_agent, job_id=job_id, label="agent resource teardown",
        timeout_seconds=timeout_seconds)


def _run_with_fire_claim_heartbeat(job: dict, run) -> bool:
    """Run ``run`` while keeping this job's owned durable fire claim fresh."""
    claim = job.get("fire_claim")
    owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
    if not owner:
        return run(None)

    job_id = str(job.get("id") or "")
    stop = threading.Event()
    lost_ownership = threading.Event()

    def _finish_unstarted(error: str) -> None:
        execution_id = job.get("execution_id")
        if not execution_id:
            return
        try:
            finish_execution(execution_id, success=False, error=error)
        except Exception:
            logger.warning(
                "Job '%s': failed to close unstarted execution ledger row",
                job_id,
                exc_info=True)

    try:
        owns_fire_claim = heartbeat_fire_claim(job_id, expected_owner=owner)
    except Exception:
        logger.warning("Job '%s': initial fire_claim validation failed", job_id, exc_info=True)
        _finish_unstarted("Fire claim ownership could not be validated before execution started.")
        return True

    if owns_fire_claim is False:
        logger.warning("Job '%s': fire claim ownership was already lost before execution", job_id)
        _finish_unstarted("Fire claim ownership lost before execution started.")
        return True

    def _heartbeat_loop() -> None:
        last_confirmed = time.monotonic()
        while not stop.wait(_RUN_CLAIM_HEARTBEAT_SECONDS):
            try:
                if not heartbeat_fire_claim(job_id, expected_owner=owner):
                    if self_removal_delivery_allowed(job_id):
                        # Record dropped by this run; nothing left to keep fresh.
                        continue
                    # One miss is a sample, not a verdict (#113357): a latch cancels the live
                    # agent run and ends the lease refresh, so confirm before acting on it.
                    if stop.wait(_FIRE_CLAIM_MISS_CONFIRM_SECONDS) or heartbeat_fire_claim(
                            job_id, expected_owner=owner):
                        last_confirmed = time.monotonic()
                        continue
                    lost_ownership.set()
                    logger.warning(
                        "Job '%s': fire claim ownership lost; interrupting stale run",
                        job_id)
                    return
                last_confirmed = time.monotonic()
            except Exception:
                logger.debug("Job '%s': fire_claim heartbeat failed", job_id, exc_info=True)
                if (
                    time.monotonic() - last_confirmed
                    >= _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS
                ):
                    lost_ownership.set()
                    logger.warning(
                        "Job '%s': fire_claim could not be renewed within %.1fs; "
                        "interrupting uncertain run",
                        job_id,
                        _FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS)
                    return

    heartbeat_thread = _start_heartbeat_thread(
        _heartbeat_loop, "cron-fire-claim-heartbeat",
        lambda: logger.warning(
            "Job '%s': could not start fire_claim heartbeat", job_id, exc_info=True))
    if heartbeat_thread is None:
        _finish_unstarted("Fire claim heartbeat could not be started; execution was not run.")
        return True

    try:
        return run(lost_ownership)
    finally:
        stop.set()
        heartbeat_thread.join(timeout=1.0)


def run_one_job(
    job: dict, *, adapters=None, loop=None, verbose: bool = False,
    extra_prompt: Optional[str] = None, cancel_event: Optional[_CancelEventLike] = None,
) -> bool:
    """Run ONE due job end-to-end: execute → save output → deliver → mark. Shared by the built-in
    ticker and external providers' ``fire_due``; does NOT decide due-ness or acquire the initial
    claim (callers use the store CAS) but keeps it alive. True if processed (a job failure is
    recorded via ``mark_job_run``), False only if processing raised. ``cancel_event``: optional
    transport-level cancel (dashboard drain)."""
    # Every gateway path (built-in scheduler, external providers, and direct
    # API fires) crosses this seam.  Ensure the detached worker has a durable
    # attempt to adopt before any launch can occur.
    if not job.get("execution_id"):
        execution = create_execution(
            job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))
        job["execution_id"] = execution["id"]

    execution_id = str(job["execution_id"])
    note_cron_execution(job)
    external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id
    if not external_owner:
        try:
            handed_off = _launch_external_cron_worker(job)
        except Exception as handoff_error:
            # Arbitrary waiter errors must not claim dispatch failure or resend
            # a worker's notice. A recovered unknown execution is different:
            # its worker never committed a terminal outcome.
            post_handoff = isinstance(handoff_error, _ExternalWorkerPostHandoffError)
            stage = "failed after handoff" if post_handoff else "dispatch failed"
            error = f"Restart-safe cron worker {stage}: {handoff_error}"
            if post_handoff:
                from cron.scheduler_worker_failure import record_unknown_worker_outcome

                if record_unknown_worker_outcome(job, error=str(handoff_error), adapters=adapters, loop=loop):
                    return True
            logger.error("Job '%s': %s", job["id"], error)
            claim = job.get("fire_claim")
            owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
            delivery_error = delivery_outcome = None
            try:
                # A pre-handoff dispatch failure is a job failure like any other: it
                # must open an incident and leave through the job's failure lane
                # (#123401). Without this the outage is silent — no cron_incidents
                # row, no ping — while executions.db keeps piling up failed rows.
                if not post_handoff:
                    # Returns before _run_one_job_body / record_unknown_worker_outcome install the scope; get_secret fails closed without one under multiplex.
                    with _fire_secret_scope():
                        delivery_error, delivery_outcome = _deliver_crash_failure(
                            job, error, adapters=adapters, loop=loop)
                from cron.unreachable_retry import is_retry_run
                mark_job_run(
                    job["id"],
                    False,
                    error,
                    delivery_error=delivery_error,
                    **({"expected_fire_owner": owner} if owner else {}),
                    # A ladder re-run's occurrence already counted toward repeat.
                    **({"ladder_rung": True} if is_retry_run(job) else {}),
                )
            finally:
                finish_execution(
                    execution_id, success=False, error=error,
                    delivery_outcome=delivery_outcome)
            return True
        if handed_off:
            from cron.scheduler_worker_failure import record_unknown_worker_outcome

            record_unknown_worker_outcome(job, adapters=adapters, loop=loop)
            return True
    if extra_prompt is None:
        # Gateway-forwarded manual run stamps its prompt on the job via trigger_job; the fire that
        # consumes the manual occurrence picks it up here. Single-fire: mark_job_run clears it.
        _stamped = job.get("manual_run_prompt")
        if _stamped and job.get("manual_run_at"):
            extra_prompt = str(_stamped)
    claim = job.get("fire_claim")
    fire_owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
    execution_token = object()
    profile_home = _get_hermes_home().resolve()
    _fire_key = _inflight_key(job["id"])
    with _running_lock:
        _running_fire_owners.setdefault(_fire_key, {})[execution_token] = (
            fire_owner or None, profile_home)
    try:
        with self_removal_delivery_scope(job["id"]):
            return _run_with_fire_claim_heartbeat(
                job,
                lambda lost_ownership: _run_one_job_body(
                    job,
                    adapters=adapters,
                    loop=loop,
                    verbose=verbose,
                    extra_prompt=extra_prompt,
                    claim_lost=lost_ownership,
                    transport_cancel=cancel_event,
                    execution_token=execution_token))
    finally:
        with _running_lock:
            executions = _running_fire_owners.get(_fire_key)
            if executions is not None:
                executions.pop(execution_token, None)
                if not executions:
                    _running_fire_owners.pop(_fire_key, None)


_OWNERSHIP_LOST_INTERRUPTED = "Interrupted by shutdown before terminal completion."


def _record_fire_ownership_lost(
    job: dict, fire_owner: Optional[str], execution_id: str,
) -> None:
    """Bookkeeping after fire-claim ownership loss. A transport-level cancel (dashboard drain) is
    not a real loss — we still own the claim, so record the interruption via the owner-fenced
    terminal write instead of leaving fire_claim/last_status stale; otherwise discard.
    An interrupted ladder re-run re-ran an occurrence that already counted, so its terminal
    write must not spend another repeat slot (same as every other terminal path)."""
    from cron.unreachable_retry import is_retry_run
    job_id = job["id"]
    if fire_owner is not None and heartbeat_fire_claim(job_id, expected_owner=fire_owner):
        mark_job_run(
            job_id, False, _OWNERSHIP_LOST_INTERRUPTED, expected_fire_owner=fire_owner,
            **({"ladder_rung": True} if is_retry_run(job) else {}),
        )
        finish_execution(execution_id, success=False, error=_OWNERSHIP_LOST_INTERRUPTED)
    else:
        finish_execution(
            execution_id, success=False,
            error="Fire claim ownership lost; stale result was discarded.")


def _classify_delivery_outcome(
    *, delivery_error, should_deliver: bool, unresolved_origin: bool,
    normalized_deliver: str, incident_acked: bool, success: bool,
    delivery_queued=None, notification_suppressed: bool = False,
) -> str:
    if delivery_error:
        return "failed"
    if should_deliver and delivery_queued:
        return "queued"
    if notification_suppressed:
        return "suppressed"
    if should_deliver and unresolved_origin:
        return "not_configured"
    if should_deliver and normalized_deliver != "local":
        return "delivered"
    if incident_acked and not success:
        # Failure ping withheld for a known signature: operator acked it, or it was already
        # alerted inside the reminder cooldown (vs. plain "suppressed").
        return "suppressed_acked"
    return "suppressed"


def _compose_run_delivery(
    job: dict, *, success: bool, error, final_response: str, output_file,
    agent_declared: bool = False,
) -> tuple[str, bool, bool, bool, Optional[str]]:
    """Text to deliver for a finished run. Returns ``(deliver_content, blocked_config,
    silent_alert, incident_acked, failure_incident_id)``; ``silent_alert``: an alert-once marker
    says the operator was already told, deliver nothing. ``agent_declared``: *error* is the
    agent's own ``[CRON_FAILURE]`` evidence, delivered verbatim."""
    err = str(error) if error else ""
    # Failed jobs always deliver, except blocked-config runs, which alert exactly ONCE.
    blocked_config_silent = BLOCKED_CONFIG_SILENT_MARKER in err
    blocked_config = blocked_config_silent or BLOCKED_CONFIG_MARKER in err
    incident_acked = False
    failure_incident_id = None
    if blocked_config and not success:
        # Bypass the generic failure summarizer (its auth/timeout heuristics would mislabel this).
        _pf_text = re.sub(r"\[blocked_config[^\]]*\]\s*", "", err).strip()
        from cron.scheduler_failure_copy import blocked_config_notice
        deliver_content = blocked_config_notice(job.get("name") or job["id"], _pf_text)
    elif success:
        deliver_content = final_response
        _resolve_incidents_for_recovered_job(job)
    else:
        # Record the job+error signature once; withhold the per-run ping while the operator
        # already acked it (closed) or was already told (alerted, inside the reminder cooldown).
        # Best-effort: a ledger failure never breaks delivery.
        incident_acked, failure_incident_id = _upsert_incident_for_failure(
            job, error or "", output_file=output_file
        )
        if incident_acked:
            deliver_content = ""
        elif agent_declared:
            # The agent already diagnosed the failure in prose; the summarizer's substring
            # heuristics would re-diagnose it ("timed out" -> blame the model service, "401" ->
            # "sign in again") and attach the wrong remediation. Deliver the evidence as-is.
            from cron.scheduler_failure_copy import generic_failure_notice
            deliver_content = generic_failure_notice(
                job.get("name") or job["id"], job["id"], err.strip().rstrip("."),
            ) + _failure_streak_nudge(job)
        else:
            from cron.quota_hold import hold_notice
            deliver_content = (
                _summarize_cron_failure_for_delivery(job, error) + _failure_streak_nudge(job)
                # The one alert on entering a provider-window hold says so (#89376).
                + hold_notice(job, job.get("_quota_hold_seconds"))
            )
    return deliver_content, blocked_config, blocked_config_silent, incident_acked, failure_incident_id


class _FireClaimLostDuringSideEffect(Exception):
    """Raised inside a side-effect fence when the durable fire claim is no longer ours."""


class _FireOwnership:
    """Fire-claim ownership checks for one run (``owner`` is None when the job carries no claim)."""

    def __init__(
        self, job: dict, claim_lost: Optional[_CancelEventLike] = None,
        transport_cancel: Optional[_CancelEventLike] = None,
    ):
        self.job = job
        # Two handles: the heartbeat's raw ``lost_ownership`` event and the caller's transport
        # ``cancel_event``. A sampled miss latches on the raw one only — the combined view's
        # ``set()`` would propagate into the transport event this run does not own (#105861).
        self.claim_lost = claim_lost
        self.transport_cancel = transport_cancel
        # What ``run_job`` receives as its ``cancel_event``: either source cancels the run.
        self.cancel_event: Optional[_CancelEventLike] = (
            _CombinedCancelEvent(claim_lost, transport_cancel)
            if claim_lost is not None or transport_cancel is not None
            else None)
        claim = job.get("fire_claim")
        self.owner = str(claim.get("by") or "") if isinstance(claim, dict) else None

    def transport_cancelled(self) -> bool:
        return self.transport_cancel is not None and self.transport_cancel.is_set()

    def side_effect_fence(self):
        if self.owner is None:
            return contextlib.nullcontext(True)
        return fire_claim_fence(self.job["id"], expected_owner=self.owner)

    def lost(self) -> bool:
        if self.transport_cancelled():
            return True
        if self.owner is None:
            return False
        if self_removal_delivery_allowed(self.job["id"]):
            # The run deleted its own record; there is no claim left to re-resolve.
            return False
        # The heartbeat's latched miss is one sample, not the verdict (#113357): the store is
        # re-checked here, and a claim that still validates keeps the run's real outcome. The
        # owner-fenced mark_job_run / fire_claim_fence remain the authority for side effects.
        latched = self.claim_lost is not None and self.claim_lost.is_set()
        try:
            if heartbeat_fire_claim(self.job["id"], expected_owner=self.owner):
                return False
        except Exception:
            logger.debug(
                "Job '%s': fire_claim ownership validation failed", self.job["id"], exc_info=True)
            # Unreachable store after a latched miss stays fail-closed; a first miss is best-effort.
            return latched
        if self.claim_lost is not None:
            self.claim_lost.set()
        return True


@dataclass
class _RunDelivery:
    """Mutable outcome of the save/compose/deliver phase, read back by the bookkeeping tail."""
    job: dict
    success: bool
    error: Optional[str]
    delivery_attempted: bool = False
    delivery_error: Optional[str] = None
    should_deliver: bool = False
    unresolved_origin: bool = False
    blocked_config: bool = False
    # True when ``error`` is the agent's own ``[CRON_FAILURE]`` evidence rather than a runtime
    # error string, so composition must not run it through the provider-error heuristics.
    agent_declared: bool = False
    incident_acked: bool = False
    failure_incident_id: Optional[str] = None
    side_effect_ownership_lost: bool = False


def _save_compose_deliver(
    d: _RunDelivery, fence: _FireOwnership, final_response: str, output: str, *,
    adapters, loop, verbose: bool, execution_token,
) -> None:
    """Save output, compose the notice and deliver it (both side effects run under the fire-claim
    fence; a lost claim raises ``_FireClaimLostDuringSideEffect`` for the caller)."""
    job = d.job
    with fence.side_effect_fence() as owns_output:
        if not owns_output:
            raise _FireClaimLostDuringSideEffect
        # remove_job() already deleted this job's output dir; saving would re-create an orphan.
        output_file = (
            None if self_removal_delivery_allowed(job["id"])
            else save_job_output(job["id"], output))
    if verbose and output_file is not None:
        logger.info("Output saved to: %s", output_file)

    # A shutdown-killed tool subprocess can leave a plausible final_response from truncated
    # output; force the honest "interrupted" failure path. Peek-only (consumed later).
    if d.success and _is_interrupted(job["id"], execution_token):
        d.success = False
        d.error = (
            "Interrupted by gateway shutdown before the run finished "
            "(tool subprocess was killed mid-flight)."
        )

    (
        deliver_content, d.blocked_config, _silent_alert, d.incident_acked, d.failure_incident_id,
    ) = _compose_run_delivery(
        job, success=d.success, error=d.error, final_response=final_response,
        output_file=output_file, agent_declared=d.agent_declared)
    # Whitespace-only == empty: skip delivery; the guard below marks it a soft failure.
    d.should_deliver = bool(deliver_content.strip()) and not _silent_alert
    if d.should_deliver and not d.success and job.get("_model_unreachable"):
        # The model was never reached and a bounded automatic re-run will be scheduled
        # (cron/unreachable_retry.py): hold the failure notice — the re-run either
        # delivers the real result or, once the ladder is exhausted, the next failure
        # alerts normally. Mirrors Cowork's silent 5/15/30-minute re-runs.
        from cron.unreachable_retry import will_retry
        if will_retry(job):
            d.should_deliver = False
            logger.info(
                "Job '%s': suppressing failure notice — automatic re-run pending", job["id"])
    # Not a substring check: bare "SILENT"/"NO_REPLY" or a report quoting "[SILENT]" must
    # not be swallowed; bracketed-prefix / trailing-line tolerance is kept.
    if d.should_deliver and d.success and _is_cron_silence_response(deliver_content):
        # Cron silence suppression — see _is_cron_silence_response. Replaces the old `SILENT_MARKER in
        # ...upper()` substring check, which both leaked bracketless near-markers ("SILENT" / "NO_REPLY")
        # and wrongly swallowed a real report that merely quoted "[SILENT]" mid-sentence (#51438, #46917).
        logger.info("Job '%s': agent returned %s — skipping delivery", job["id"], SILENT_MARKER)
        d.should_deliver = False

    if d.should_deliver and fence.lost():
        d.should_deliver = False
        logger.warning("Job '%s': skipping delivery after fire claim ownership loss", job["id"])

    if not d.should_deliver:
        return
    d.unresolved_origin = (
        _normalize_deliver_value(_delivery_lane_value(job, for_failure=not d.success)) == "origin"
        and not _resolve_delivery_targets(job, for_failure=not d.success)
    )
    try:
        with fence.side_effect_fence() as owns_delivery:
            if not owns_delivery:
                raise _FireClaimLostDuringSideEffect
            d.delivery_attempted = True
            d.delivery_error = _deliver_result(
                job,
                deliver_content,
                adapters=adapters,
                loop=loop,
                # Failure summaries (and drift/blocked-config alerts composed into deliver_content
                # on the failure path) honor the job's failure_deliver override (NS-788).
                for_failure=not d.success,
            )
    except Exception as de:
        if isinstance(de, _FireClaimLostDuringSideEffect):
            raise
        d.delivery_error = str(de)
        logger.error("Delivery failed for job %s: %s", job["id"], de)


def _finish_interrupted_run(job: dict, execution_id: str, delivery_error: Optional[str]) -> None:
    """Shutdown already wrote last_status, so mark_job_run is skipped (a second call would skip a
    fire or auto-delete the job); an unsent notice is recorded via update_job instead."""
    if delivery_error:
        try:
            # The gateway shutdown already wrote last_status for this run, so mark_job_run is skipped below
            # — but it could not know that the notice we just tried to send never left the process (the
            # adapters were torn down first, #82232). Record the delivery failure on its own via update_job:
            # mark_job_run also advances next_run_at and the repeat counter, and running that a second time
            # for one run would skip a fire or auto-delete the job early.
            from cron.jobs import update_job
            update_job(job["id"], {"last_delivery_error": delivery_error})
        except Exception as _rec_err:
            logger.debug(
                "Failed recording delivery_error for interrupted job %s: %s", job["id"], _rec_err)
    finish_execution(
        execution_id, success=False,
        error="Interrupted by gateway shutdown before terminal completion.")


def _finish_completed_run(d: _RunDelivery, fire_owner: Optional[str], execution_id: str) -> bool:
    """mark_job_run (owner-fenced) + execution ledger row for a run that reached delivery."""
    job = d.job
    if not d.should_deliver and job.get("last_delivery_queued"):
        from cron.jobs import update_job
        update_job(job["id"], {"last_delivery_queued": None})
        job["last_delivery_queued"] = None
    mark_kwargs: dict = {"delivery_error": d.delivery_error}
    if not d.success and job.pop("_model_unreachable", False):
        # Never-reached-the-model failure: schedule the Cowork-style bounded re-run
        # (cron/unreachable_retry.py) inside the same fenced store write.
        mark_kwargs["model_unreachable"] = True
    from cron.unreachable_retry import is_retry_run
    if is_retry_run(job):
        # A re-run of an occurrence that already counted: must not spend another repeat slot.
        mark_kwargs["ladder_rung"] = True
    _hold_s = job.pop("_quota_hold_seconds", None)
    if not d.success and _hold_s:
        # Provider window closed for a known duration: park past it (cron/quota_hold.py, #89376).
        mark_kwargs["quota_hold_seconds"] = _hold_s
        mark_kwargs["recover_consumed_fire"] = bool(job.get("_scheduled_instant"))
    if d.success and not d.delivery_error and d.should_deliver and job.get("last_delivery_queued"):
        mark_kwargs["status"] = "delivery_queued"
    if fire_owner is not None:
        mark_kwargs["expected_fire_owner"] = fire_owner
    if d.blocked_config:
        mark_kwargs["status"] = "blocked_config"
    # A run that removed its own record has nothing left to mark; the delivery above is its result.
    marked = self_removal_delivery_allowed(job["id"]) or mark_job_run(
        job["id"], d.success, d.error, **mark_kwargs)
    if fire_owner is not None and not marked:
        finish_execution(
            execution_id, success=False,
            error="Fire claim ownership lost before terminal completion.")
        return True
    delivery_outcome = _classify_delivery_outcome(
        delivery_error=d.delivery_error,
        delivery_queued=job.get("last_delivery_queued"),
        notification_suppressed=bool(job.get("_notification_all_targets_suppressed")),
        should_deliver=d.should_deliver,
        unresolved_origin=d.unresolved_origin,
        # Read the lane the notice was actually routed through (failure_deliver on failure).
        normalized_deliver=_normalize_deliver_value(_delivery_lane_value(job, for_failure=not d.success)),
        incident_acked=d.incident_acked,
        success=d.success,
    )
    if delivery_outcome in ("delivered", "not_configured") and not d.success:
        # Failure ping left the process (or had a configured target): mark the incident alerted.
        _mark_incident_alerted(d.failure_incident_id)
    finish_execution(
        execution_id, success=d.success, error=d.error, delivery_outcome=delivery_outcome)
    return True


def _deliver_crash_failure(
    job: dict, err_text: str, *, adapters, loop,
) -> tuple[Optional[str], str]:
    """Failure notice for a run that raised out of run_job. Returns (delivery_error, outcome)."""
    normalized_deliver = _normalize_deliver_value(_delivery_lane_value(job, for_failure=True))
    # Same ack gate as the normal failure delivery: acked signatures stay silent here too.
    incident_acked, failure_incident_id = _upsert_incident_for_failure(job, err_text)
    if incident_acked:
        return None, "suppressed_acked"
    delivery_error = None
    try:
        delivery_error = _deliver_result(
            job,
            # Same text as the normal failure delivery: this run also counts toward
            # failure_streak, so the nudge must leave through here too.
            _summarize_cron_failure_for_delivery(job, err_text) + _failure_streak_nudge(job),
            adapters=adapters,
            loop=loop,
            for_failure=True,
        )
    except Exception as delivery_exc:
        delivery_error = str(delivery_exc)
        logger.error("Delivery failed for job %s: %s", job["id"], delivery_exc)
    unresolved_origin = bool(
        not delivery_error
        and normalized_deliver == "origin"
        and not _resolve_delivery_targets(job, for_failure=True)
    )
    delivery_outcome = _classify_delivery_outcome(
        delivery_error=delivery_error, should_deliver=True, unresolved_origin=unresolved_origin,
        normalized_deliver=normalized_deliver, incident_acked=False, success=False,
        delivery_queued=job.get("last_delivery_queued"),
        notification_suppressed=bool(job.get("_notification_all_targets_suppressed")))
    if delivery_outcome in ("delivered", "not_configured"):
        _mark_incident_alerted(failure_incident_id)
    return delivery_error, delivery_outcome



def _install_fire_secret_scope() -> "tuple[contextvars.Token, Optional[contextvars.Token]]":
    """Install the firing profile's secret scope for the span ``_run_one_job_body`` runs, delivery
    included, and return the tokens ``_reset_fire_secret_scope`` needs.

    Hydrate the profile's external secret sources BEFORE freezing the scope — the order
    gateway/run.py and the external cron worker already use: ``build_profile_secret_scope`` only
    READS the per-home source map, so a scope frozen first would carry no vault-backed value.

    For a fire routed to a profile other than the process's own (``routed_profile_fire`` on the
    fire's home — every entry point: ticker, dashboard Run now, CLI) also run under multiplex semantics — for
    exactly this span and no wider. The desktop backend ticks every local profile from a process
    that is not a multiplexer, so nothing else isolates that fire; and switching the context on
    here rather than at the tick means no read is ever fail-closed without a scope to read — the
    restart-safe handoff in ``run_one_job`` runs before this and keeps today's semantics (#107692).
    """
    from agent.secret_scope import (
        build_profile_secret_scope, set_multiplex_context, set_secret_scope)
    from cron.scheduler_provider import routed_profile_fire
    from hermes_cli.env_loader import hydrate_profile_secret_sources

    home = Path(_get_hermes_home())
    hydrate_profile_secret_sources(home)
    scope_token = set_secret_scope(build_profile_secret_scope(home), profile_home=str(home))
    context_token = set_multiplex_context(True) if routed_profile_fire() else None
    return scope_token, context_token


def _reset_fire_secret_scope(tokens: "tuple[contextvars.Token, Optional[contextvars.Token]]") -> None:
    """Undo ``_install_fire_secret_scope`` — the context first, so multiplex semantics never
    outlive the scope they depend on."""
    from agent.secret_scope import reset_multiplex_context, reset_secret_scope

    scope_token, context_token = tokens
    if context_token is not None:
        reset_multiplex_context(context_token)
    reset_secret_scope(scope_token)


@contextlib.contextmanager
def _fire_secret_scope():
    """``_install_fire_secret_scope`` for the ``with`` block, reset on the way out."""
    tokens = _install_fire_secret_scope()
    try:
        yield
    finally:
        _reset_fire_secret_scope(tokens)


def _start_owned_run(job: dict, execution_id: str) -> "Optional[contextvars.Token]":
    """Win the run's ``claimed`` → ``running`` CAS and bind its cron identity; ``None`` when the run
    lost ownership first. A restart-safe worker already won it by adopting the row."""
    if os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id:
        record = get_execution(execution_id)
    elif (record := mark_execution_running(execution_id)) is None:
        return None
    return enter_cron_execution(job, execution_id, record or {})


def _run_one_job_body(
    job: dict, *, adapters=None, loop=None, verbose: bool = False,
    extra_prompt: Optional[str] = None, claim_lost: Optional[_CancelEventLike] = None,
    transport_cancel: Optional[_CancelEventLike] = None,
    execution_token: Optional[object] = None,
) -> bool:
    fence = _FireOwnership(job, claim_lost, transport_cancel)
    fire_owner = fence.owner
    _side_effect_fence = fence.side_effect_fence
    _fire_claim_ownership_lost = fence.lost

    execution_id = job.get("execution_id")
    if not execution_id:
        execution_id = create_execution(
            job["id"], source="direct", scheduled_instant=job.get("_scheduled_instant"))["id"]
    delivery_attempted = False
    delivery_error = None

    _fire_scope_tokens = None
    _terminal_scope_token = None
    _identity_token = None
    try:
        # get_secret() fails closed outside a scope; the ticker thread has none. Delivery adapters
        # resolve credentials, so the scope must span delivery too (reset in the outer finally) —
        # including the crash notice in the except below when claim_dispatch or
        # mark_execution_running raises, so install it before either.
        _fire_scope_tokens = _install_fire_secret_scope()
        # Commit a finite one-shot's dispatch BEFORE its side effect so a tick dying mid-run cannot
        # re-fire it forever on restart. No-op for recurring/infinite jobs (at-most-times).
        # This lives here in the shared body so BOTH the built-in ticker and the external provider (Chronos
        # fire_due) get at-most-times semantics. See #38758.
        if not claim_dispatch(job["id"]):
            logger.info(
                "Job '%s': one-shot dispatch limit reached — skipping",
                job.get("name", job["id"]))
            finish_execution(
                execution_id, success=False,
                error="Dispatch claim rejected; execution was not started.")
            return True  # not an error — already handled/removed

        # Claimed durably before dispatch; becomes running only right before the actual run.
        # Detached workers transition to running while adopting; in-process paths must win the
        # claimed->running CAS here before any user script or agent side effect may begin.
        # The identity plugins read via ctx.current_cron_execution() is bound only on that win.
        _identity_token = _start_owned_run(job, execution_id)
        if _identity_token is None:
            logger.warning("Cron job %s lost execution ownership before start; skipping", job["id"])
            return True

        # Bind the firing profile's COMPLETE terminal policy for this fire — agent build, run, delivery
        # and bookkeeping (gateway/run.py _profile_runtime_scope does the same per turn) — else the
        # ticker reads process-global TERMINAL_* env another profile pinned (#68559, #98581). A
        # malformed policy installs a refusal scope: terminal execution raises instead.
        from tools.terminal_scope import (
            install_profile_terminal_scope)

        _terminal_scope_token = install_profile_terminal_scope(_get_hermes_home())
        # Defer agent teardown until AFTER delivery: closing first races the live send against a
        # torn-down async client (#58720). run_job hands the agent back via this list instead.
        _deferred_agents: list = []

        def _teardown_deferred() -> None:
            # run_job's finally still hands back the agent when it raises; tear it down here so a failed run
            # never leaks its async resources (#10200), then re-raise into the outer handler. BaseException
            # (not just Exception) so a KeyboardInterrupt/SystemExit mid-run still triggers teardown before
            # propagating.
            # Tear down the deferred agent(s) now that save + delivery have run (or raised). Must happen on
            # every path so cron agents never leak their subprocesses/clients (#10200).
            for _deferred_agent in _deferred_agents:
                _teardown_cron_agent(_deferred_agent, job["id"])

        _run_kwargs = {
            "defer_agent_teardown": _deferred_agents,
            "extra_prompt": extra_prompt,
            "execution_id": execution_id}
        if fence.cancel_event is not None:
            _run_kwargs["cancel_event"] = fence.cancel_event
        try:
            success, output, final_response, error = run_job(job, **_run_kwargs)
        except BaseException:
            # run_job hands back the agent even when raising; tear down so a failed run never leaks.
            # BaseException so KeyboardInterrupt/SystemExit mid-run still trigger teardown.
            _teardown_deferred()
            raise

        if _fire_claim_ownership_lost():
            _teardown_deferred()
            _record_fire_ownership_lost(job, fire_owner, execution_id)
            return True

        # An agent can finish its own turn after a delegated child has failed. Let it explicitly
        # declare that semantic failure so the existing failure path updates status, streaks,
        # ledger, and notification routing instead of recording a false healthy result.
        agent_declared = False
        if success and not job.get("no_agent"):
            marker_error = _cron_failure_marker_error(final_response)
            if marker_error is not None:
                success, error, agent_declared = False, marker_error, True

        # Agent is still live through delivery; wrap ALL of save/compose/deliver in try/finally so a
        # raise anywhere still tears the deferred agent down.
        d = _RunDelivery(job=job, success=success, error=error, agent_declared=agent_declared)
        try:
            _save_compose_deliver(
                d, fence, final_response, output, adapters=adapters, loop=loop, verbose=verbose,
                execution_token=execution_token)
        except _FireClaimLostDuringSideEffect:
            d.side_effect_ownership_lost = True
        finally:
            delivery_attempted, delivery_error = d.delivery_attempted, d.delivery_error
            # Every path must tear down deferred agent(s) so they never leak subprocesses/clients.
            _teardown_deferred()

        if d.side_effect_ownership_lost:
            # The claim died inside a side-effect fence: the side effect did NOT complete.
            _record_fire_ownership_lost(job, fire_owner, execution_id)
            return True

        # Empty final_response is a soft failure so last_status is not "ok".
        if d.success and not final_response.strip():
            d.success = False
            d.error = "Agent completed but produced empty response (model error, timeout, or misconfiguration)"

        if _fire_claim_ownership_lost():
            # #105861: the claim check is one sample; a miss AFTER a completed delivery must not
            # overwrite the delivered run's terminal status — ok, or a failure whose notice already
            # left with its real error — so fall through to _finish_completed_run, whose owner-fenced
            # mark_job_run is authoritative either way. An explicit transport cancel stays fail-closed.
            transport_cancelled = fence.transport_cancelled()
            if d.delivery_attempted and not d.delivery_error and not transport_cancelled:
                logger.warning(
                    "Job '%s': fire claim ownership lost after completed delivery; "
                    "recording the delivered run's terminal status",
                    job["id"])
            else:
                if transport_cancelled:
                    logger.warning(
                        "Job '%s': transport cancellation arrived before terminal completion; "
                        "recording the interrupted run",
                        job["id"])
                _record_fire_ownership_lost(job, fire_owner, execution_id)
                return True

        if _consume_interrupted_flag(job["id"], execution_token):
            _finish_interrupted_run(job, execution_id, delivery_error)
            return True

        return _finish_completed_run(d, fire_owner, execution_id)

    except BaseException as e:  # noqa: BLE001 — deliberate: see below
        # BaseException, not Exception: CancelledError/KeyboardInterrupt/SystemExit propagate here.
        # Without mark_job_run(False) a finite one-shot is wedged: claim_dispatch consumed
        # repeat.completed but last_run_at is never written. Record first, then re-raise
        # non-Exception. Owner fencing still applies.
        # BaseException, not Exception (#73973): the inner run_job handler re-raises CancelledError /
        # KeyboardInterrupt / SystemExit after agent teardown, and none of those are Exception subclasses.
        # If they escape without mark_job_run(False), a finite one-shot is left wedged — claim_dispatch()
        # already consumed repeat.completed, but last_run_at is never written, so the job sits in state
        # "scheduled" until the run-claim TTL expires and the dispatch-limit guard removes it with no output
        # and no error. Owner fencing still applies: a stale worker must not record over a replacement claim
        # owner.
        _err_text = str(e) or type(e).__name__
        logger.error(
            "Error processing job %s: %s",
            job["id"],
            _err_text,
            exc_info=(type(e), e, e.__traceback__))
        delivery_outcome = "suppressed"
        # Owner fencing: a stale worker whose claim was taken over (or transport-cancelled) must not
        # send a failure alert on top of the replacement run's; fall through to fenced bookkeeping.
        if (
            isinstance(e, Exception)
            and not delivery_attempted
            and not isinstance(e, _FireClaimLostDuringSideEffect)
            and not _fire_claim_ownership_lost()
        ):
            delivery_error, delivery_outcome = _deliver_crash_failure(
                job, _err_text, adapters=adapters, loop=loop)
        try:
            if (
                not _consume_interrupted_flag(job["id"], execution_token)
                and not self_removal_delivery_allowed(job["id"])  # no record left to mark
            ):
                mark_kwargs = {}
                if fire_owner is not None:
                    mark_kwargs["expected_fire_owner"] = fire_owner
                if isinstance(e, Exception):
                    mark_kwargs["delivery_error"] = delivery_error
                from cron.unreachable_retry import is_retry_run
                if is_retry_run(job):
                    # A crashed ladder re-run: its occurrence already counted toward repeat.
                    mark_kwargs["ladder_rung"] = True
                mark_job_run(job["id"], False, _err_text, **mark_kwargs)
        except Exception as record_err:
            # Never let bookkeeping mask the original interruption.
            logger.error("Failed to record interrupted run for job %s: %s", job["id"], record_err)
        try:
            finish_execution(
                execution_id, success=False, error=_err_text, delivery_outcome=delivery_outcome)
        except Exception as record_err:
            logger.error("Failed to finish execution record for job %s: %s", job["id"], record_err)
        if not isinstance(e, Exception):
            raise
        return False
    finally:
        # Function-level on purpose: must scope delivery, deferred teardown, claim-loss handling and
        # bookkeeping — not just run_job. Do not move into the run block's finally.
        exit_cron_execution(_identity_token)
        if _fire_scope_tokens is not None:
            _reset_fire_secret_scope(_fire_scope_tokens)
        if _terminal_scope_token is not None:
            from tools.terminal_scope import reset_terminal_scope

            reset_terminal_scope(_terminal_scope_token)


def _wait_for_external_cron_worker_body(
    process: subprocess.Popen,
    *,
    execution_id: str,
    stderr_path: Optional[Path] = None,
) -> bool:
    """Preserve ``run_one_job``'s synchronous contract after handoff.

    The worker owns the durable execution and survives this gateway process.
    The caller nevertheless waits while it remains alive so manual/background
    callers do not release their in-process guard or report stale job state.
    A gateway replacement may kill this waiter; it does not kill the scoped
    worker or change its ledger ownership.
    """
    def _is_terminal() -> bool:
        current = get_execution(execution_id)
        return bool(current and current.get("status") in _TERMINAL_STATES)

    # The worker commits its terminal row before its process exits, so exit is
    # the correct wakeup.  Each ledger read opens a connection and re-runs
    # schema init; polling it at 50ms for an hours-long agent run is ~72k
    # opens/hour of pure contention with the worker's own writes.
    while True:
        try:
            returncode = process.wait(timeout=1.0)
        except subprocess.TimeoutExpired:
            if not _is_terminal():
                continue
            # Recovery can win between the timeout and this ledger read. If
            # the worker has since exited, consume its diagnostics below before
            # cleanup; only a still-live terminal worker needs background reaping.
            returncode = process.poll()
            if returncode is None:
                from cron.scheduler_detached_worker import reap_terminal_worker_in_background

                reap_terminal_worker_in_background(process)
                return True
        # The worker can commit its terminal row and exit between the first
        # read and wait(). Re-read the exact attempt before declaring that
        # it died without terminalizing.
        current = get_execution(execution_id)
        if current and current.get("status") in ("completed", "failed"):
            return True
        # If the adopted worker died without terminalizing, its owner is
        # now provably gone. Recover to ``unknown`` rather than routing the
        # exception through the pre-handoff dispatch-failure path, which
        # would falsely assert that no side effect could have happened.
        #
        # Record the cause THIS waiter observed instead of letting the blanket
        # sweep stamp "Scheduler restarted ...", and then RAISE: the sweep leaves
        # the row terminal, so returning success here made run_one_job skip
        # mark_job_run entirely and the job's fire claim outlived its lost
        # execution, refusing the next manual fire with "Job is already being
        # fired" (#128509). Raising routes the existing post-handoff bookkeeping
        # instead: retire the claim and report the uncertain run, not a pre-dispatch failure.
        from cron.scheduler_worker_failure import external_worker_exited_reason

        reason = external_worker_exited_reason(returncode)
        if not terminalize_dead_owner(execution_id, reason=reason):
            recover_interrupted_executions()
            # A concurrent sweep may have won the unknown transition. Still
            # surface this waiter's exit code and stderr instead of discarding them.
            current = get_execution(execution_id)
            if current and current.get("status") in ("completed", "failed"):
                return True
        stderr_tail = ""
        if stderr_path is not None:
            from cron.scheduler_diagnostics import external_worker_stderr_tail

            stderr_tail = external_worker_stderr_tail(stderr_path)
        raise RuntimeError(f"{reason}{stderr_tail}")


class _ExternalWorkerPostHandoffError(RuntimeError):
    """The waiter failed after the worker was spawned and may own the execution."""


def _wait_for_external_cron_worker(
    process: subprocess.Popen,
    *,
    execution_id: str,
    job_id: Optional[str] = None,
    handoff_files: tuple[Path, ...] = (),
    stderr_path: Optional[Path] = None,
) -> bool:
    try:
        return _wait_for_external_cron_worker_body(
            process, execution_id=execution_id, stderr_path=stderr_path
        )
    except Exception as wait_error:
        raise _ExternalWorkerPostHandoffError(str(wait_error)) from wait_error
    finally:
        if job_id is not None:
            _key = _inflight_key(job_id)
            with _running_lock:
                _restart_safe_waiter_job_ids.discard(_key)
                _scope_isolated_job_ids.discard(_key)
        # The execution is terminal or its worker is dead: nobody will read a
        # payload or acknowledgement left behind by a late/unread handoff.
        for stale in handoff_files:
            try:
                stale.unlink(missing_ok=True)
            except OSError:
                pass


def _launch_external_cron_worker(job: dict) -> bool:
    """Launch *job* outside the managed gateway process when required.

    Returns ``False`` outside a managed systemd gateway (in-process path).  In
    managed topology the job always goes to an external worker with the #101940
    ownership handoff: in a transient user scope, or — when no user D-Bus
    session exists and ``cron.require_restart_safe_scope`` is false — as a
    direct subprocess (process separation kept, cgroup isolation lost).

    A fire routed to a profile other than the process's own is multiplexed at THIS boundary too.
    ``run_one_job`` switches the context on in ``_install_fire_secret_scope``, which runs AFTER
    this handoff, so a routed desktop fire on the managed path serialized ``multiplex_active=False``
    and built the worker environment with the launch profile's residue and no scrub (review on
    f5f88d5058). Enable it for exactly this span; the worker then re-establishes it from the payload.
    """
    execution_id = str(job["execution_id"])
    job_id = str(job["id"])
    handoff_dir = _get_hermes_home() / "cron" / "external-workers"
    payload_path = handoff_dir / f"{execution_id}.json"
    ack_path = handoff_dir / f"{execution_id}.ready"
    # Captured so a worker that dies before its acknowledgement can name the cause (#112729).
    stderr_path = handoff_dir / f"{execution_id}.stderr"
    command = [
        sys.executable,
        "-m",
        "cron.scheduler",
        "--external-worker-file",
        str(payload_path),
        "--ack-file",
        str(ack_path),
    ]

    from agent.secret_scope import is_multiplex_active
    from cron.scheduler_provider import routed_profile_fire
    from tools.environments.local import build_subprocess_env, strip_launch_profile_env
    from tools.process_registry import (
        restart_safe_gateway_child_argv,
        scoped_spawn_lost_user_bus,
        systemd_user_bus_env,
    )

    # A fire routed to another profile is multiplexed at this handoff even when the process flag is
    # off (the desktop ticker is not a multiplexer): the payload says so, and the worker env is built
    # under that context so the scrub and the launch-residue strip both apply. The context is set
    # for exactly the env build; nothing else here reads it.
    multiplex_active = is_multiplex_active() or routed_profile_fire()
    try:
        require_restart_safe_scope = bool(
            (load_config_readonly().get("cron") or {}).get("require_restart_safe_scope", False)
        )
    except Exception:
        require_restart_safe_scope = False
    dispatch = restart_safe_gateway_child_argv(
        command,
        unit_suffix=f"cron-{job_id}-exec-{execution_id}",
        require_restart_safe_scope=require_restart_safe_scope,
    )
    if dispatch.mode == "in_process":
        return False

    if mark_execution_handoff_pending(execution_id) is None:
        raise RuntimeError(
            "cron execution claim changed before external worker handoff"
        )

    _ensure_cron_dir(handoff_dir)
    try:
        handoff_dir.chmod(0o700)
    except OSError:
        pass
    fd = os.open(payload_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as payload_file:
            json.dump(
                {
                    "job": job,
                    "profile_home": str(_get_hermes_home().resolve()),
                    "multiplex_active": multiplex_active,
                },
                payload_file,
            )
            payload_file.flush()
            os.fsync(payload_file.fileno())
    except BaseException:
        payload_path.unlink(missing_ok=True)
        raise

    profile_home = _get_hermes_home().resolve()
    # Same hydrate -> scope -> (routed) multiplex-context install the in-process fire uses, for exactly
    # the env build; the helper's reset order keeps the context from outliving its scope.
    with _fire_secret_scope():
        # No restore_managed_env here: the worker re-runs load_hermes_dotenv -> _apply_managed_env at
        # import, and strip_launch_profile_env leaves managed keys in place.
        worker_env = strip_launch_profile_env(build_subprocess_env(
            scrub_secrets=multiplex_active,
            inherit_profile_home=True,
            extra={"HERMES_HOME": str(profile_home)},
        ))
    worker_env = systemd_user_bus_env(worker_env)
    # Unattended worker: the gateway sets HERMES_EXEC_ASK at startup (interactive launches set
    # the other two), and an inherited presence var makes every env-fallback consumer in the
    # child (`_is_interactive_cli`, sudo prompting, `check_cronjob_requirements`) believe a
    # human is present to answer (#110932).
    for _presence_var in (
        "HERMES_INTERACTIVE",
        "HERMES_GATEWAY_SESSION",
        "HERMES_EXEC_ASK",
    ):
        worker_env.pop(_presence_var, None)
    # `-m cron.scheduler` has no hermes_cli.main bootstrap; pin this checkout explicitly
    # (PYTHONSAFEPATH / stale editable mapping, #112729), hand the child the committed
    # dependency generation (#122222), and mark it so its own entry runs the PM dependency
    # boot. See cron/scheduler_worker_env.py and cron/worker_bootstrap.py.
    from cron.scheduler_worker_env import pin_hermes_tree_on_pythonpath
    repo_root = Path(__file__).resolve().parent.parent
    worker_env = pin_hermes_tree_on_pythonpath(worker_env, repo_root)
    worker_env[WORKER_MARKER] = "1"
    try:
        stderr_fd = os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
        try:
            process = subprocess.Popen(
                dispatch.argv,
                cwd=str(repo_root),
                env=worker_env,
                stdin=subprocess.DEVNULL,
                stdout=subprocess.DEVNULL,
                stderr=stderr_fd,
                start_new_session=True,
                creationflags=windows_hide_flags(),
            )
        finally:
            os.close(stderr_fd)
    except BaseException:
        payload_path.unlink(missing_ok=True)
        stderr_path.unlink(missing_ok=True)
        raise

    with _running_lock:
        _restart_safe_waiter_job_ids.add(_inflight_key(job_id))

    # Same window the dead-owner recovery ledger grants a pending handoff: a cold
    # worker start (imports + secret hydration) measures ~10-12s in the field, and
    # a dispatch deadline shorter than the adoption grace made the two guards
    # around one handoff disagree.
    deadline = time.monotonic() + HANDOFF_ADOPTION_GRACE_SECONDS
    while time.monotonic() < deadline:
        if ack_path.exists():
            try:
                acknowledgement = json.loads(ack_path.read_text(encoding="utf-8-sig"))
            except Exception:
                logger.exception(
                    "Cron external worker %s published an unreadable acknowledgement; "
                    "treating handoff as ownership-uncertain",
                    execution_id,
                )
                return _wait_for_external_cron_worker(
                    process,
                    execution_id=execution_id,
                    job_id=job_id,
                    handoff_files=(payload_path, stderr_path),
                    stderr_path=stderr_path,
                )
            finally:
                ack_path.unlink(missing_ok=True)
            if (
                not isinstance(acknowledgement, dict)
                or acknowledgement.get("execution_id") != execution_id
            ):
                logger.error(
                    "Cron external worker acknowledgement mismatch for %s; "
                    "treating handoff as ownership-uncertain",
                    execution_id,
                )
                return _wait_for_external_cron_worker(
                    process,
                    execution_id=execution_id,
                    job_id=job_id,
                    handoff_files=(payload_path, stderr_path),
                    stderr_path=stderr_path,
                )
            logger.info(
                "Cron job '%s' handed to restart-safe worker pid=%s execution=%s dispatch=%s",
                job_id,
                acknowledgement.get("pid"),
                execution_id,
                dispatch.mode,
            )
            _record_external_cron_worker(
                job_id,
                acknowledgement.get("pid") or process.pid,
                scope_isolated=dispatch.mode == "scoped",
            )
            return _wait_for_external_cron_worker(
                process,
                execution_id=execution_id,
                job_id=job_id,
                handoff_files=(payload_path, stderr_path),
                stderr_path=stderr_path,
            )
        returncode = process.poll()
        if returncode is not None:
            with _running_lock:
                _restart_safe_waiter_job_ids.discard(_inflight_key(job_id))
            payload_path.unlink(missing_ok=True)
            from cron.scheduler_diagnostics import external_worker_stderr_tail
            stderr_tail = external_worker_stderr_tail(stderr_path)
            stderr_path.unlink(missing_ok=True)
            if dispatch.mode == "scoped" and scoped_spawn_lost_user_bus(worker_env):
                # systemd-run itself failed before any worker ran, so the captured stderr
                # holds nothing useful: name the cause, not the exit code.
                raise RuntimeError(
                    "restart-safe systemd scope could not be created: the user D-Bus session at "
                    f"/run/user/{os.getuid()}/bus disappeared after the gateway started. On a "  # windows-footgun: ok — scoped dispatch exists only on Linux
                    "system-level service install, run `sudo loginctl enable-linger <gateway-user>`; "
                    "the next fire dispatches without scope isolation."
                )
            raise RuntimeError(
                f"cron external worker exited before ownership acknowledgement "
                f"(exit {returncode}){stderr_tail}"
            )
        time.sleep(0.05)

    # The child may have adopted the durable row just before publishing its
    # acknowledgement.  Never fall back to in-process execution on an uncertain
    # handoff: that could duplicate side effects.  The execution owner/dead-owner
    # recovery ledger remains the authority.
    logger.warning(
        "Cron external worker for job '%s' did not acknowledge within %.0fs; "
        "leaving the durable execution claim untouched",
        job_id,
        HANDOFF_ADOPTION_GRACE_SECONDS,
    )
    return _wait_for_external_cron_worker(
        process,
        execution_id=execution_id,
        job_id=job_id,
        handoff_files=(payload_path, ack_path, stderr_path),
        stderr_path=stderr_path,
    )


def _run_external_worker_payload(payload_path: Path, ack_path: Path) -> bool:
    """Adopt and execute one gateway-dispatched cron payload.

    The execution row is created by the gateway before spawn, then transferred
    here before the ready acknowledgement is published.  No side effect runs
    unless that durable ownership transfer succeeds.
    """
    try:
        payload = json.loads(payload_path.read_text(encoding="utf-8-sig"))
        job = payload["job"]
        profile_home = Path(payload["profile_home"]).resolve()
        execution_id = str(job["execution_id"])
    except Exception:
        logger.exception("Cron external worker could not load payload %s", payload_path)
        return False
    finally:
        try:
            payload_path.unlink(missing_ok=True)
        except OSError:
            pass

    from agent.secret_scope import (
        build_profile_secret_scope,
        is_multiplex_active,
        reset_secret_scope,
        set_multiplex_active,
        set_secret_scope,
    )
    from cron.executions import adopt_claimed_execution
    from hermes_cli.env_loader import hydrate_profile_secret_sources
    from hermes_constants import (
        reset_hermes_home_override,
        set_hermes_home_override,
    )

    previous_multiplex = is_multiplex_active()
    home_token = secret_token = None
    try:
        home_token = set_hermes_home_override(profile_home)
        multiplex_active = bool(payload.get("multiplex_active", False))
        set_multiplex_active(multiplex_active)
        # Plugin secret sources (``ctx.register_secret_source()``) only exist after plugin
        # discovery; this process starts with the builtin registry alone, so hydrating without it
        # silently dropped every plugin-sourced credential (#121929). Runs under the home override
        # so a multiplexed worker loads the OWNING profile's plugins, not the launch profile's.
        from hermes_cli.plugins import discover_plugins

        discover_plugins()
        hydrate_profile_secret_sources(profile_home)
        secret_token = set_secret_scope(build_profile_secret_scope(profile_home), profile_home=str(profile_home))
        # This process never ran gateway startup, so the OWNING profile's ``hooks:`` block (shell
        # hooks + ``hooks.outbound``) was never registered and cron sessions silently lost it
        # (#131764). Same helper the gateway uses per profile; it must run inside the secret scope
        # because ``secret_env`` targets resolve through ``get_secret`` (raises unscoped under multiplex).
        from gateway.run_startup import GatewayStartupMixin

        GatewayStartupMixin._register_config_hooks(
            "Cron external worker: config hook registration failed", level=logging.WARNING)
        with use_cron_store(profile_home):
            if adopt_claimed_execution(execution_id) is None:
                logger.error(
                    "Cron external worker refused execution %s: durable ownership "
                    "could not be established",
                    execution_id,
                )
                return False
            try:
                ack_path.parent.mkdir(parents=True, exist_ok=True)
                # Publish via write-to-temp + atomic rename. Writing ack_path in place
                # (the old approach) let O_CREAT make the empty file visible to the
                # scheduler's exists()-then-read polling loop before the JSON body was
                # written, occasionally handing it a 0-byte file and a JSONDecodeError.
                # os.replace() is a single atomic syscall on the same filesystem, so
                # readers only ever see the file fully absent or fully written.
                ack_tmp_path = ack_path.with_name(f"{ack_path.name}.tmp{os.getpid()}")
                fd = os.open(ack_tmp_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
                try:
                    with os.fdopen(fd, "w", encoding="utf-8") as ack_file:
                        json.dump({"pid": os.getpid(), "execution_id": execution_id}, ack_file)
                        ack_file.flush()
                        os.fsync(ack_file.fileno())
                    os.replace(ack_tmp_path, ack_path)
                except BaseException:
                    ack_tmp_path.unlink(missing_ok=True)
                    raise
            except Exception:
                logger.exception(
                    "Cron external worker could not publish ready acknowledgement for %s",
                    execution_id,
                )
                return False
            old_external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER")
            os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = execution_id
            try:
                completed = run_one_job(job, adapters=None, loop=None, verbose=False)
                # Successful return: the worker recorded its outcome. If it raises,
                # retain fd 2's pathname until the waiter consumes the traceback.
                with contextlib.suppress(OSError):
                    ack_path.with_suffix(".stderr").unlink(missing_ok=True)
                return completed
            finally:
                if old_external_execution is None:
                    os.environ.pop("_HERMES_CRON_EXTERNAL_WORKER", None)
                else:
                    os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = old_external_execution
    finally:
        if secret_token is not None:
            reset_secret_scope(secret_token)
        set_multiplex_active(previous_multiplex)
        if home_token is not None:
            reset_hermes_home_override(home_token)


def _notify_provider_jobs_changed() -> None:
    """Best-effort: tell the active scheduler provider the job set changed. Call AFTER a successful
    store mutation so an external provider can re-provision/cancel the one-shot; no-op for the
    built-in. Kept out of cron/jobs.py (import cycle). Never raises."""
    try:
        from cron.scheduler_provider import resolve_cron_scheduler
        resolve_cron_scheduler().on_jobs_changed()
    except Exception as e:
        logger.debug("on_jobs_changed notify failed: %s", e)


class CronSchedulerRegistrationError(RuntimeError):
    """A job was persisted but its first external trigger was not registered."""

    def __init__(self, job: dict, cause: Exception) -> None:
        self.job = job
        self.cause = cause
        super().__init__(
            f"Cron job '{job['id']}' was saved, but its first scheduler "
            f"registration failed ({type(cause).__name__}). Do not create a "
            "duplicate. Pause/resume or update the job to retry registration."
        )

    def user_message(self) -> str:
        """Human-facing variant for chat/CLI surfaces (no exception class name)."""
        label = self.job.get("name") or self.job["id"]
        return (
            f"Saved cron job '{label}', but couldn't register it with the "
            "external scheduler yet. The job is kept — don't re-create it; "
            "pause/resume or edit it (e.g. via /cron) to retry registration."
        )

    def to_dict(self) -> dict:
        """Return the public partial-failure contract without provider details."""
        return {
            "error": str(self),
            "job_id": self.job["id"],
            "job_saved": True,
            "scheduler_registered": False,
            "retry_create": False}


def create_job_with_scheduler_registration(**kwargs) -> dict:
    """Persist one job and register its first trigger with the active provider."""
    from cron.jobs import create_job
    from cron.scheduler_provider import resolve_cron_scheduler

    job = create_job(**kwargs)
    if not job.get("enabled", True):
        return job
    try:
        resolve_cron_scheduler().register_job(job)
    except Exception as exc:
        raise CronSchedulerRegistrationError(job, exc) from exc
    return job


# Dead-owner reap is throttled (opens the executions ledger). Tests may reset
# _last_dead_owner_reap_at to {} to force a reap next tick.
# Dead-owner claim reclaim throttle (#86721): recover_interrupted_executions opens the executions ledger, so
# the per-tick reap is rate-limited rather than run on every idle 60s cycle.
# The throttle is keyed by profile home: under multiplex_profiles the ticker
# ticks every profile each cycle, and a process-global slot would let the
# first profile starve all the others.
_DEAD_OWNER_REAP_INTERVAL_SECONDS = 300.0
_last_dead_owner_reap_at: Dict[str, float] = {}

# Worktree prune throttle: the cron tick is the only reliably periodic process on gateway boxes.
_WORKTREE_MAINTENANCE_INTERVAL_SECONDS = 6 * 3600.0
_last_worktree_maintenance_at: Optional[float] = None
_worktree_maintenance_lock = threading.Lock()


def _worktree_maintenance_repos() -> List[str]:
    """Repos whose ``.worktrees/`` to keep pruned: the hermes checkout plus job workdir repo roots,
    filtered to those that actually have a ``.worktrees/`` dir."""
    repos: set = set()

    # Hermes source checkout (git installs only; wheel installs have no .git).
    with contextlib.suppress(Exception):
        install_root = Path(__file__).resolve().parent.parent
        if (install_root / ".git").exists():
            repos.add(str(install_root))

    with contextlib.suppress(Exception):
        from cron.jobs import load_jobs

        for job in load_jobs():
            workdir = str(job.get("workdir") or "").strip()
            if not workdir or not Path(workdir).is_dir():
                continue
            try:
                probe = subprocess.run(
                    ["git", "rev-parse", "--show-toplevel"],
                    capture_output=True, text=True, encoding="utf-8",
                    errors="replace", timeout=5, cwd=workdir)
                if probe.returncode == 0 and probe.stdout.strip():
                    repos.add(probe.stdout.strip())
            except Exception:
                continue

    return [r for r in sorted(repos) if (Path(r) / ".worktrees").is_dir()]


def _maybe_run_worktree_maintenance() -> None:
    """Throttled worktree prune from the cron tick, on a daemon thread so the tick never waits on
    git. Same conservative pruner as ``hermes -w`` startup (dirty/unpushed/locked trees untouched).
    Errors never propagate: GC is hygiene, not scheduling."""
    global _last_worktree_maintenance_at
    now = time.monotonic()
    with _worktree_maintenance_lock:
        if (
            _last_worktree_maintenance_at is not None
            and now - _last_worktree_maintenance_at
            < _WORKTREE_MAINTENANCE_INTERVAL_SECONDS
        ):
            return
        _last_worktree_maintenance_at = now

    def _run() -> None:
        try:
            repos = _worktree_maintenance_repos()
            if not repos:
                return
            from hermes_cli.worktree_ops import _prune_stale_worktrees

            for repo in repos:
                try:
                    _prune_stale_worktrees(repo)
                except Exception:
                    logger.debug("Cron worktree maintenance failed for %s", repo, exc_info=True)
        except Exception:
            logger.debug("Cron worktree maintenance skipped", exc_info=True)

    threading.Thread(target=_run, name="cron-worktree-prune", daemon=True).start()


def _acquire_tick_lock(lock_file):
    """Open + non-blocking lock the tick file (fcntl / msvcrt). Returns the fd, or None on genuine
    contention. A real OSError (esp. EMFILE/ENFILE) must NOT pass as contention — the scheduler
    would look healthy while no job runs — so it is re-raised for the ticker to record a FAILED
    tick."""
    lock_fd = None
    try:
        # Cross-platform file locking: fcntl on Unix, msvcrt on Windows. Only genuine lock contention
        # (another ticker holds the lock) skips the tick silently. A real OSError — most importantly
        # EMFILE/ENFILE from fd exhaustion — must NOT be swallowed as "another instance holds the lock":
        # that previously made the scheduler appear healthy (tick returned 0, heartbeat recorded success)
        # while no job ever ran again (#87644).
        lock_fd = open(lock_file, "w", encoding="utf-8")
        if fcntl:
            fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        elif msvcrt:
            msvcrt.locking(lock_fd.fileno(), msvcrt.LK_NBLCK, 1)
        return lock_fd
    except OSError as exc:
        if lock_fd is not None:
            with contextlib.suppress(OSError):
                lock_fd.close()
            if _is_lock_contention_errno(exc):
                logger.info("Tick skipped — another instance holds the lock")
                return None
        if _is_fd_exhaustion(exc):
            # fd reclamation is the ticker loop's job (scheduler_provider.py); here would double it.
            logger.error(
                "Cron tick could not acquire tick lock: %s — scheduler will "
                "attempt fd reclamation and retry with backoff",
                exc)
        else:
            logger.error("Cron tick could not acquire tick lock: %s", exc)
        raise


def _release_tick_lock(lock_fd) -> None:
    if fcntl:
        with contextlib.suppress((OSError, IOError)):
            fcntl.flock(lock_fd, fcntl.LOCK_UN)
    elif msvcrt:
        with contextlib.suppress((OSError, IOError)):
            msvcrt.locking(lock_fd.fileno(), msvcrt.LK_UNLCK, 1)
    lock_fd.close()


def _maybe_reap_dead_owners() -> None:
    """Dead-owner reclaim: a run that died mid-flight would leave its row 'claimed' forever. Rows
    whose owner process is proved gone are released (_owner_is_live), as are rows whose live owner
    holds a claim older than the derived stale bound (the process is not killed). Throttled."""
    # Dead-owner claim reclaim (#86721): execution rows carry their owner pid + process start time, but
    # recovery previously ran only at scheduler STARTUP. A one-shot `hermes cron run` that claimed a job and
    # died mid-run (its runner thread lived in the exiting CLI process) left the row 'claimed' forever while
    # the long-lived gateway ticker kept running — blocking every future run of that job. Reap provably-dead
    # owners periodically so stale claims auto-clear without a gateway restart. Throttled so idle 60s ticks
    # don't pay a ledger connection every cycle (#33612).
    _reap_key = hermes_home_key(_get_hermes_home())
    _reap_now = time.monotonic()
    _last_reap = _last_dead_owner_reap_at.get(_reap_key)
    if (
        _last_reap is not None
        and _reap_now - _last_reap < _DEAD_OWNER_REAP_INTERVAL_SECONDS
    ):
        return
    _last_dead_owner_reap_at[_reap_key] = _reap_now
    try:
        from cron.executions import recover_interrupted_executions

        _reclaimed = recover_interrupted_executions()
        if _reclaimed:
            logger.warning(
                "Reclaimed %d cron execution(s) whose owner process died "
                "before reaching a terminal state (marked unknown)",
                _reclaimed)
    except Exception as _reap_exc:
        logger.debug("Dead-owner execution reclaim failed: %s", _reap_exc)


def _sweep_stale_inflight_for_tick(due_jobs: list) -> None:
    """Bound the in-flight set BEFORE the dedup guard so a leaked claim is force-released now
    rather than eating every later fire until restart. Skipped when nothing is in flight."""
    _local_home = hermes_home_key(_get_hermes_home())
    if not any(key[0] == _local_home for key in _running_job_ids):
        return
    _sweep_jobs = due_jobs
    with contextlib.suppress(Exception):
        _inflight_ids = {key[1] for key in _running_job_ids if key[0] == _local_home}
        _due_ids = {j.get("id") for j in due_jobs if isinstance(j, dict)}
        if not _inflight_ids <= _due_ids:
            from cron.jobs import load_jobs as _load_all_jobs

            _sweep_jobs = _load_all_jobs()
    try:
        sweep_stale_inflight(_sweep_jobs)
    except Exception as e:
        logger.warning("Stale in-flight sweep failed: %s", e)


def _resolve_max_parallel_workers() -> Optional[int]:
    """Max workers: env > config.yaml > unbounded (HERMES_CRON_MAX_PARALLEL=1 restores serial)."""
    try:
        _env_par = cron_env_setting("HERMES_CRON_MAX_PARALLEL").strip()
        if _env_par:
            return int(_env_par) or None
    except (ValueError, TypeError):
        logger.warning("Invalid HERMES_CRON_MAX_PARALLEL value; defaulting to unbounded")
    with contextlib.suppress(Exception):
        _ucfg = load_config() or {}
        _cfg_par = (_ucfg.get("cron", {}) if isinstance(_ucfg, dict) else {}).get("max_parallel_jobs")
        if _cfg_par is not None:
            return int(_cfg_par) or None
    return None


def _sweep_mcp_orphans() -> None:
    """Reap MCP stdio orphans (only PIDs flagged by tools.mcp_tool._run_stdio's finally block);
    run AFTER jobs finish so live sessions are never touched."""
    try:
        from tools.mcp_tool_lifecycle import _kill_orphaned_mcp_children
        _kill_orphaned_mcp_children()
    except Exception as _e:
        logger.debug("Post-tick MCP orphan cleanup failed: %s", _e)


def _process_due_job(job: dict, adapters, loop, verbose: bool) -> bool:
    """Run one due job via the shared ``run_one_job`` body."""
    # Claim only when the worker actually starts, so a queued lease can't expire first.
    claimed = claim_job_for_fire(job["id"], return_job=True)
    if not claimed:
        finish_execution(
            job["execution_id"], success=False, error="Fire claim lost; execution was not started.")
        return True
    # CAS returns the persisted record; bool fallback only for older test doubles.
    claimed_job = dict(claimed) if isinstance(claimed, dict) else dict(job)
    claimed_job["execution_id"] = job["execution_id"]
    claimed_job["_scheduled_instant"] = job.get("_scheduled_instant")
    return run_one_job(claimed_job, adapters=adapters, loop=loop, verbose=verbose)


def _submit_with_guard(job: dict, pool: concurrent.futures.ThreadPoolExecutor, process_job):
    """Submit with the in-flight dedup guard; None if a prior tick's run is still in flight.
    Running-set membership is released in the worker's finally."""
    job_id = job["id"]
    job_label = job.get("name", job_id)

    def _clear_run_claim_best_effort() -> None:
        """Best-effort claim cleanup on dispatch-failure paths. Only one-shots carry a run_claim;
        clear_run_claim takes _jobs_lock + full load/save and can raise on degraded paths
        (shutdown, EMFILE) — a claim expiring at TTL beats crashing the tick.

        Only one-shot jobs carry a ``run_claim`` (stamped by get_due_jobs, #59229), so recurring jobs skip
        the call entirely — clear_run_claim acquires _jobs_lock (blocking cross-process flock) and does a
        full load_jobs read, and the dispatch-failure paths fire exactly when the process can least afford N
        pointless lock/read round-trips (interpreter shutdown, EMFILE).  clear_run_claim itself does
        load_jobs/save_jobs file I/O; on those degraded paths it can raise, and these early-exits exist
        precisely to skip cleanly — a stale claim expiring at the TTL is a better outcome than crashing the
        tick (#86522).
        """
        _schedule = job.get("schedule")
        if not (isinstance(_schedule, dict) and _schedule.get("kind") == "once"):
            return
        try:
            clear_run_claim(job_id)
        except Exception as claim_err:
            logger.warning(
                "Could not clear run_claim for job '%s' after dispatch "
                "failure: %s (claim will expire at TTL)",
                job_label, claim_err)

    def _not_dispatched_shutdown() -> None:
        logger.warning("Job '%s' not dispatched — interpreter is shutting down", job_label)

    # During interpreter shutdown pool.submit raises; skip — the job fires on the next tick.
    # If the interpreter is finalizing (gateway SIGTERM / restart / OOM), scheduling any new delivery is
    # futile — asyncio.run and a fresh ThreadPoolExecutor both raise "cannot schedule new futures after
    # interpreter shutdown". Skip gracefully with a warning rather than emitting an ERROR traceback on every
    # restart-race (#58720, #55924).
    # A tick can race gateway teardown: once the interpreter is finalizing, ``pool.submit`` raises "cannot
    # schedule new futures after interpreter shutdown" and crashes the tick. Skip cleanly — the job stays
    # due and will fire on the next healthy tick (#58720, #55924).
    if _interpreter_shutting_down():
        _not_dispatched_shutdown()
        _clear_run_claim_best_effort()
        return None
    registration_owner = object()
    if not try_register_running_job(job_id, owner=registration_owner):
        logger.info("Job '%s' already running — skipping", job_label)
        return None
    # The home the claim was registered under. The pool worker's ``finally`` runs OUTSIDE
    # ``ctx.run``, where the per-profile cron scope is not bound, so releasing without it would
    # discard the LAUNCH home's key and leak every secondary profile's claim.
    _claim_home = _get_hermes_home()
    # Record the attempt before dispatch; recovery marks abandoned rows unknown (no retry).
    try:
        execution = create_execution(
            job_id, source="builtin", scheduled_instant=job.get("_scheduled_instant"))
        dispatched_job = dict(job, execution_id=execution["id"])
        note_cron_execution(dispatched_job)
        _ctx = contextvars.copy_context()
    except Exception as execution_err:
        # Release the claim so the next tick retries instead of wedging "already running".
        release_running_job(job_id, home=_claim_home, owner=registration_owner)
        _clear_run_claim_best_effort()
        logger.exception(
            "Job '%s' not dispatched: execution creation failed: %s", job_label, execution_err)
        return None

    def _run_and_release(j=dispatched_job, ctx=_ctx, home=_claim_home):
        try:
            return ctx.run(process_job, j)
        finally:
            release_running_job(j["id"], home=home, owner=registration_owner)

    try:
        fut = pool.submit(_run_and_release)
    except Exception as submit_err:
        release_running_job(job_id, home=_claim_home, owner=registration_owner)
        _clear_run_claim_best_effort()
        finish_execution(
            execution["id"], success=False, error=f"Executor dispatch failed: {submit_err}")
        if isinstance(submit_err, RuntimeError) and _interpreter_shutting_down(submit_err):
            _not_dispatched_shutdown()
        else:
            logger.error("Job '%s' not dispatched: %s", job_label, submit_err)
        return None

    with _running_lock:
        _submit_key = _inflight_key(job_id, _claim_home)
        if _running_registration_owners.get(_submit_key) is registration_owner:
            _running_futures[_submit_key] = fut
    return fut


def _sweep_mcp_orphans_when_all_done(futures: list) -> None:
    """Async (gateway ticker) mode: sweep via a done-callback after the LAST job completes; sweep
    inline when nothing was dispatched (all skipped / no due jobs)."""
    if not futures:
        _sweep_mcp_orphans()
        return
    _remaining = [len(futures)]

    def _on_done(_f: concurrent.futures.Future) -> None:
        _remaining[0] -= 1
        with contextlib.suppress(Exception):
            _exc = _f.exception()
            if _exc is not None:
                logger.error(
                    "Cron job future failed in async mode: %s", _exc,
                    exc_info=(type(_exc), _exc, _exc.__traceback__))
        if _remaining[0] <= 0:
            _sweep_mcp_orphans()

    for _f in futures:
        _f.add_done_callback(_on_done)


from cron.scheduler_tick import tick  # noqa: E402


# ---------------------------------------------------------------------------
# Split modules. Imported at the bottom (import cycle: they late-bind ``cron.scheduler`` as
# ``_sched``). Only names this module itself calls; everything else lives in the split module.
# ---------------------------------------------------------------------------
from cron.scheduler_delivery import (  # noqa: E402
    _deliver_result, _delivery_lane_value, _normalize_deliver_value, _resolve_delivery_target,
    _resolve_delivery_targets,
)
from cron.scheduler_script import (  # noqa: E402
    _get_session_db_timeout, _run_job_script_with_claim_heartbeat, _start_heartbeat_thread,
)
from cron.scheduler_prompt import (  # noqa: E402
    _PROMPT_FRAME, _PROMPT_HEADING, _PROMPT_SEPARATOR, _RESPONSE_FRAME, _RESPONSE_HEADING,
    _RESPONSE_TERMINATOR, _block_and_pause_job, _build_job_prompt, _guard_job_credential_exfil,
    _parse_wake_gate,
)
from cron.scheduler_preflight import (  # noqa: E402
    BLOCKED_CONFIG_MARKER, BLOCKED_CONFIG_SILENT_MARKER, _cron_preflight_enabled,
    _empty_requested_mcp_toolsets, _is_transient_provider_resolve_error, _preflight_job_config,
)


# `python -m cron.scheduler` entry: MUST stay below the split-module imports so the worker /
# tick paths see every name they need.
if __name__ == "__main__":
    if "--external-worker-file" in sys.argv:
        import argparse

        parser = argparse.ArgumentParser(add_help=False)
        parser.add_argument("--external-worker-file", type=Path, required=True)
        parser.add_argument("--ack-file", type=Path, required=True)
        args = parser.parse_args()
        # The gateway spawns this worker with stdout on DEVNULL and stderr on a
        # capture file it only reads back if we die before the ack; without a
        # log handler every adoption/ack failure below would otherwise be
        # invisible to the persistent log.
        try:
            from hermes_logging import setup_logging

            setup_logging(hermes_home=_get_hermes_home(), mode="cron")
        except Exception:
            pass
        raise SystemExit(
            0 if _run_external_worker_payload(args.external_worker_file, args.ack_file) else 1
        )
    tick(verbose=True)