2bd75984ab
The previous commit stopped a re-run from the unreachable-model ladder from counting toward repeat, but only on the normal _finish_completed_run path. The other places that record a failed fire on the same claimed snapshot still counted it: the crash/interrupt tail of _run_one_job_body, the pre-handoff dispatch failure in run_one_job, and the recovered unknown worker outcome in scheduler_worker_failure. A re-run that crashed or failed to dispatch therefore still bumped repeat.completed a second time for an occurrence that had already counted. All three now pass ladder_rung=True when is_retry_run(job). The worker-failure job is the same run_one_job snapshot, which carries _scheduled_instant (stamped by the due scan and copied by _process_due_job), so is_retry_run applies there too. Manual runs carry no _scheduled_instant and still count. The kept test gains a "crash" row: a re-run whose run_job raises must leave repeat.completed at 1.
62 lines
2.5 KiB
Python
62 lines
2.5 KiB
Python
"""Job-level visibility for a recovered, acknowledged worker death."""
|
|
|
|
|
|
def record_unknown_worker_outcome(
|
|
job: dict, *, error=None, adapters=None, loop=None
|
|
) -> bool:
|
|
"""Project only this fire's uncertain outcome, without retrying its side effects.
|
|
|
|
A terminal ledger row alone does not retire the job claim or notify its owner.
|
|
Fence the notice and bookkeeping together so recovery cannot overwrite a newer
|
|
fire or duplicate a failure the worker already recorded.
|
|
"""
|
|
from cron import scheduler
|
|
from cron.unreachable_retry import is_retry_run
|
|
|
|
execution = scheduler.get_execution(str(job["execution_id"]))
|
|
if not execution or execution.get("status") != "unknown":
|
|
return False
|
|
claim = job.get("fire_claim")
|
|
owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
|
|
if not owner:
|
|
return False
|
|
with scheduler.fire_claim_fence(job["id"], expected_owner=owner) as owns_claim:
|
|
if not owns_claim:
|
|
return True
|
|
error = error or execution["error"]
|
|
scheduler.logger.error("Job '%s': %s", job["id"], error)
|
|
scope_tokens = scheduler._install_fire_secret_scope()
|
|
delivery_error = None
|
|
try:
|
|
delivery_error, _ = scheduler._deliver_crash_failure(
|
|
job, error, adapters=adapters, loop=loop
|
|
)
|
|
finally:
|
|
scheduler._reset_fire_secret_scope(scope_tokens)
|
|
scheduler.mark_job_run(
|
|
job["id"],
|
|
False,
|
|
error,
|
|
delivery_error=delivery_error,
|
|
expected_fire_owner=owner,
|
|
# A ladder re-run's occurrence already counted toward repeat.
|
|
**({"ladder_rung": True} if is_retry_run(job) else {}),
|
|
)
|
|
return True
|
|
|
|
|
|
def external_worker_exited_reason(returncode: int | None) -> str:
|
|
"""Cause for an attempt whose restart-safe worker this process watched exit.
|
|
|
|
The generic dead-owner sweep can only say the owner vanished, so it names a
|
|
scheduler restart — but this waiter never lost its scheduler: it handed the
|
|
attempt to the external worker and then saw that worker exit with this status.
|
|
The run is still ``unknown`` rather than ``failed`` (whether side effects ran is
|
|
still unknown); only the cause becomes truthful (#128509).
|
|
"""
|
|
return (
|
|
f"Restart-safe cron worker exited with status {returncode} after adopting this "
|
|
"execution, without writing a durable terminal state; whether side effects ran "
|
|
"is unknown."
|
|
)
|