fix(cron): fire a cron job's unreachable-model retry instead of re-anchoring it

unreachable_retry.plan_retry parks a cron job at now + ladder delay, an
instant that is off the expression's lattice. The stale-cron guard on the
due scan classified it as a direct schedule edit (STALE_CRON_EXPR_EDIT) and
re-anchored it to the natural occurrence without firing, so the 5/15/30-minute
ladder never ran for cron jobs — the same failure mode the quota-hold recovery
fire had to exempt itself from. Record the ladder instant in the retry state
and let _reanchor_stale_cron accept either planner's parked instant as an
authorized one-shot off-lattice fire; interval jobs and legacy state without
the instant keep the previous behaviour.
This commit is contained in:
kshitijk4poor
2026-09-24 16:19:44 +05:30
committed by kshitij
parent da46661e4a
commit 3e3cfec49d
3 changed files with 33 additions and 11 deletions

View File

@@ -11,7 +11,7 @@ from datetime import datetime, timedelta, timezone
import pytest
from cron import unreachable_retry as ur
from cron.jobs import create_job, get_job, mark_job_run
from cron.jobs import create_job, get_due_jobs, get_job, mark_job_run
@pytest.fixture
@@ -26,9 +26,13 @@ def _iso(dt: datetime) -> str:
return dt.isoformat()
def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(tmp_cron_home):
def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(
tmp_cron_home, monkeypatch,
):
"""Failed-unreachable runs re-fire on the 5/15/30-minute ladder instead of waiting a
full period, and the ladder stops after its last rung (falls back to the schedule)."""
full period, and the ladder stops after its last rung (falls back to the schedule). A
cron job's ladder instant is off its lattice yet must be due, not re-anchored as a stale
expression edit."""
# Interval, not a cron expression: the natural next fire is always a full day out. A
# fixed clock time ("0 3 * * *") makes the 30-minute rung land past the natural fire
# in the half hour before it, and plan_retry rightly yields to the schedule (CI red).
@@ -51,6 +55,16 @@ def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(tmp_cro
assert j.get(ur.STATE_KEY) is None
assert datetime.fromisoformat(j["next_run_at"]) - now > timedelta(hours=1)
pinned = datetime(2026, 9, 18, 12, 1, tzinfo=timezone.utc)
monkeypatch.setattr("cron.jobs._hermes_now", lambda: pinned)
monkeypatch.setattr(ur, "_hermes_now", lambda: pinned)
weekly = create_job("weekly digest", "0 12 * * 5")
assert mark_job_run(weekly["id"], False, "ConnectError: dns", model_unreachable=True)
retry_at = datetime.fromisoformat(get_job(weekly["id"])["next_run_at"])
assert retry_at == pinned + timedelta(seconds=ur.RETRY_DELAYS_SECONDS[0])
monkeypatch.setattr("cron.jobs._hermes_now", lambda: retry_at + timedelta(seconds=1))
assert weekly["id"] in {due["id"] for due in get_due_jobs()}
def test_reaching_the_model_resets_ladder_and_oneshots_never_retry(tmp_cron_home):
"""Any run that reached the model clears retry state; one-shots (pre-claimed