fix(cron): fire a cron job's unreachable-model retry instead of re-anchoring it
unreachable_retry.plan_retry parks a cron job at now + ladder delay, an instant that is off the expression's lattice. The stale-cron guard on the due scan classified it as a direct schedule edit (STALE_CRON_EXPR_EDIT) and re-anchored it to the natural occurrence without firing, so the 5/15/30-minute ladder never ran for cron jobs — the same failure mode the quota-hold recovery fire had to exempt itself from. Record the ladder instant in the retry state and let _reanchor_stale_cron accept either planner's parked instant as an authorized one-shot off-lattice fire; interval jobs and legacy state without the instant keep the previous behaviour.
This commit is contained in:
@@ -11,7 +11,7 @@ from datetime import datetime, timedelta, timezone
|
||||
import pytest
|
||||
|
||||
from cron import unreachable_retry as ur
|
||||
from cron.jobs import create_job, get_job, mark_job_run
|
||||
from cron.jobs import create_job, get_due_jobs, get_job, mark_job_run
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -26,9 +26,13 @@ def _iso(dt: datetime) -> str:
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(tmp_cron_home):
|
||||
def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(
|
||||
tmp_cron_home, monkeypatch,
|
||||
):
|
||||
"""Failed-unreachable runs re-fire on the 5/15/30-minute ladder instead of waiting a
|
||||
full period, and the ladder stops after its last rung (falls back to the schedule)."""
|
||||
full period, and the ladder stops after its last rung (falls back to the schedule). A
|
||||
cron job's ladder instant is off its lattice yet must be due, not re-anchored as a stale
|
||||
expression edit."""
|
||||
# Interval, not a cron expression: the natural next fire is always a full day out. A
|
||||
# fixed clock time ("0 3 * * *") makes the 30-minute rung land past the natural fire
|
||||
# in the half hour before it, and plan_retry rightly yields to the schedule (CI red).
|
||||
@@ -51,6 +55,16 @@ def test_unreachable_failure_pulls_next_run_earlier_then_ladder_exhausts(tmp_cro
|
||||
assert j.get(ur.STATE_KEY) is None
|
||||
assert datetime.fromisoformat(j["next_run_at"]) - now > timedelta(hours=1)
|
||||
|
||||
pinned = datetime(2026, 9, 18, 12, 1, tzinfo=timezone.utc)
|
||||
monkeypatch.setattr("cron.jobs._hermes_now", lambda: pinned)
|
||||
monkeypatch.setattr(ur, "_hermes_now", lambda: pinned)
|
||||
weekly = create_job("weekly digest", "0 12 * * 5")
|
||||
assert mark_job_run(weekly["id"], False, "ConnectError: dns", model_unreachable=True)
|
||||
retry_at = datetime.fromisoformat(get_job(weekly["id"])["next_run_at"])
|
||||
assert retry_at == pinned + timedelta(seconds=ur.RETRY_DELAYS_SECONDS[0])
|
||||
monkeypatch.setattr("cron.jobs._hermes_now", lambda: retry_at + timedelta(seconds=1))
|
||||
assert weekly["id"] in {due["id"] for due in get_due_jobs()}
|
||||
|
||||
|
||||
def test_reaching_the_model_resets_ladder_and_oneshots_never_retry(tmp_cron_home):
|
||||
"""Any run that reached the model clears retry state; one-shots (pre-claimed
|
||||
|
||||
Reference in New Issue
Block a user