hermes-agent/tests/cron/test_parallel_pool.py

"""Tests for the persistent parallel pool and running-job guard in cron/scheduler.py.

These verify the fix for the tick-blocking issue where as_completed(timeout=600)
prevented the ticker thread from firing, causing all other jobs to be fast-forwarded.
"""

import concurrent.futures
import threading
from unittest.mock import patch

import pytest


class TestPersistentPool:
    """_get_parallel_pool returns a persistent ThreadPoolExecutor."""

    def test_pool_is_reused(self, monkeypatch):
        """Same pool instance returned when max_workers doesn't change."""
        import cron.scheduler as sched

        # Reset module state.
        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()

        pool1 = sched._get_parallel_pool(4)
        pool2 = sched._get_parallel_pool(4)
        assert pool1 is pool2

        # Cleanup.
        sched._shutdown_parallel_pool()


    def test_shutdown_clears_pool(self, monkeypatch):
        """_shutdown_parallel_pool resets state."""
        import cron.scheduler as sched

        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._get_parallel_pool(2)

        sched._shutdown_parallel_pool()
        assert not sched._parallel_pools
        assert not sched._parallel_pool_max_workers


class TestRunningJobGuard:
    """_running_job_ids prevents double-dispatch of active jobs."""

    @pytest.mark.parametrize("with_workdir", [False, True])
    def test_running_set_prevents_double_dispatch(self, tmp_path, monkeypatch, with_workdir):
        """A job already in _running_job_ids is skipped on the next tick."""
        import cron.scheduler as sched

        # Reset state.
        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._running_job_ids.clear()

        job = {
            "id": "guard-job",
            "name": "guard-test",
            "prompt": "test",
            "schedule": "every 5m",
            "enabled": True,
            "next_run_at": "2020-01-01T00:00:00",
            "deliver": "local",
        }

        if with_workdir:
            job["workdir"] = str(tmp_path)

        # Simulate the job already running.
        sched._running_job_ids.add(sched._inflight_key("guard-job"))

        dispatched = []
        monkeypatch.setattr(sched, "get_due_jobs", lambda: [job])
        monkeypatch.setattr(sched, "claim_job_for_fire", lambda *_a, **_kw: True)
        monkeypatch.setattr(sched, "run_job", lambda j, **_kw: dispatched.append(j["id"]) or (True, "out", "resp", None))
        monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)

        n = sched.tick(verbose=False)
        assert n == 0  # skipped, not dispatched
        assert dispatched == []

        sched._running_job_ids.discard(sched._inflight_key("guard-job"))
        sched._shutdown_parallel_pool()


    def test_fire_claim_is_acquired_only_when_executor_worker_starts(self, monkeypatch):
        """Queue wait must not consume the durable claim TTL."""
        import cron.scheduler as sched

        sched._running_job_ids.clear()
        job = {
            "id": "queued-job",
            "name": "queued",
            "prompt": "test",
            "schedule": "every 5m",
            "enabled": True,
            "next_run_at": "2020-01-01T00:00:00",
            "deliver": "local",
        }
        submitted = []
        claim_calls = []

        class DeferredPool:
            def submit(self, callback):
                future = concurrent.futures.Future()
                submitted.append((callback, future))
                return future

        monkeypatch.setattr(sched, "get_due_jobs", lambda: [job])
        monkeypatch.setattr(sched, "_get_parallel_pool", lambda _workers: DeferredPool())
        monkeypatch.setattr(
            sched,
            "create_execution",
            lambda *_a, **_kw: {"id": "execution-1"},
        )
        monkeypatch.setattr(
            sched,
            "claim_job_for_fire",
            lambda job_id, **kwargs: claim_calls.append((job_id, kwargs))
            or {**job, "fire_claim": {"by": "worker-owner", "at": "now"}},
        )
        monkeypatch.setattr(sched, "run_one_job", lambda *_a, **_kw: True)

        assert sched.tick(verbose=False, sync=False) == 1
        assert claim_calls == []
        assert len(submitted) == 1

        callback, future = submitted[0]
        result = callback()
        future.set_result(result)

        assert claim_calls == [("queued-job", {"return_job": True})]
        assert sched._inflight_key("queued-job") not in sched._running_job_ids


    def test_create_execution_failure_does_not_wedge_running_set(self, tmp_path, monkeypatch):
        """create_execution failures clear the running lock and still allow next jobs."""
        import cron.scheduler as sched

        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._running_job_ids.clear()

        failing_job = {
            "id": "failing-job",
            "name": "failing-job",
            "prompt": "test",
            "schedule": "every 5m",
            "enabled": True,
            "next_run_at": "2020-01-01T00:00:00",
            "deliver": "local",
        }
        healthy_job = {
            "id": "healthy-job",
            "name": "healthy-job",
            "prompt": "test",
            "schedule": "every 5m",
            "enabled": True,
            "next_run_at": "2020-01-01T00:00:00",
            "deliver": "local",
        }

        called = []

        def create_execution_side_effect(job_id, source, **kwargs):
            if job_id == "failing-job":
                raise RuntimeError("execution ledger unavailable")
            return {"id": f"{job_id}-execution"}

        monkeypatch.setattr(sched, "get_due_jobs", lambda: [failing_job, healthy_job])
        monkeypatch.setattr(sched, "advance_next_runs", lambda *_a, **_kw: 0)
        monkeypatch.setattr(sched, "create_execution", create_execution_side_effect)
        monkeypatch.setattr(sched, "run_job", lambda j, **_kw: called.append(j["id"]) or (True, "out", "resp", None))
        monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "finish_execution", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "claim_dispatch", lambda *_a, **_kw: True)
        monkeypatch.setattr(
            sched,
            "claim_job_for_fire",
            lambda job_id, **_kw: dict(
                healthy_job, fire_claim={"by": "test-owner", "at": "now"}
            )
            if job_id == "healthy-job"
            else None,
        )
        monkeypatch.setattr(sched, "mark_execution_running", lambda *_a, **_kw: {})
        monkeypatch.setattr(sched, "heartbeat_fire_claim", lambda *_a, **_kw: True)

        n = sched.tick(verbose=False)

        assert n == 1
        assert called == ["healthy-job"]
        assert sched._inflight_key("failing-job") not in sched._running_job_ids
        assert sched._inflight_key("healthy-job") not in sched._running_job_ids

        sched._shutdown_parallel_pool()


class TestSyncMode:
    """tick() blocks by default (sync=True); tick(sync=False) returns immediately."""

    def test_sync_true_blocks_and_returns_correct_count(self, tmp_path, monkeypatch):
        """sync=True waits for jobs and returns actual results."""
        import cron.scheduler as sched

        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._running_job_ids.clear()

        jobs = [
            {"id": f"job-{i}", "name": f"Job {i}", "prompt": "test",
             "schedule": "every 5m", "enabled": True,
             "next_run_at": "2020-01-01T00:00:00", "deliver": "local"}
            for i in range(3)
        ]

        monkeypatch.setattr(sched, "get_due_jobs", lambda: jobs)
        monkeypatch.setattr(sched, "claim_job_for_fire", lambda *_a, **_kw: True)
        monkeypatch.setattr(sched, "run_job", lambda j, **_kw: (True, "out", "resp", None))
        monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: "/tmp/out")
        monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)

        n = sched.tick(verbose=False)
        assert n == 3

        sched._shutdown_parallel_pool()

    @pytest.mark.parametrize("with_workdir", [False, True])
    def test_sync_false_returns_immediately(self, tmp_path, monkeypatch, with_workdir):
        """The ticker returns while its dispatched worker is still blocked."""
        import cron.scheduler as sched

        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._running_job_ids.clear()

        job = {
            "id": "slow-job",
            "name": "slow",
            "prompt": "test",
            "schedule": "every 5m",
            "enabled": True,
            "next_run_at": "2020-01-01T00:00:00",
            "deliver": "local",
        }

        if with_workdir:
            job["workdir"] = str(tmp_path)

        worker_started = threading.Event()
        release = threading.Event()
        worker_finished = threading.Event()

        def slow_run(j, *, defer_agent_teardown=None, **_kw):
            worker_started.set()
            try:
                assert release.wait(timeout=10), "test never released the worker"
            finally:
                worker_finished.set()
            return True, "out", "resp", None

        monkeypatch.setattr(sched, "get_due_jobs", lambda: [job])
        monkeypatch.setattr(sched, "claim_job_for_fire", lambda *_a, **_kw: True)
        monkeypatch.setattr(sched, "run_job", slow_run)
        monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: "/tmp/out")
        monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)

        try:
            assert sched.tick(verbose=False, sync=False) == 1
            assert worker_started.wait(timeout=10)
            assert not worker_finished.is_set()
        finally:
            release.set()
            sched._shutdown_parallel_pool()


class TestTickBatchAdvance:
    """The tick's pre-dispatch advance must go through advance_next_runs
    exactly once with the whole due set — a revert to the per-job loop
    (or back to advance_next_run) must fail this test, not slip past the
    helper-level I/O pin."""

    def test_tick_calls_advance_next_runs_once_with_all_due_ids(self, tmp_path, monkeypatch):
        import cron.scheduler as sched

        sched._parallel_pools.clear()
        sched._parallel_pool_max_workers.clear()
        sched._running_job_ids.clear()

        jobs = [
            {"id": f"job-{i}", "name": f"Job {i}", "prompt": "test",
             "schedule": "every 5m", "enabled": True,
             "next_run_at": "2020-01-01T00:00:00", "deliver": "local"}
            for i in range(4)
        ]

        advance_calls = []
        monkeypatch.setattr(sched, "get_due_jobs", lambda: jobs)
        monkeypatch.setattr(
            sched, "advance_next_runs",
            lambda ids: advance_calls.append(list(ids)) or len(list(ids)))
        monkeypatch.setattr(sched, "run_job", lambda j, **_kw: (True, "out", "resp", None))
        monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: "/tmp/out")
        monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
        monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)

        n = sched.tick(verbose=False)

        assert n == 4
        assert advance_calls == [["job-0", "job-1", "job-2", "job-3"]], (
            f"tick must batch-advance the due set in ONE call; got {advance_calls}")

        sched._shutdown_parallel_pool()