4ce832eeb9
A cron job delivering to Bot Chat booked "timed out after 600s" for turns that finished in seconds: cron/scheduler_delivery.py::_deliver_to_bot_chat waited for the `hermes chat -Q` child to EXIT, but when the bot's turn had messaged a teammate (message_agent -> notify_on_complete runner) the child then runs the one-shot exit linger, bounded by terminal.oneshot_completion_wait_seconds (default 600) — the same default as cron.bot_chat_delivery_timeout_seconds. The cap counts from the claim, the linger only starts when the turn ends, so the cap expired first on every such delivery, booked a completed turn as a timeout, held the job's fire fence for the full cap, and killed the child mid-linger — tearing down the reply the linger exists to protect (#90879). Ordering the two bounds cannot fix it (the turn has no duration bound; the linger has its own contract), so the cap stops competing with the linger: - hermes_cli/quiet_single_query.py: the -Q child accepts a per-process report path (HERMES_QUIET_TURN_REPORT_FILE), popped before the turn like HERMES_TURN_AUTHOR so nothing the turn spawns inherits it; cli.py writes {pid, exit_code, error} there the moment the turn ends, BEFORE the linger. - cron: _run_bot_chat_turn polls the child and that report under the cap. Report present -> the delivery is booked from it (real exit code and stream tails when the child exits within a short grace) and the still- lingering child is left running, drained and reaped by a daemon thread. No report by the cap -> the turn never ended: killed and booked as a timeout, exactly as before. The linger itself is untouched. Live repro (real _deliver_to_bot_chat, real `hermes chat -Q` against a loopback provider whose turn spawns `sleep 90` with notify_on_complete, cap 45s): base books the timeout at 45.7s for a turn that ended at +5s and kills the child; fixed head books success at 11.2s, the child lingers, the teammate follow-up turn runs at +97s and the child exits on its own. Supersedes #113649's cap = delivery + linger (the thread shows a headroom only moves the race and lengthens the fence hold); analysis credit to the reporter and the thread's independent verification. Fixes #113608 Co-authored-by: KoNit-K <konit.block@protonmail.com>
71 lines
3.8 KiB
Python
71 lines
3.8 KiB
Python
"""Cron admission must not become a second writer or a completed-delivery claim."""
|
|
from pathlib import Path
|
|
from unittest.mock import Mock
|
|
|
|
from cron import scheduler_delivery as delivery
|
|
from tools import bot_live_delivery as mailbox
|
|
|
|
|
|
def test_live_delivery_retry_keeps_receipt_across_owner_loss(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
|
source = tmp_path / "custom-home"
|
|
monkeypatch.setenv("HERMES_HOME", str(source))
|
|
subprocess_run = Mock(side_effect=AssertionError("live owner must not spawn CLI"))
|
|
monkeypatch.setattr(delivery, "_run_bot_chat_turn", subprocess_run)
|
|
from hermes_cli.profiles import get_profile_dir
|
|
|
|
for profile, home in [("", source), ("research", get_profile_dir("research"))]:
|
|
owner = dict(profile_home=str(home.resolve()), session_id="bot", lease_id="lease",
|
|
live_session_id="live")
|
|
discovery = Mock(return_value=owner)
|
|
monkeypatch.setattr(mailbox, "find_canonical_live_owner", discovery)
|
|
job = dict(id="digest", name="Digest", execution_id="first-run")
|
|
pending = delivery._deliver_to_bot_chat(job, "payload", profile)
|
|
assert pending and "queued" in pending
|
|
records = list((home / "runtime/bot_live_delivery").glob("*.json"))
|
|
assert len(records) == 1
|
|
key = records[0].stem
|
|
record = mailbox.read_delivery_result(home, key)
|
|
assert record and record["message"].endswith("payload")
|
|
discovery.side_effect = AssertionError("receipt must precede discovery")
|
|
assert delivery._deliver_to_bot_chat(dict(job), "payload", profile) == pending
|
|
mailbox.claim_pending_delivery(home, owner)
|
|
mailbox.complete_delivery(home, key, status="ambiguous", error="owner died")
|
|
outcome = delivery._deliver_to_bot_chat(dict(job), "payload", profile)
|
|
assert outcome and "ambiguous" in outcome
|
|
discovery.side_effect = None
|
|
next_job = dict(job, execution_id="next-run")
|
|
outcome = delivery._deliver_to_bot_chat(next_job, "payload", profile)
|
|
assert outcome and "queued" in outcome
|
|
assert len(list((home / "runtime/bot_live_delivery").glob("*.json"))) == 2
|
|
subprocess_run.assert_not_called()
|
|
|
|
|
|
def test_result_records_pending_until_terminal_receipt(tmp_path, monkeypatch):
|
|
from cron import jobs
|
|
from gateway import config
|
|
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
monkeypatch.delenv("_HERMES_CRON_EXTERNAL_WORKER", raising=False)
|
|
owner = dict(profile_home=str(tmp_path.resolve()), session_id="bot", lease_id="lease",
|
|
live_session_id="live")
|
|
monkeypatch.setattr(mailbox, "find_canonical_live_owner", lambda home: owner)
|
|
monkeypatch.setattr(delivery._sched, "load_config", lambda: {})
|
|
monkeypatch.setattr(config, "load_gateway_config", lambda: None)
|
|
monkeypatch.setattr(delivery, "_run_bot_chat_turn", Mock(side_effect=AssertionError("CLI")))
|
|
updates = []
|
|
monkeypatch.setattr(jobs, "update_job", lambda key, values: updates.append(values))
|
|
job = dict(id="digest", execution_id="run", deliver="bot-chat")
|
|
error = delivery._deliver_result(job, "payload")
|
|
assert error is None
|
|
queued = updates[-1]["last_delivery_queued"]
|
|
assert queued and next(iter(queued.values()))["status"] == "queued"
|
|
assert delivery._sched._classify_delivery_outcome(
|
|
delivery_error=error, delivery_queued=queued, should_deliver=True, unresolved_origin=False,
|
|
normalized_deliver="bot-chat", incident_acked=False, success=True) == "queued"
|
|
record = mailbox.claim_pending_delivery(tmp_path, owner)
|
|
assert record is not None
|
|
mailbox.complete_delivery(tmp_path, record["delivery_id"], status="settled", reply="done")
|
|
assert delivery._deliver_result(job, "payload") is None
|
|
assert updates[-1]["last_delivery_queued"] is None
|