"""Tests for async (background) delegation — tools/async_delegation.py. Covers the dispatch handle, non-blocking behavior, completion-event delivery onto the shared process_registry.completion_queue, the rich re-injection block formatting, capacity rejection, and crash handling. """ import json import os import sqlite3 import subprocess import sys import threading import time import pytest from tools import async_delegation as ad from tools.process_registry import process_registry from tools.process_registry_notifications import format_process_notification @pytest.fixture(autouse=True) def _clean_state(): ad._reset_for_tests() while not process_registry.completion_queue.empty(): process_registry.completion_queue.get_nowait() yield # Give just-released workers a beat to finalize BEFORE draining, so their # completion events land now instead of leaking into the next test's # queue (worker threads push events asynchronously; a drain that races an # in-flight _finalize misses it). deadline = time.monotonic() + 2.0 while ad.active_count() and time.monotonic() < deadline: time.sleep(0.02) ad._reset_for_tests() while not process_registry.completion_queue.empty(): process_registry.completion_queue.get_nowait() def _drain_one(timeout=5.0): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if not process_registry.completion_queue.empty(): return process_registry.completion_queue.get_nowait() time.sleep(0.02) return None def _drain_for(delegation_id, timeout=5.0): """Drain until the event for *delegation_id* appears (discarding others). Completion events are pushed asynchronously by worker threads, so a straggler from a PREVIOUS test can land after that test's teardown drain and leak into the current test's queue. Matching on delegation_id makes the assertion immune to that cross-test leak. """ deadline = time.monotonic() + timeout while time.monotonic() < deadline: if not process_registry.completion_queue.empty(): evt = process_registry.completion_queue.get_nowait() if evt.get("delegation_id") == delegation_id: return evt continue time.sleep(0.02) return None def test_schema_init_preserves_shared_state_db_journal_mode(tmp_path): """The delegation ledger is a guest in state.db, not its mode owner.""" conn = sqlite3.connect(tmp_path / "state.db") try: assert conn.execute("PRAGMA journal_mode=DELETE").fetchone()[0] == "delete" ad._initialize_schema(conn) assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "delete" assert conn.execute( "SELECT name FROM sqlite_master WHERE type='table' " "AND name='async_delegations'" ).fetchone() == ("async_delegations",) finally: conn.close() def test_schema_init_preserves_shared_state_db_wal_mode(tmp_path): """Schema initialization must not replace an existing WAL mode.""" conn = sqlite3.connect(tmp_path / "state.db") try: assert conn.execute("PRAGMA journal_mode=WAL").fetchone()[0] == "wal" ad._initialize_schema(conn) assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal" assert conn.execute( "SELECT name FROM sqlite_master WHERE type='table' " "AND name='async_delegations'" ).fetchone() == ("async_delegations",) finally: conn.close() @pytest.mark.platforms("macos") def test_connect_preserves_wal_and_applies_macos_durability_barriers( tmp_path, monkeypatch ): """Each ledger connection must carry the macOS write barriers.""" monkeypatch.setenv("HERMES_HOME", str(tmp_path)) seed = sqlite3.connect(tmp_path / "state.db") try: assert seed.execute("PRAGMA journal_mode=WAL").fetchone()[0] == "wal" finally: seed.close() conn = ad._connect() try: assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "wal" assert conn.execute("PRAGMA synchronous").fetchone()[0] == 2 assert conn.execute("PRAGMA checkpoint_fullfsync").fetchone()[0] == 1 finally: conn.close() def test_async_executor_workers_are_daemon_threads(): gate = threading.Event() def runner(): gate.wait(timeout=60) return {"status": "completed", "summary": "done"} res = ad.dispatch_async_delegation( goal="daemon check", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, max_async_children=1, ) assert res["status"] == "dispatched" deadline = time.monotonic() + 2 worker = None while time.monotonic() < deadline: worker = next( (t for t in threading.enumerate() if t.name.startswith("async-delegate")), None, ) if worker is not None: break time.sleep(0.02) assert worker is not None assert worker.daemon is True gate.set() assert _drain_one() is not None def test_completion_event_lands_on_shared_queue_with_session_key(): def runner(): return {"status": "completed", "summary": "the result", "api_calls": 3, "duration_seconds": 2.0, "model": "test-model"} res = ad.dispatch_async_delegation( goal="compute X", context="some context", toolsets=["web", "file"], role="leaf", model="test-model", session_key="agent:main:cli:dm:local", parent_session_id="20260703_parent_sid", runner=runner, max_async_children=3, ) assert res["status"] == "dispatched" evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" assert evt["summary"] == "the result" assert evt["session_key"] == "agent:main:cli:dm:local" assert evt["parent_session_id"] == "20260703_parent_sid" assert evt["delegation_id"] == res["delegation_id"] def test_rich_reinjection_block_is_self_contained(): def runner(): return {"status": "completed", "summary": "The answer is 42.", "api_calls": 7, "duration_seconds": 3.5, "model": "test-model"} ad.dispatch_async_delegation( goal="Compute the meaning of life", context="User is a philosopher. Respond tersely.", toolsets=["web"], role="leaf", model="test-model", session_key="", runner=runner, max_async_children=3, ) evt = _drain_one() assert evt is not None text = format_process_notification(evt) assert text is not None for needle in [ "Compute the meaning of life", "User is a philosopher", "The answer is 42.", ]: assert needle in text, f"missing {needle!r}" def test_dispatch_rejected_at_capacity(): ev = threading.Event() def blocker(): ev.wait(timeout=60) return {"status": "completed", "summary": "x"} for i in range(2): r = ad.dispatch_async_delegation( goal=f"task{i}", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=2, ) assert r["status"] == "dispatched" r3 = ad.dispatch_async_delegation( goal="task3", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=2, ) assert r3["status"] == "rejected" assert "capacity reached" in r3["error"] ev.set() def test_interrupt_all_signals_running_children(): ev = threading.Event() interrupted = {"count": 0} # No short internal timeout: the blocker holds until interrupt_fn fires. # The old ev.wait(timeout=5) made this test a change-detector for CI # worker load — on a CPU-starved runner the 5s expired before # interrupt_all() ran, the record finalized, and interrupt_all() found # nothing running (n == 0). The pytest-level timeout is the real # runaway guard. def blocker(): ev.wait(timeout=60) return {"status": "interrupted", "summary": None, "error": "cancelled"} def interrupt_fn(): interrupted["count"] += 1 ev.set() r = ad.dispatch_async_delegation( goal="long task", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, interrupt_fn=interrupt_fn, max_async_children=3, ) n = ad.interrupt_all(reason="test") assert n == 1 assert interrupted["count"] == 1 # child still emits a completion event after interrupt. Match on THIS # delegation's id — straggler 'completed' events from a previous test's # workers can finalize after that test's teardown drain and leak into # this queue (observed on loaded CI workers). evt = _drain_for(r["delegation_id"]) assert evt is not None assert evt["status"] == "interrupted" def _fast_stale_monitor(monkeypatch, *, idle=0.15, in_tool=0.3, grace=0.15): """Shrink the stale-monitor cadence so tests run in milliseconds.""" monkeypatch.setattr(ad, "_STALE_CHECK_INTERVAL", 0.03) monkeypatch.setattr(ad, "_STALE_IDLE_SECONDS", idle) monkeypatch.setattr(ad, "_STALE_IN_TOOL_SECONDS", in_tool) monkeypatch.setattr(ad, "_STALL_GRACE_SECONDS", grace) def test_stalled_runner_is_interrupted_then_finalized(monkeypatch): _fast_stale_monitor(monkeypatch) gate = threading.Event() interrupted = {"count": 0} def stuck_runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "too late"} def interrupt_fn(): interrupted["count"] += 1 res = ad.dispatch_async_delegation( goal="stuck child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=stuck_runner, interrupt_fn=interrupt_fn, max_async_children=1, # Frozen progress token: the child never advances an API call. progress_fn=lambda: ((0, None), False), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) try: assert evt is not None assert evt["type"] == "async_delegation" assert evt["status"] == "stalled" assert evt["delegation_id"] == res["delegation_id"] assert evt["api_calls"] == 0 assert "stopped responding" in evt["error"] # status carries "stalled"; the text is for the user # Interrupt was requested BEFORE force-finalization (grace window). assert interrupted["count"] >= 1 assert ad.active_count() == 0 finally: gate.set() # If the ignored runner eventually returns, it must not enqueue a second # completion for a delegation the monitor already finalized. assert _drain_one(timeout=0.5) is None def test_progressing_runner_is_never_stalled(monkeypatch): """A child that keeps advancing is left alone no matter how long it runs.""" _fast_stale_monitor(monkeypatch) gate = threading.Event() ticks = {"n": 0} def slow_but_alive_runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "done", "api_calls": 7} def progress_fn(): # Token advances on every sample — simulates a child making steady # API-call progress. ticks["n"] += 1 return (ticks["n"], None), False res = ad.dispatch_async_delegation( goal="slow child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=slow_but_alive_runner, max_async_children=1, progress_fn=progress_fn, ) assert res["status"] == "dispatched" # Run well past the (shrunk) idle threshold — several monitor sweeps. time.sleep(0.6) assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" assert evt["summary"] == "done" def test_stalling_runner_that_honors_interrupt_keeps_its_result(monkeypatch): """Interrupt-responsive children finalize through the NORMAL path. The monitor's interrupt gives a wedged-looking child a grace window; if the runner returns during it, the real result (partial work, api_calls) is delivered instead of a synthetic stalled event. """ _fast_stale_monitor(monkeypatch, grace=5.0) interrupted = threading.Event() def runner(): # "Wedged" until interrupted, then unwinds and reports partial work. interrupted.wait(timeout=10) return { "status": "interrupted", "summary": "partial work saved", "api_calls": 3, } res = ad.dispatch_async_delegation( goal="responsive child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, interrupt_fn=interrupted.set, max_async_children=1, progress_fn=lambda: ((3, None), False), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "interrupted" assert evt["summary"] == "partial work saved" assert evt["api_calls"] == 3 assert ad.active_count() == 0 def test_streaming_child_counts_as_alive(monkeypatch): """A child mid-stream (api_call_count frozen, last_activity_ts ticking) must never be stalled — streamed chunks tick _touch_activity, and the progress token includes that timestamp (same liveness signal as the compaction inactivity budget, PR #71508).""" _fast_stale_monitor(monkeypatch) gate = threading.Event() now = {"ts": 1000.0} def progress_fn(): # api_call_count and current_tool frozen (long streaming response in # flight), but the activity timestamp advances with every chunk. now["ts"] += 1.0 return ((1, None, now["ts"]),), False res = ad.dispatch_async_delegation( goal="streaming child", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: (gate.wait(timeout=10), {"status": "completed", "summary": "streamed"})[1], progress_fn=progress_fn, ) assert res["status"] == "dispatched" time.sleep(0.6) # several sweeps past the shrunk idle threshold assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" def test_stalled_event_carries_structured_stall_metadata(monkeypatch): """The terminal stalled event must expose machine-readable stall context (#51690) — quiet duration, tripped threshold, phase, grace — mirroring the sync path's timeout_seconds/timed_out_after_seconds/timeout_phase.""" _fast_stale_monitor(monkeypatch) gate = threading.Event() res = ad.dispatch_async_delegation( goal="stall metadata", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: {} if gate.wait(timeout=10) else {}, progress_fn=lambda: ((0, "terminal"), True), ) assert res["status"] == "dispatched" evt = _drain_for(res["delegation_id"], timeout=5.0) try: assert evt is not None assert evt["status"] == "stalled" assert evt["stalled_after_quiet_seconds"] >= 0.3 # in-tool threshold assert evt["stall_threshold_seconds"] == ad._STALE_IN_TOOL_SECONDS assert evt["stall_phase"] == "in_tool" assert evt["stall_grace_seconds"] == ad._STALL_GRACE_SECONDS finally: gate.set() def test_list_async_delegations_exposes_live_activity(monkeypatch): """list_async_delegations must expose per-child live activity sampled from progress_fn plus seconds_since_progress, for /agents UIs (#51690).""" monkeypatch.setattr(ad, "_STALE_CHECK_INTERVAL", 0.03) gate = threading.Event() base_ts = time.time() - 12.0 res = ad.dispatch_async_delegation( goal="live listing", context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1, runner=lambda: {} if gate.wait(timeout=10) else {}, progress_fn=lambda: (((3, "web_search", base_ts),), True), ) try: time.sleep(0.1) # let the monitor stamp _progress_ts at least once item = next( d for d in ad.list_async_delegations() if d["delegation_id"] == res["delegation_id"] ) assert item["status"] == "running" assert item["in_tool"] is True assert "seconds_since_progress" in item (child,) = item["children_activity"] assert child["api_calls"] == 3 assert child["current_tool"] == "web_search" assert 10.0 <= child["seconds_since_activity"] <= 20.0 # Callables and private bookkeeping must never leak. assert "progress_fn" not in item assert "interrupt_fn" not in item assert not any(k.startswith("_") for k in item) finally: gate.set() def test_in_tool_stall_uses_higher_threshold(monkeypatch): """A frozen child inside a tool gets the in-tool ceiling, not the idle one.""" _fast_stale_monitor(monkeypatch, idle=0.1, in_tool=10.0, grace=0.1) gate = threading.Event() def runner(): gate.wait(timeout=10) return {"status": "completed", "summary": "long tool finished"} res = ad.dispatch_async_delegation( goal="long tool child", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=runner, max_async_children=1, # Frozen token but in_tool=True — a legitimately slow terminal # command / web fetch. Must NOT be stalled at the idle threshold. progress_fn=lambda: ((1, "terminal"), True), ) assert res["status"] == "dispatched" time.sleep(0.5) # far past idle threshold, well under in-tool threshold assert ad.active_count() == 1 assert process_registry.completion_queue.empty() gate.set() evt = _drain_for(res["delegation_id"], timeout=5.0) assert evt is not None assert evt["status"] == "completed" def test_real_process_restart_restores_owned_completion_once(tmp_path): """Real-import E2E: a fresh interpreter restores a prior process's result.""" repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) env = {**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": repo} producer = r''' import time from tools import async_delegation as ad r = ad.dispatch_async_delegation( goal="restart", context=None, toolsets=None, role="leaf", model="m", session_key="owner-session", parent_session_id="durable-parent", runner=lambda: {"status": "completed", "summary": "after restart"}, ) deadline = time.time() + 5 while ad.active_count() and time.time() < deadline: time.sleep(.01) print(r["delegation_id"]) ''' first = subprocess.run( [sys.executable, "-c", producer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) delegation_id = first.stdout.strip().splitlines()[-1] # The ledger replays on the first consumer, not at import (#123265). consumer = r''' import json from tools.process_registry import process_registry process_registry.restore_completions() evt = process_registry.completion_queue.get_nowait() print(json.dumps(evt, sort_keys=True)) ''' second = subprocess.run( [sys.executable, "-c", consumer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) evt = json.loads(second.stdout.strip().splitlines()[-1]) assert evt["delegation_id"] == delegation_id assert evt["session_key"] == "owner-session" assert evt["parent_session_id"] == "durable-parent" assert evt["summary"] == "after restart" acker = f''' from tools import async_delegation as ad assert ad.mark_completion_delivered({delegation_id!r}) ''' subprocess.run( [sys.executable, "-c", acker], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) probe = subprocess.run( [sys.executable, "-c", "from tools.process_registry import process_registry; process_registry.restore_completions(); print(process_registry.completion_queue.qsize())"], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True, ) assert probe.stdout.strip().splitlines()[-1] == "0" # --------------------------------------------------------------------------- # Integration: delegate_task(background=True) routing # --------------------------------------------------------------------------- def test_delegate_task_background_routes_async_and_does_not_block(monkeypatch): """delegate_task(background=True) returns a handle without running the child synchronously, and the child completes on the background thread. A single task is dispatched as a one-item background batch unit.""" from unittest.mock import MagicMock import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" fake_child._subagent_id = "s1" gate = threading.Event() def slow_child(task_index, goal, child=None, parent_agent=None, **kw): gate.wait(timeout=60) # a sync impl would hang delegate_task here return { "task_index": 0, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", } creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } # monkeypatch (not `with`) so patches outlive delegate_task's return and # remain active while the background worker runs. monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_run_single_child", slow_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) out = dt.delegate_task( goal="the real task", context="ctx", background=True, parent_agent=parent, ) import json parsed = json.loads(out) assert parsed["status"] == "dispatched" assert parsed["mode"] == "background" assert parsed["delegation_id"].startswith("deleg_") # Non-blocking invariant: delegate_task returned while the child is STILL # blocked on the closed gate, so no completion event exists yet. assert process_registry.completion_queue.empty() assert ad.active_count() == 1 # one background batch unit, not finished gate.set() evt = _drain_one() assert evt is not None assert evt["type"] == "async_delegation" # Single task rides the batch path → carries a 1-item results list. assert evt.get("is_batch") is True assert len(evt["results"]) == 1 assert evt["results"][0]["summary"] == "done: the real task" text = format_process_notification(evt) assert text is not None assert "the real task" in text def test_delegate_task_background_uses_live_tui_agent_session_id(monkeypatch): """TUI async delegation must route to the live/compressed agent id. Regression: delegate_task captured the stale approval/session context key after compression rotated parent_agent.session_id. The resulting completion was orphaned and could be consumed by an unrelated desktop session poller. """ import json from unittest.mock import MagicMock import tools.delegate_tool as dt from gateway.session_context import clear_session_vars, set_session_vars from tools.approval_context import reset_current_session_key, set_current_session_key parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "post-compress-tip" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None fake_child = MagicMock() fake_child._delegate_role = "leaf" creds = { "model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None, } monkeypatch.setattr(dt, "_build_child_agent", lambda **kw: fake_child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) monkeypatch.setattr( dt, "_run_single_child", lambda *a, **k: { "task_index": 0, "status": "completed", "summary": "done", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed", }, ) approval_token = set_current_session_key("pre-compress-parent") session_tokens = set_session_vars( source="tui", session_key="pre-compress-parent", ui_session_id="origin-tab", ) try: out = dt.delegate_task(goal="bg task", background=True, parent_agent=parent) assert json.loads(out)["status"] == "dispatched" evt = _drain_one() finally: reset_current_session_key(approval_token) clear_session_vars(session_tokens) assert evt is not None assert evt["type"] == "async_delegation" assert evt["session_key"] == "post-compress-tip" assert evt["origin_ui_session_id"] == "origin-tab" def test_concurrent_dispatch_respects_capacity(): """Two threads racing dispatch with cap=1 must yield exactly one accept (capacity check and record insert are atomic under the records lock).""" gate = threading.Event() def blocker(): gate.wait(timeout=60) return {"status": "completed", "summary": "x"} results = [] barrier = threading.Barrier(2) def racer(): barrier.wait(timeout=5) results.append( ad.dispatch_async_delegation( goal="race", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=1, ) ) threads = [threading.Thread(target=racer) for _ in range(2)] for t in threads: t.start() for t in threads: t.join(timeout=10) statuses = sorted(r["status"] for r in results) assert statuses == ["dispatched", "rejected"] gate.set() # --------------------------------------------------------------------------- # Gateway routing: session_key -> platform/chat_id, rich formatting, injection # --------------------------------------------------------------------------- def _make_async_evt(**over): evt = { "type": "async_delegation", "delegation_id": "deleg_x1", "session_key": "agent:main:telegram:dm:12345:678", "goal": "Investigate flaky test", "context": "repo /tmp/p", "toolsets": ["terminal"], "role": "leaf", "model": "m", "status": "completed", "summary": "Found the bug in test_foo", "api_calls": 4, "duration_seconds": 12.0, "dispatched_at": 1000.0, "completed_at": 1012.0, } evt.update(over) return evt def test_gateway_formatter_renders_async_block(): from gateway.run import _format_gateway_process_notification txt = _format_gateway_process_notification(_make_async_evt()) assert txt is not None assert "ASYNC DELEGATION COMPLETE" in txt assert "Found the bug in test_foo" in txt assert "Investigate flaky test" in txt def test_gateway_cli_origin_event_left_unrouted(): """An empty session_key (CLI origin) is left without routing fields.""" from gateway.run import GatewayRunner runner = object.__new__(GatewayRunner) evt = _make_async_evt(session_key="") runner._enrich_async_delegation_routing(evt) assert "platform" not in evt def test_single_task_truncation_banner_when_max_iterations(): """A single async subagent that hit its iteration cap (exit_reason= max_iterations) must surface a TRUNCATED marker in the formatted result, even though status stays 'completed' (a summary exists).""" evt = _make_async_evt( status="completed", summary="Did part of the work then ran out of budget.", exit_reason="max_iterations", ) text = format_process_notification(evt) assert text is not None assert "TRUNCATED" in text assert "max_iterations" in text # The summary is still shown, just flagged. assert "Did part of the work" in text def test_single_task_no_banner_when_clean(): """A cleanly-finished subagent must NOT get a truncation banner.""" evt = _make_async_evt(status="completed", summary="All done.", exit_reason="completed") text = format_process_notification(evt) assert text is not None assert "TRUNCATED" not in text def test_batch_truncation_banner_marks_only_truncated_task(): """In a batch, only the task that hit max_iterations gets the TRUNCATED marker; a clean sibling keeps the normal check icon.""" evt = _make_async_evt( is_batch=True, goals=["clean task", "truncated task"], results=[ { "task_index": 0, "status": "completed", "summary": "finished cleanly", "api_calls": 5, "exit_reason": "completed", "truncated": False, }, { "task_index": 1, "status": "completed", "summary": "cut off mid-work", "api_calls": 250, "exit_reason": "max_iterations", "truncated": True, }, ], ) text = format_process_notification(evt) assert text is not None assert "TRUNCATED" in text # The clean task's summary and the truncated one's both render... assert "finished cleanly" in text assert "cut off mid-work" in text # ...but the banner is tied to the truncated task, not the clean one. trunc_pos = text.index("cut off mid-work") clean_pos = text.index("finished cleanly") banner_pos = text.index("TRUNCATED") # The header banner for task 2 appears after task 1's summary. assert banner_pos > clean_pos def _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro-4", provider="openrouter"): """Pin the delegation config the notice renderer reads (adapts the #97667 tests to the shipped implementation, which reads the configured model from config rather than the event's model field).""" import tools.process_registry_notifications as _prn monkeypatch.setattr( _prn, "_delegation_config", lambda: {"model": model, "provider": provider} ) def test_batch_model_rejection_notice_prepended(monkeypatch): """A rejected delegation model must surface ONE config-level notice above the per-task blocks instead of staying buried in each summary (#97654).""" rejection = "HTTP 400: upstage/solar-pro-4 is not a valid model ID" _patch_delegation_cfg(monkeypatch) evt = _make_async_evt( is_batch=True, model="upstage/solar-pro-4", goals=["task a", "task b"], results=[ { "task_index": 0, "status": "completed", "summary": rejection, "api_calls": 1, "duration_seconds": 0.74, "exit_reason": "max_iterations", "truncated": True, }, { "task_index": 1, "status": "completed", "summary": rejection, "api_calls": 1, "duration_seconds": 0.71, "exit_reason": "max_iterations", "truncated": True, }, ], ) text = format_process_notification(evt) assert text is not None assert "SUBAGENT MODEL REJECTED" in text assert "upstage/solar-pro-4" in text assert "delegation.model" in text # The notice precedes the per-task blocks, not just trails them. assert text.index("SUBAGENT MODEL REJECTED") < text.index("TASK 1/2") def test_batch_model_rejection_notice_absent_when_clean(monkeypatch): """Ordinary summaries must not grow a model-rejection notice.""" _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro4") evt = _make_async_evt( is_batch=True, model="upstage/solar-pro4", goals=["task a"], results=[ { "task_index": 0, "status": "completed", "summary": "did the work", "api_calls": 3, "exit_reason": "completed", "truncated": False, }, ], ) text = format_process_notification(evt) assert text is not None assert "SUBAGENT MODEL REJECTED" not in text def test_batch_model_rejection_notice_requires_configured_model_in_text(monkeypatch): """A model_not_found pattern naming a DIFFERENT model than the configured delegation model is task-level noise, not a config-level rejection.""" _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro4") evt = _make_async_evt( is_batch=True, model="upstage/solar-pro4", goals=["task a"], results=[ { "task_index": 0, "status": "completed", "summary": "HTTP 400: other/model-x is not a valid model ID", "api_calls": 1, "exit_reason": "max_iterations", "truncated": True, }, ], ) text = format_process_notification(evt) assert text is not None assert "SUBAGENT MODEL REJECTED" not in text # --------------------------------------------------------------------------- # Per-group completion units: ungrouped tasks return alone, a `group` returns # together, and the units of one call share ONE capacity slot. # --------------------------------------------------------------------------- def _grouped_fanout(monkeypatch, tasks, gates): """delegate_task(tasks) in the background with gated fake children; returns the parsed handle.""" from unittest.mock import MagicMock import tools.delegate_tool as dt parent = MagicMock() parent._delegate_depth = 0 parent.session_id = "sess" parent._interrupt_requested = False parent._active_children = [] parent._active_children_lock = None def child(task_index, goal, child=None, parent_agent=None, **kw): gates[task_index].wait(timeout=60) return {"task_index": task_index, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed"} def build(**kw): c = MagicMock() c._delegate_role = "leaf" c._subagent_id = f"s{kw['task_index']}" return c creds = {"model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None} monkeypatch.setattr(dt, "_build_child_agent", build) monkeypatch.setattr(dt, "_run_single_child", child) monkeypatch.setattr(dt, "_resolve_delegation_credentials", lambda *a, **k: creds) return json.loads(dt.delegate_task(tasks=tasks, background=True, parent_agent=parent)) def test_ungrouped_task_completes_alone_and_group_completes_together(monkeypatch): """With delegation.independent_completions on, a finished ungrouped task must not wait for its siblings; tasks sharing a `group` must.""" import tools.delegate_tool as dt monkeypatch.setattr(dt, "_load_config", lambda: {"independent_completions": True}) gates = [threading.Event() for _ in range(4)] tasks = [ {"goal": "review PR 1 thoroughly and report"}, {"goal": "compare approach A in detail", "group": "cmp"}, {"goal": "compare approach B in detail", "group": "cmp"}, {"goal": "review PR 2 thoroughly and report"}, ] handle = _grouped_fanout(monkeypatch, tasks, gates) assert handle["status"] == "dispatched" by_group = {tuple(u["task_indexes"]): u["group"] for u in handle["units"]} assert by_group == {(0,): None, (1, 2): "cmp", (3,): None} gates[3].set() evt = _drain_one() assert [r["task_index"] for r in evt["results"]] == [3] # PR 2 landed while everything else still runs assert "review PR 2" in format_process_notification(evt) gates[1].set() assert _drain_one(timeout=0.5) is None # half a group is not a completion gates[2].set() evt = _drain_one() assert evt["group"] == "cmp" and [r["task_index"] for r in evt["results"]] == [1, 2] gates[0].set() assert [r["task_index"] for r in _drain_one()["results"]] == [0] def test_units_of_one_call_share_a_single_capacity_slot(): """Splitting a call into per-task completions must not consume more pool capacity than the call did.""" gate = threading.Event() def blocker(): gate.wait(timeout=60) return {"results": [], "total_duration_seconds": 0} common = dict(goals=["a", "b"], context=None, toolsets=None, role="leaf", model="m", session_key="", runner=blocker, max_async_children=1) first = ad.dispatch_async_delegation_batch(delegation_id="deleg_call-1", task_indexes=[0], **common) second = ad.dispatch_async_delegation_batch(delegation_id="deleg_call-2", task_indexes=[1], slot_key="deleg_call-1", **common) other = ad.dispatch_async_delegation_batch(delegation_id="deleg_other", **common) assert (first["status"], second["status"], other["status"]) == ("dispatched", "dispatched", "rejected") assert ad.active_task_count() == 2 gate.set() def test_multi_task_call_is_one_completion_unless_independent_completions(monkeypatch): """Default: a background fan-out returns as ONE message when every task is done, so an orchestrator is not woken N times per call; `group` is inert until delegation.independent_completions is on.""" import tools.delegate_tool as dt monkeypatch.setattr(dt, "_load_config", lambda: {}) gates = [threading.Event() for _ in range(3)] tasks = [{"goal": "review PR 1 thoroughly and report"}, {"goal": "review PR 2 thoroughly and report", "group": "g"}, {"goal": "review PR 3 thoroughly and report", "group": "g"}] handle = _grouped_fanout(monkeypatch, tasks, gates) assert handle["status"] == "dispatched" and "units" not in handle gates[0].set() gates[1].set() assert _drain_one(timeout=0.5) is None # two of three done: no message yet gates[2].set() evt = _drain_one() assert sorted(r["task_index"] for r in evt["results"]) == [0, 1, 2] def test_units_beyond_slot_count_still_start_and_are_not_stalled_while_queued(monkeypatch): """Units of one call share a slot, so live units can exceed the slot cap; every unit must still get a worker, and a unit must not be judged stalled for time it spent waiting to start.""" _fast_stale_monitor(monkeypatch, idle=0.3, grace=0.2) started, release = [], threading.Event() frozen = lambda: (((0, None, None),), False) # noqa: E731 - child never progresses => token never changes def blocker(uid): def run(): started.append(uid) release.wait(timeout=10) return {"results": [{"task_index": 0, "status": "completed"}], "total_duration_seconds": 0} return run common = dict(goals=["x"], context=None, toolsets=None, role="leaf", model="m", session_key="", max_async_children=1) ad.dispatch_async_delegation_batch(delegation_id="deleg_c-1", runner=blocker("c-1"), progress_fn=frozen, **common) ad.dispatch_async_delegation_batch(delegation_id="deleg_c-2", runner=blocker("c-2"), slot_key="deleg_c-1", **common) deadline = time.monotonic() + 2.0 while len(started) < 2 and time.monotonic() < deadline: time.sleep(0.02) assert sorted(started) == ["c-1", "c-2"] # second unit started despite a 1-slot pool # A unit whose runner has NOT started yet must not accrue stall time: pin the pool so it stays queued. monkeypatch.setattr(ad, "_get_executor", lambda n: ad._executor) ad.dispatch_async_delegation_batch(delegation_id="deleg_q", runner=blocker("q"), progress_fn=frozen, **{**common, "max_async_children": 3}) time.sleep(0.7) # > idle + grace with the unit still queued with ad._records_lock: assert ad._records["deleg_q"]["status"] == "running" assert "q" not in started release.set() def test_child_finished_before_crash_is_recovered_with_its_result(tmp_path): """Real-import E2E: a 2-task group unit whose owner dies mid-run replays the finished child's real result and marks only the unfinished sibling unknown — a crash costs the stragglers, never the finished work.""" repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) env = {**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": repo} producer = r''' import os, sys, time from unittest.mock import MagicMock import tools.delegate_tool as dt parent = MagicMock(); parent._delegate_depth = 0; parent.session_id = "sess"; parent._interrupt_requested = False parent._active_children = []; parent._active_children_lock = None def child(task_index, goal, child=None, parent_agent=None, **kw): if task_index == 1: time.sleep(600) return {"task_index": task_index, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed"} def build(**kw): c = MagicMock(); c._delegate_role = "leaf"; c._subagent_id = f"s{kw['task_index']}"; return c creds = {"model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None} dt._build_child_agent = build; dt._run_single_child = child; dt._resolve_delegation_credentials = lambda *a, **k: creds dt.delegate_task(tasks=[{"goal": "fast member of the group task", "group": "g"}, {"goal": "slow member of the group task", "group": "g"}], background=True, parent_agent=parent) time.sleep(2.0) sys.stdout.flush(); os._exit(1) ''' subprocess.run([sys.executable, "-c", producer], cwd=repo, env=env, text=True, capture_output=True, timeout=30) consumer = r''' import json, queue from tools import async_delegation as ad q = queue.Queue(); ad.restore_undelivered_completions(q) print(json.dumps(q.get_nowait(), sort_keys=True)) ''' second = subprocess.run([sys.executable, "-c", consumer], cwd=repo, env=env, text=True, capture_output=True, timeout=15, check=True) evt = json.loads(second.stdout.strip().splitlines()[-1]) by_index = {r["task_index"]: r for r in evt["results"]} assert by_index[0]["status"] == "completed" and by_index[0]["summary"] == "done: fast member of the group task" assert by_index[1]["status"] == "unknown" assert "1/2 child results were recorded" in evt["error"] assert "done: fast member" in format_process_notification(evt) def test_one_child_unit_keeps_its_finished_child_when_the_owner_dies(tmp_path): """#116000: a detached unit with exactly ONE child had NO durable record of that child at all — only the multi-child join path called ``record_unit_child`` — so an owner death (OOM-kill / orphaning) anywhere in the window after the child returned (host-owned finalize, transcripts, manifest, then the durable completion write) replayed a bare "outcome unknown" and threw the finished work away. Real-import E2E: a one-task background ``delegate_task`` child completes, the owner is killed while blocked inside that window, and a fresh process must replay the child's real result to the parent.""" repo = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) marker = tmp_path / "child-returned.flag" env = {**os.environ, "HERMES_HOME": str(tmp_path), "PYTHONPATH": repo, "REPRO_MARKER": str(marker)} producer = r''' import os, sys, time from unittest.mock import MagicMock import tools.delegate_tool as dt import tools.delegate_tool_dispatch as dtd parent = MagicMock(); parent._delegate_depth = 0; parent.session_id = "sess"; parent._interrupt_requested = False parent._active_children = []; parent._active_children_lock = None def child(task_index, goal, child=None, parent_agent=None, **kw): return {"task_index": task_index, "status": "completed", "summary": f"done: {goal}", "api_calls": 1, "duration_seconds": 0.1, "model": "m", "exit_reason": "completed"} def build(**kw): c = MagicMock(); c._delegate_role = "leaf"; c._subagent_id = f"s{kw['task_index']}"; return c creds = {"model": "m", "provider": None, "base_url": None, "api_key": None, "api_mode": None, "command": None, "args": None} dt._build_child_agent = build; dt._run_single_child = child; dt._resolve_delegation_credentials = lambda *a, **k: creds def held_finalize(*a, **k): # The child's result exists; the owner still has host-owned finalize + transcripts + manifest + the durable # write to do. Block HERE so the driver kills the owner inside that window: deterministic, no race. open(os.environ["REPRO_MARKER"], "w").write("child-returned") time.sleep(600) dtd._finalize_child_results = held_finalize dt.delegate_task(tasks=[{"goal": "single background subagent"}], background=True, parent_agent=parent) time.sleep(600) ''' proc = subprocess.Popen([sys.executable, "-u", "-c", producer], cwd=repo, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: deadline = time.monotonic() + 60 while not marker.exists() and proc.poll() is None and time.monotonic() < deadline: time.sleep(0.05) assert marker.exists(), "the child never returned, so the death window was never reached" finally: proc.kill() proc.wait(timeout=20) time.sleep(0.3) # let the OS reap the owner before recovery asks whether its pid is alive consumer = r''' import json, queue from tools import async_delegation as ad q = queue.Queue(); ad.restore_undelivered_completions(q) print(json.dumps(q.get_nowait(), sort_keys=True)) ''' second = subprocess.run([sys.executable, "-u", "-c", consumer], cwd=repo, env=env, text=True, capture_output=True, timeout=30, check=True) evt = json.loads(second.stdout.strip().splitlines()[-1]) (entry,) = evt["results"] # the finished child, not a fabricated "unknown" assert entry["status"] == "completed" and entry["summary"] == "done: single background subagent" assert "1/1 child results were recorded" in evt["error"] assert "done: single background subagent" in format_process_notification(evt) @pytest.mark.platforms("posix") # POSIX mode bits not enforced on Windows def test_connect_creates_state_db_0o600_under_permissive_umask(tmp_path, monkeypatch): """``_connect`` shares state.db with hermes_state.SessionDB -- a fresh HERMES_HOME must land the file (and its WAL sidecar, if created) at 0o600 even under a permissive process umask, not the SessionDB-only path.""" import stat monkeypatch.setenv("HERMES_HOME", str(tmp_path)) old_umask = os.umask(0o022) try: conn = ad._connect() conn.close() finally: os.umask(old_umask) db_path = tmp_path / "state.db" assert stat.S_IMODE(db_path.stat().st_mode) == 0o600 for suffix in ("-wal", "-shm"): sidecar = tmp_path / f"state.db{suffix}" if sidecar.exists(): assert stat.S_IMODE(sidecar.stat().st_mode) == 0o600 def test_persist_failure_still_delivers_result_and_frees_slot(monkeypatch): """A failing terminal durable write (locked/full state.db) must not eat the completion event or park the record on ``finalizing`` (#76605, #112030): the event is the only delivery path and ``finalizing`` counts against ``max_concurrent_children`` forever.""" def boom(event, result): raise RuntimeError("database is locked") monkeypatch.setattr(ad, "_persist_completion", boom) res = ad.dispatch_async_delegation( goal="g", context=None, toolsets=None, role="leaf", model="m", session_key="", runner=lambda: {"status": "completed", "summary": "done"}, max_async_children=1, ) evt = _drain_for(res["delegation_id"]) assert evt is not None and evt["status"] == "completed" and evt["summary"] == "done" deadline = time.monotonic() + 2.0 while ad.active_count() and time.monotonic() < deadline: time.sleep(0.02) assert ad.active_count() == 0 with ad._records_lock: assert ad._records[res["delegation_id"]]["status"] == "completed" def test_prune_never_evicts_live_records(): """Retention pruning drops TERMINAL records only; ``stalling``/``finalizing`` are live work (#76605, #112030). A stalling record has no ``completed_at`` so it sorts oldest and was the first eviction candidate, sending its late runner return into the missing-record path.""" with ad._records_lock: for status, ts in (("stalling", 1.0), ("finalizing", 2.0), ("running", 3.0)): ad._records[f"live-{status}"] = {"delegation_id": f"live-{status}", "status": status, "dispatched_at": ts, "completed_at": None} for i in range(ad._MAX_RETAINED_COMPLETED + 1): ad._records[f"done-{i}"] = {"delegation_id": f"done-{i}", "status": "completed", "dispatched_at": 100.0 + i, "completed_at": 200.0 + i} ad._prune_completed_locked() survivors = set(ad._records) assert {"live-stalling", "live-finalizing", "live-running"} <= survivors assert "done-0" not in survivors and len(survivors - {"live-stalling", "live-finalizing", "live-running"}) == ad._MAX_RETAINED_COMPLETED