cdd95c8dcf
_notify_active_sessions_of_shutdown now walks _served_home_channel_configs(), which reads self.config.platforms eagerly instead of only self.adapters, so the __new__-built runner in TestGatewayCleanupWiring raised AttributeError on stop(). GatewayRunner.__init__ always binds config before stop() is reachable, so the runner in the test now mirrors production with an empty GatewayConfig().
465 lines
17 KiB
Python
465 lines
17 KiB
Python
"""Tests for zombie process cleanup — verifies processes spawned by tools
|
|
are properly reaped when agent sessions end.
|
|
|
|
Reproduction for issue #7131: zombie process accumulation on long-running
|
|
gateway deployments.
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
def _spawn_sleep(seconds: float = 60) -> subprocess.Popen:
|
|
"""Spawn a portable long-lived Python sleep process (no shell wrapper)."""
|
|
return subprocess.Popen(
|
|
[sys.executable, "-c", f"import time; time.sleep({seconds})"],
|
|
)
|
|
|
|
|
|
def _pid_alive(pid: int) -> bool:
|
|
"""Return True if a process with the given PID is still running."""
|
|
try:
|
|
os.kill(pid, 0)
|
|
return True
|
|
except (ProcessLookupError, PermissionError):
|
|
return False
|
|
|
|
|
|
|
|
class TestAgentCloseMethod:
|
|
"""Verify AIAgent.close() exists, is idempotent, and calls cleanup."""
|
|
|
|
def test_close_calls_cleanup_functions(self):
|
|
"""close() should release every session-owned execution backend."""
|
|
from unittest.mock import patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-cleanup"
|
|
agent._process_owner_task_ids = {"sa-owned"}
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
|
|
with patch("tools.process_registry.process_registry") as mock_registry, \
|
|
patch("run_agent.cleanup_vm") as mock_cleanup_vm, \
|
|
patch("run_agent.cleanup_browser") as mock_cleanup_browser, \
|
|
patch("tools.computer_use.tool.release_computer_use_session") as mock_cleanup_cua:
|
|
mock_registry.list_sessions.return_value = [
|
|
{"session_id": "owned", "owner_task_id": "sa-owned", "status": "running"},
|
|
{"session_id": "foreign", "owner_task_id": "parent", "status": "running"},
|
|
{"session_id": "finished", "owner_task_id": "sa-owned", "status": "exited"},
|
|
]
|
|
agent.close()
|
|
|
|
mock_registry.kill_process.assert_called_once_with(
|
|
"owned", source="agent_close", consume_output=True,
|
|
)
|
|
mock_registry.kill_all.assert_not_called()
|
|
mock_cleanup_vm.assert_called_once_with("test-close-cleanup")
|
|
mock_cleanup_browser.assert_called_once_with("test-close-cleanup")
|
|
mock_cleanup_cua.assert_called_once_with("test-close-cleanup")
|
|
|
|
def test_close_is_idempotent(self):
|
|
"""close() can be called multiple times without error."""
|
|
from unittest.mock import patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-idempotent"
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
|
|
agent.close()
|
|
agent.close()
|
|
agent.close()
|
|
|
|
def test_close_releases_computer_use_when_earlier_cleanup_fails(self):
|
|
"""One failed cleanup step must not strand the computer-use session."""
|
|
from unittest.mock import patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-after-failure"
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
|
|
with patch(
|
|
"tools.process_registry.process_registry.list_sessions",
|
|
side_effect=RuntimeError("process cleanup failed"),
|
|
), patch(
|
|
"tools.computer_use.tool.release_computer_use_session",
|
|
) as mock_cleanup_cua:
|
|
agent.close()
|
|
|
|
mock_cleanup_cua.assert_called_once_with(
|
|
"test-close-after-failure"
|
|
)
|
|
|
|
def test_soft_client_release_preserves_computer_use_session(self):
|
|
"""Cache eviction is not a hard session boundary."""
|
|
from unittest.mock import patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-soft-release"
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
|
|
with patch(
|
|
"tools.computer_use.tool.release_computer_use_session",
|
|
) as mock_cleanup_cua:
|
|
agent.release_clients()
|
|
|
|
mock_cleanup_cua.assert_not_called()
|
|
|
|
def test_close_propagates_to_children(self):
|
|
"""close() should call close() on all active child agents."""
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-children"
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
|
|
child_1 = MagicMock()
|
|
child_2 = MagicMock()
|
|
agent._active_children = [child_1, child_2]
|
|
|
|
agent.close()
|
|
|
|
child_1.close.assert_called_once()
|
|
child_2.close.assert_called_once()
|
|
assert agent._active_children == []
|
|
|
|
def test_close_ends_owned_session_row(self):
|
|
"""close() finalizes the agent's owned SQLite session row."""
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-session-row"
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
agent._end_session_on_close = True
|
|
agent._session_db = MagicMock()
|
|
|
|
agent.close()
|
|
|
|
agent._session_db.end_session.assert_called_once_with(
|
|
"test-close-session-row", "agent_close"
|
|
)
|
|
|
|
def test_close_skips_session_end_for_forwarded_continuation_agents(self):
|
|
"""Helper agents that handed session ownership forward opt out."""
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
with patch("run_agent.AIAgent.__init__", return_value=None):
|
|
from run_agent import AIAgent
|
|
agent = AIAgent.__new__(AIAgent)
|
|
agent.session_id = "test-close-forwarded-session"
|
|
agent._active_children = []
|
|
agent._active_children_lock = threading.Lock()
|
|
agent.client = None
|
|
agent._end_session_on_close = False
|
|
agent._session_db = MagicMock()
|
|
|
|
agent.close()
|
|
|
|
agent._session_db.end_session.assert_not_called()
|
|
|
|
|
|
|
|
|
|
class TestGatewayCleanupWiring:
|
|
"""Verify gateway lifecycle calls close() on agents."""
|
|
|
|
def test_gateway_stop_calls_close(self):
|
|
"""gateway stop() should call close() on all running agents."""
|
|
import asyncio
|
|
import threading
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from gateway.config import GatewayConfig
|
|
from gateway.run import GatewayRunner
|
|
|
|
runner = object.__new__(GatewayRunner)
|
|
runner._running = True
|
|
runner._running_agents = {}
|
|
runner._running_agents_ts = {}
|
|
# __init__ always binds ``config``; the shutdown notice now walks every served
|
|
# profile's configured home channels from it, not just the live adapters.
|
|
runner.config = GatewayConfig()
|
|
runner.adapters = {}
|
|
runner._background_tasks = set()
|
|
runner._pending_messages = {}
|
|
runner._pending_approvals = {}
|
|
runner._pending_model_notes = {}
|
|
runner._shutdown_event = asyncio.Event()
|
|
runner._exit_reason = None
|
|
runner._exit_code = None
|
|
runner._stop_task = None
|
|
runner._draining = False
|
|
runner._restart_requested = False
|
|
runner._restart_task_started = False
|
|
runner._restart_detached = False
|
|
runner._restart_via_service = False
|
|
runner._restart_drain_timeout = 0.1
|
|
runner._voice_mode = {}
|
|
runner._session_model_overrides = {}
|
|
runner._update_prompt_pending = {}
|
|
runner._busy_input_mode = "interrupt"
|
|
runner._agent_cache = {}
|
|
runner._agent_cache_lock = threading.Lock()
|
|
runner._shutdown_all_gateway_honcho = lambda: None
|
|
runner._update_runtime_status = MagicMock()
|
|
|
|
mock_agent_1 = MagicMock()
|
|
mock_agent_2 = MagicMock()
|
|
runner._running_agents = {
|
|
"session-1": mock_agent_1,
|
|
"session-2": mock_agent_2,
|
|
}
|
|
|
|
loop = asyncio.new_event_loop()
|
|
try:
|
|
with patch("gateway.status.remove_pid_file"), \
|
|
patch("gateway.status.write_runtime_status"), \
|
|
patch("tools.terminal_tool.cleanup_all_environments"), \
|
|
patch("tools.browser_tool_lifecycle.cleanup_all_browsers"):
|
|
loop.run_until_complete(GatewayRunner.stop(runner))
|
|
finally:
|
|
loop.close()
|
|
|
|
mock_agent_1.close.assert_called()
|
|
mock_agent_2.close.assert_called()
|
|
|
|
def test_evict_does_not_call_close(self):
|
|
"""_evict_cached_agent() should NOT call close() — it's also used
|
|
for non-destructive refreshes (model switch, branch, fallback)."""
|
|
import threading
|
|
from unittest.mock import MagicMock
|
|
|
|
from gateway.run import GatewayRunner
|
|
|
|
runner = object.__new__(GatewayRunner)
|
|
runner._agent_cache_lock = threading.Lock()
|
|
|
|
mock_agent = MagicMock()
|
|
runner._agent_cache = {"session-key": (mock_agent, 12345)}
|
|
|
|
GatewayRunner._evict_cached_agent(runner, "session-key")
|
|
|
|
mock_agent.close.assert_not_called()
|
|
assert "session-key" not in runner._agent_cache
|
|
|
|
|
|
class TestDelegationCleanup:
|
|
"""Verify subagent delegation cleans up child agents."""
|
|
|
|
def test_run_single_child_calls_close(self, monkeypatch, tmp_path):
|
|
"""_run_single_child finally block should call close() on child."""
|
|
from unittest.mock import MagicMock
|
|
from hermes_constants import (
|
|
get_hermes_home,
|
|
reset_hermes_home_override,
|
|
set_hermes_home_override,
|
|
)
|
|
from agent import relay_runtime
|
|
from tools.delegate_tool import _run_single_child
|
|
|
|
parent = MagicMock()
|
|
parent._active_children = []
|
|
parent._active_children_lock = threading.Lock()
|
|
|
|
child = MagicMock()
|
|
child.session_id = "child-session"
|
|
child._delegate_saved_tool_names = ["tool1"]
|
|
observed = {}
|
|
|
|
def run_conversation(**_kwargs):
|
|
observed["hermes_home"] = get_hermes_home()
|
|
raise RuntimeError("test abort")
|
|
|
|
child.run_conversation.side_effect = run_conversation
|
|
relay_host = MagicMock()
|
|
monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host)
|
|
|
|
parent._active_children.append(child)
|
|
|
|
profile_home = tmp_path / "profile-a"
|
|
token = set_hermes_home_override(profile_home)
|
|
try:
|
|
result = _run_single_child(
|
|
task_index=0,
|
|
goal="test goal",
|
|
child=child,
|
|
parent_agent=parent,
|
|
)
|
|
finally:
|
|
reset_hermes_home_override(token)
|
|
|
|
child.close.assert_called_once()
|
|
assert observed["hermes_home"] == profile_home
|
|
relay_host.unregister_subagent.assert_called_once_with(
|
|
{"child_session_id": "child-session"}
|
|
)
|
|
assert child not in parent._active_children
|
|
assert result["status"] == "error"
|
|
|
|
def test_active_child_turn_owns_relay_scope_cleanup(self, monkeypatch):
|
|
from unittest.mock import MagicMock
|
|
|
|
from agent import relay_runtime
|
|
from tools.delegate_tool import _run_single_child
|
|
|
|
parent = MagicMock()
|
|
parent._active_children = []
|
|
parent._active_children_lock = threading.Lock()
|
|
child = MagicMock()
|
|
child.session_id = "active-child-session"
|
|
child._delegate_saved_tool_names = ["tool1"]
|
|
child.run_conversation.side_effect = RuntimeError("test abort")
|
|
parent._active_children.append(child)
|
|
relay_host = MagicMock()
|
|
monkeypatch.setattr(relay_runtime, "get_runtime", lambda **kwargs: relay_host)
|
|
monkeypatch.setattr(
|
|
relay_runtime.SESSION_COORDINATOR,
|
|
"has_active_turn",
|
|
lambda **_kwargs: True,
|
|
)
|
|
|
|
result = _run_single_child(
|
|
task_index=0,
|
|
goal="test active turn cleanup",
|
|
child=child,
|
|
parent_agent=parent,
|
|
)
|
|
|
|
assert result["status"] == "error"
|
|
relay_host.unregister_subagent.assert_not_called()
|
|
|
|
def test_timed_out_child_keeps_relay_session_until_its_turn_exits(
|
|
self, monkeypatch, tmp_path
|
|
):
|
|
from unittest.mock import MagicMock
|
|
|
|
from agent import relay_runtime
|
|
from hermes_constants import (
|
|
reset_hermes_home_override,
|
|
set_hermes_home_override,
|
|
)
|
|
from tools.delegate_tool import _run_single_child
|
|
|
|
relay_runtime._reset_for_tests()
|
|
profile_home = tmp_path / "profile-timeout"
|
|
profile_token = set_hermes_home_override(profile_home)
|
|
child_started = threading.Event()
|
|
release_child = threading.Event()
|
|
child_finished = threading.Event()
|
|
parent = MagicMock()
|
|
parent._active_children = []
|
|
parent._active_children_lock = threading.Lock()
|
|
child = MagicMock()
|
|
child.session_id = "timed-out-child"
|
|
child._delegate_saved_tool_names = ["tool1"]
|
|
child.get_activity_summary.return_value = {"api_call_count": 1}
|
|
parent._active_children.append(child)
|
|
relay_host = MagicMock()
|
|
monkeypatch.setattr(relay_runtime, "get_runtime", lambda **_kwargs: relay_host)
|
|
monkeypatch.setattr("tools.delegate_tool._get_child_timeout", lambda: 2)
|
|
|
|
# The parent's cap must not elapse before the worker thread has opened the child's turn, or
|
|
# the "late result" scenario degrades into "child never started" on a loaded runner. Gate the
|
|
# settle wait the timeout rides on (``_ChildRun.run`` waits on ``heartbeat.settled``).
|
|
class _GatedSettle(threading.Event):
|
|
def wait(self, timeout=None):
|
|
child_started.wait(timeout=5)
|
|
return super().wait(timeout)
|
|
|
|
import tools.delegate_tool as delegate_tool
|
|
|
|
real_start_heartbeat = delegate_tool._start_heartbeat
|
|
|
|
def start_gated_heartbeat(*args, **kwargs):
|
|
heartbeat = real_start_heartbeat(*args, **kwargs)
|
|
heartbeat.settled = _GatedSettle()
|
|
return heartbeat
|
|
|
|
monkeypatch.setattr(delegate_tool, "_start_heartbeat", start_gated_heartbeat)
|
|
|
|
def run_conversation(**kwargs):
|
|
lease = relay_runtime.SESSION_COORDINATOR.acquire_conversation(
|
|
profile_key=relay_runtime.current_profile_key(),
|
|
session_id=child.session_id,
|
|
platform="subagent",
|
|
)
|
|
turn = relay_runtime.SESSION_COORDINATOR.begin_turn(
|
|
lease,
|
|
turn_id="timed-out-child-turn",
|
|
task_id=kwargs["task_id"],
|
|
)
|
|
child_started.set()
|
|
try:
|
|
assert release_child.wait(timeout=30), "test did not release the child"
|
|
return {
|
|
"final_response": "late result",
|
|
"completed": True,
|
|
"interrupted": False,
|
|
"api_calls": 1,
|
|
"messages": [],
|
|
}
|
|
finally:
|
|
relay_runtime.SESSION_COORDINATOR.end_turn(
|
|
turn,
|
|
outcome="cancelled",
|
|
)
|
|
relay_runtime.SESSION_COORDINATOR.release_conversation(lease)
|
|
child_finished.set()
|
|
|
|
child.run_conversation.side_effect = run_conversation
|
|
try:
|
|
result = _run_single_child(
|
|
task_index=0,
|
|
goal="test timed-out turn cleanup",
|
|
child=child,
|
|
parent_agent=parent,
|
|
)
|
|
|
|
assert child_started.is_set()
|
|
assert result["status"] == "timeout"
|
|
assert relay_runtime.SESSION_COORDINATOR.has_active_turn(
|
|
profile_key=str(profile_home),
|
|
session_id=child.session_id,
|
|
)
|
|
relay_host.unregister_subagent.assert_not_called()
|
|
|
|
release_child.set()
|
|
assert child_finished.wait(timeout=5)
|
|
assert not relay_runtime.SESSION_COORDINATOR.has_active_turn(
|
|
profile_key=str(profile_home),
|
|
session_id=child.session_id,
|
|
)
|
|
finally:
|
|
release_child.set()
|
|
if child_started.is_set():
|
|
assert child_finished.wait(timeout=10)
|
|
reset_hermes_home_override(profile_token)
|
|
relay_runtime._reset_for_tests()
|