Files
hermes-agent/tests/gateway/test_hosted_room_gateway_lifecycle.py
Teknium 28834a2098 test: raise tight wall-clock bounds that flaked on loaded CI runners
Seven test files asserted sub-2s wall-clock bounds (elapsed < 0.5/1.0s,
stop(timeout=1.0), event waits of 0.5-2s). Under CI load these fired on
healthy code: main run 33455779041 alone flaked 6 of them in one pass
(observed 1.01s vs 0.5, 1.20s vs 1.0, 3.61s vs 3.0, 1.55s vs 1.0,
stop(1.0) returning False, lease TTL 0.1s expiring before the authority
change was observed).

Per the AGENTS.md flake policy (waits >= 2s), bounds are raised to 5s+
while keeping their teeth: every hang path they guard blocks for 10s+
(release.wait holds), so the loosened bounds still distinguish bounded
from unbounded behavior. The authority-loss test gets a 30s lease TTL so
lease expiry can no longer preempt the authority-change assertion.
2026-09-01 10:52:42 -07:00

235 lines
6.9 KiB
Python

"""Messaging-gateway ownership tests for the hosted Group Chat worker."""
from __future__ import annotations
import asyncio
import threading
import time
from types import SimpleNamespace
import pytest
from gateway import hosted_room_driver, hosted_rooms
from gateway.run import GatewayRunner
from tui_gateway.hosted_room_service import HostedRoomService
class _RPC:
def __init__(self) -> None:
self.sessions = {}
self.submits = []
def resolve_exact(self, *, profile, title, source):
del source
return self.sessions.get((profile, title))
def create(self, *, profile, title, source):
del source
session = {"session_id": f"{profile}-session", "title": title}
self.sessions[(profile, title)] = session
return session
def resume(self, *, profile, session_id, source):
del profile, source
return {"session_id": session_id}
def submit(self, **kwargs):
self.submits.append(kwargs["profile"])
kwargs["on_terminal"]({
"status": "settled",
"text": f"reply from {kwargs['profile']}",
})
return {"accepted": True}
def history(self, **kwargs):
del kwargs
return []
def info(self, **kwargs):
del kwargs
return {"active": False, "task_id": None}
def interrupt(self, **kwargs):
del kwargs
raise AssertionError("gateway lifecycle must not interrupt room work")
def _server():
return SimpleNamespace(_methods={}, _sessions={}, _sessions_lock=threading.Lock())
def _service(db_path, *, profiles=("default",)):
service = HostedRoomService(_server(), db_path=db_path)
rpc = _RPC()
service.rpc = rpc
service.runtime.rpc = rpc
service.local_profiles = lambda: profiles
return service, rpc
def _wait_for(predicate, timeout=3.0):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.01)
raise AssertionError("condition did not settle before timeout")
@pytest.mark.asyncio
async def test_messaging_gateway_supervisor_starts_without_dashboard(monkeypatch):
from tui_gateway import methods_groups
state = {"running": False, "starts": 0}
class Runtime:
def status(self):
return {"running": state["running"], "stopping": False}
service = SimpleNamespace(runtime=Runtime())
def get_service():
return service if state["running"] else None
def start_service():
state["starts"] += 1
state["running"] = True
return service
monkeypatch.setattr(methods_groups, "get_hosted_room_service", get_service)
monkeypatch.setattr(methods_groups, "start_hosted_room_service", start_service)
runner = GatewayRunner.__new__(GatewayRunner)
started = await runner._ensure_hosted_room_worker()
assert started is service
assert state == {"running": True, "starts": 1}
# A dead child is restarted, while a healthy one is left alone.
await runner._ensure_hosted_room_worker()
assert state["starts"] == 1
state["running"] = False
await runner._ensure_hosted_room_worker()
assert state["starts"] == 2
@pytest.mark.asyncio
async def test_dead_room_worker_is_restarted_by_gateway_task_supervision(monkeypatch):
from tui_gateway import methods_groups
starts = {"count": 0}
def fail_start():
starts["count"] += 1
raise RuntimeError("worker unavailable")
monkeypatch.setattr(methods_groups, "get_hosted_room_service", lambda: None)
monkeypatch.setattr(methods_groups, "start_hosted_room_service", fail_start)
monkeypatch.setattr(GatewayRunner, "_MAX_SUPERVISED_RESTARTS", 1)
monkeypatch.setattr(
GatewayRunner,
"_supervised_backoff",
staticmethod(lambda _attempt: 0),
)
runner = GatewayRunner.__new__(GatewayRunner)
runner._running = True
runner._background_tasks = set()
runner._spawn_supervised(
lambda: runner._hosted_room_worker_watcher(interval=0),
"hosted_room_worker",
)
for _ in range(200):
if starts["count"] == 2 and not runner._background_tasks:
break
await asyncio.sleep(0.01)
runner._running = False
assert starts["count"] == 2
assert runner._background_tasks == set()
def test_gateway_restart_resumes_queued_room_for_multiplexed_profile(tmp_path):
db = tmp_path / "state.db"
first, _ = _service(db, profiles=("default", "ops"))
first.create_room(
room_id="room-1",
name="Release room",
members=[
{
"member_id": "default",
"profile": "default",
"handle": "hermes",
},
{"member_id": "ops", "profile": "ops", "handle": "ops"},
],
)
first.send(
room_id="room-1",
event_id="user-1",
payload={"text": "@ops inspect", "thread_id": "thread-1"},
)
assert (
len(hosted_room_driver.list_tasks(db, room_id="room-1", status="queued")) == 1
)
resumed, rpc = _service(db, profiles=("default", "ops"))
resumed.start()
try:
_wait_for(
lambda: any(
event["kind"] == "message.member"
for event in hosted_rooms.read_events(
db, room_id="room-1", since_seq=0
)["events"]
)
)
finally:
assert resumed.stop(timeout=5.0)
assert rpc.submits == ["ops"]
assert hosted_room_driver.list_tasks(db, room_id="room-1", status="settled")
def test_dashboard_and_gateway_workers_share_one_fenced_execution_owner(tmp_path):
db = tmp_path / "state.db"
gateway, gateway_rpc = _service(db, profiles=("default", "ops"))
dashboard, dashboard_rpc = _service(db, profiles=("default", "ops"))
gateway.create_room(
room_id="room-1",
name="Release room",
members=[
{
"member_id": "default",
"profile": "default",
"handle": "hermes",
},
{"member_id": "ops", "profile": "ops", "handle": "ops"},
],
)
gateway.send(
room_id="room-1",
event_id="user-1",
payload={"text": "@ops inspect", "thread_id": "thread-1"},
)
gateway.start()
dashboard.start()
try:
_wait_for(
lambda: any(
event["kind"] == "message.member"
for event in hosted_rooms.read_events(
db, room_id="room-1", since_seq=0
)["events"]
)
)
time.sleep(0.05)
finally:
assert gateway.stop(timeout=5.0)
assert dashboard.stop(timeout=5.0)
assert len(gateway_rpc.submits) + len(dashboard_rpc.submits) == 1
events = hosted_rooms.read_events(db, room_id="room-1", since_seq=0)["events"]
assert sum(event["kind"] == "message.member" for event in events) == 1