diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py index d2d4ad1f2e..58e20a4edf 100644 --- a/gateway/run_shutdown.py +++ b/gateway/run_shutdown.py @@ -1584,6 +1584,9 @@ class GatewayShutdownMixin: self._restart_task_started = True # Refuse new turns; keep ``_running`` True so the active turn can still deliver its final response. self._draining = True + # The restart's after-turn wait is a drain window too: pollers of GET /v1/runs/{id} must see + # the boundary from the moment new turns are refused, not only once stop() begins (#115133). + self._mark_api_runs_shutdown_requested() async def _run_restart() -> None: await self._await_active_work_before_restart() diff --git a/tests/gateway/test_api_server_runs.py b/tests/gateway/test_api_server_runs.py index 9ccbda882e..0a85ddadfd 100644 --- a/tests/gateway/test_api_server_runs.py +++ b/tests/gateway/test_api_server_runs.py @@ -411,35 +411,41 @@ class TestStartRun: class TestRunStatus: - def test_shutdown_marker_is_persisted_for_live_run(self, adapter): + @pytest.mark.asyncio + async def test_drain_boundary_is_visible_to_pollers_on_live_runs_only(self, adapter): + """GET /v1/runs/{id} shows ``shutdown_requested_at`` as soon as the drain starts (#115133). + + A live run keeps ``status: running`` (it is still being served) but gains the marker, + durably (the idempotency record carries it across a restart); a run whose status is set + after the boundary inherits it; a terminal run is never touched. + """ status = adapter._set_run_status("run_live", "running") - scope = "shutdown-test-scope" - adapter._run_owners["run_live"] = scope + _claim_run(adapter, "run_live") + scope = adapter._run_owners["run_live"] adapter._run_idempotency_store.reserve( scope, "shutdown-test-key", "shutdown-test-fingerprint", "run_live", status) adapter._run_idempotency_ids.add("run_live") + adapter._run_statuses["run_done"] = { + "object": "hermes.run", "run_id": "run_done", "status": "completed"} + _claim_run(adapter, "run_done") - marked = adapter.mark_shutdown_requested() + async with TestClient(TestServer(_create_runs_app(adapter))) as client: + before = await (await client.get("/v1/runs/run_live")).json() + assert "shutdown_requested_at" not in before + + assert adapter.mark_shutdown_requested() == 1 + + live = await (await client.get("/v1/runs/run_live")).json() + assert live["status"] == "running" + marker = live["shutdown_requested_at"] + assert isinstance(marker, float) + done = await (await client.get("/v1/runs/run_done")).json() + assert "shutdown_requested_at" not in done - assert marked == 1 - marker = adapter._run_statuses["run_live"].get("shutdown_requested_at") - assert isinstance(marker, float) durable = adapter._run_idempotency_store.status_for_run(scope, "run_live") assert durable["status"].get("shutdown_requested_at") == marker - - def test_shutdown_marker_is_inherited_by_late_status(self, adapter): - adapter.mark_shutdown_requested() adapter._set_run_status("run_late", "queued") - - assert isinstance(adapter._run_statuses["run_late"].get("shutdown_requested_at"), float) - - def test_shutdown_marker_does_not_touch_terminal_run(self, adapter): - adapter._run_statuses["run_done"] = { - "object": "hermes.run", "run_id": "run_done", "status": "completed", - } - - assert adapter.mark_shutdown_requested() == 0 - assert "shutdown_requested_at" not in adapter._run_statuses["run_done"] + assert adapter._run_statuses["run_late"]["shutdown_requested_at"] == marker @pytest.mark.asyncio async def test_status_reflects_explicit_session_id(self, adapter): diff --git a/tests/gateway/test_shutdown_cache_cleanup.py b/tests/gateway/test_shutdown_cache_cleanup.py index 3123bb825f..4409541bc0 100644 --- a/tests/gateway/test_shutdown_cache_cleanup.py +++ b/tests/gateway/test_shutdown_cache_cleanup.py @@ -65,6 +65,10 @@ class _FakeGateway: # This fake has no API server adapter, so it is always idle. return 0 + def _mark_api_runs_shutdown_requested(self): + # No API server adapter -> no durable runs to stamp with the drain boundary (#115133). + return 0 + def _update_runtime_status(self, *_a, **_kw): pass diff --git a/tests/gateway/test_shutdown_executor_quiesce.py b/tests/gateway/test_shutdown_executor_quiesce.py index ad54c4c236..d37dc2d315 100644 --- a/tests/gateway/test_shutdown_executor_quiesce.py +++ b/tests/gateway/test_shutdown_executor_quiesce.py @@ -80,6 +80,7 @@ class _FakeGateway: # Real hook + counter: 0 while ``adapters`` is empty, the API-server count once a fake adapter is in. _api_server_hook = gw_mod.GatewayShutdownMixin._api_server_hook + _mark_api_runs_shutdown_requested = gw_mod.GatewayShutdownMixin._mark_api_runs_shutdown_requested _active_api_run_count = gw_mod.GatewayShutdownMixin._active_api_run_count _active_deferred_agent_worker_count = gw_mod.GatewayShutdownMixin._active_deferred_agent_worker_count diff --git a/website/docs/user-guide/features/api-server.md b/website/docs/user-guide/features/api-server.md index 94d9c9daa3..32d586ffb9 100644 --- a/website/docs/user-guide/features/api-server.md +++ b/website/docs/user-guide/features/api-server.md @@ -486,6 +486,8 @@ Poll the current run state. This is useful for dashboards that need status witho Statuses are retained briefly after terminal states (`completed`, `failed`, `cancelled`, or `interrupted`) for polling and UI reconciliation. When the gateway shuts down while a run is active, the run is persisted as `interrupted` (error `Gateway shutdown interrupted the run.`, terminal event `run.interrupted`) before the agent is asked to stop, so a durable run never survives a restart as `running`; a late result from the interrupted turn cannot overwrite it. +While the gateway is still draining (a `hermes gateway stop`/`restart` or SIGTERM with a turn in flight), every non-terminal run additionally carries `shutdown_requested_at` (Unix seconds) from the moment new turns are refused. `status` stays `running` because the turn is still being served; a poller that sees the field knows the process is on its way out and the run will end `interrupted` at the latest when the drain budget expires. Terminal runs never gain the field. + ### GET /v1/runs/\{run_id\}/events Server-Sent Events stream of the run's tool-call progress, token deltas, and lifecycle events. Designed for dashboards and thick clients that want to attach/detach without losing state.