hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

fix(api_server): stamp the drain boundary on restart too; one HTTP-level invariant; docs

Browse Source 
- request_restart() opens the same drain window as stop() (new turns refused,
  in-flight work awaited), so pollers of GET /v1/runs/{id} need the
  shutdown_requested_at marker from that moment as well; the marker is idempotent
  so stop() re-marking after the restart wait is a no-op.
- Collapse the three adapter-level tests into one invariant that drives the real
  GET /v1/runs/{id} route: live run keeps status=running but carries the marker
  (durably), a status set after the boundary inherits it, a terminal run never does.
- Shutdown-path fakes gain _mark_api_runs_shutdown_requested (the real mixin method
  where the fake already borrows _api_server_hook, a 0-stub where it has no adapter).
- Document the field on the runs API page.

Refs #115133.
This commit is contained in:
teknium1
2026-09-19 23:43:20 -07:00
committed by Teknium
parent 3c6952b33c
commit 6d335529ea
5 changed files with 36 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
gateway/run_shutdown.py
View File

@@ -1584,6 +1584,9 @@ class GatewayShutdownMixin:
self._restart_task_started = True
# Refuse new turns; keep ``_running`` True so the active turn can still deliver its final response.
self._draining = True
# The restart's after-turn wait is a drain window too: pollers of GET /v1/runs/{id} must see
# the boundary from the moment new turns are refused, not only once stop() begins (#115133).
self._mark_api_runs_shutdown_requested()
async def _run_restart() -> None:
await self._await_active_work_before_restart()

46
tests/gateway/test_api_server_runs.py
View File

@@ -411,35 +411,41 @@ class TestStartRun:
class TestRunStatus:
def test_shutdown_marker_is_persisted_for_live_run(self, adapter):
@pytest.mark.asyncio
async def test_drain_boundary_is_visible_to_pollers_on_live_runs_only(self, adapter):
"""GET /v1/runs/{id} shows ``shutdown_requested_at`` as soon as the drain starts (#115133).
A live run keeps ``status: running`` (it is still being served) but gains the marker,
durably (the idempotency record carries it across a restart); a run whose status is set
after the boundary inherits it; a terminal run is never touched.
"""
status = adapter._set_run_status("run_live", "running")
scope = "shutdown-test-scope"
adapter._run_owners["run_live"] = scope
_claim_run(adapter, "run_live")
scope = adapter._run_owners["run_live"]
adapter._run_idempotency_store.reserve(
scope, "shutdown-test-key", "shutdown-test-fingerprint", "run_live", status)
adapter._run_idempotency_ids.add("run_live")
adapter._run_statuses["run_done"] = {
"object": "hermes.run", "run_id": "run_done", "status": "completed"}
_claim_run(adapter, "run_done")
marked = adapter.mark_shutdown_requested()
async with TestClient(TestServer(_create_runs_app(adapter))) as client:
before = await (await client.get("/v1/runs/run_live")).json()
assert "shutdown_requested_at" not in before
assert adapter.mark_shutdown_requested() == 1
live = await (await client.get("/v1/runs/run_live")).json()
assert live["status"] == "running"
marker = live["shutdown_requested_at"]
assert isinstance(marker, float)
done = await (await client.get("/v1/runs/run_done")).json()
assert "shutdown_requested_at" not in done
assert marked == 1
marker = adapter._run_statuses["run_live"].get("shutdown_requested_at")
assert isinstance(marker, float)
durable = adapter._run_idempotency_store.status_for_run(scope, "run_live")
assert durable["status"].get("shutdown_requested_at") == marker
def test_shutdown_marker_is_inherited_by_late_status(self, adapter):
adapter.mark_shutdown_requested()
adapter._set_run_status("run_late", "queued")
assert isinstance(adapter._run_statuses["run_late"].get("shutdown_requested_at"), float)
def test_shutdown_marker_does_not_touch_terminal_run(self, adapter):
adapter._run_statuses["run_done"] = {
"object": "hermes.run", "run_id": "run_done", "status": "completed",
}
assert adapter.mark_shutdown_requested() == 0
assert "shutdown_requested_at" not in adapter._run_statuses["run_done"]
assert adapter._run_statuses["run_late"]["shutdown_requested_at"] == marker
@pytest.mark.asyncio
async def test_status_reflects_explicit_session_id(self, adapter):

4
tests/gateway/test_shutdown_cache_cleanup.py
View File

@@ -65,6 +65,10 @@ class _FakeGateway:
# This fake has no API server adapter, so it is always idle.
return 0
def _mark_api_runs_shutdown_requested(self):
# No API server adapter -> no durable runs to stamp with the drain boundary (#115133).
return 0
def _update_runtime_status(self, *_a, **_kw):
pass

1
tests/gateway/test_shutdown_executor_quiesce.py
View File

@@ -80,6 +80,7 @@ class _FakeGateway:
# Real hook + counter: 0 while ``adapters`` is empty, the API-server count once a fake adapter is in.
_api_server_hook = gw_mod.GatewayShutdownMixin._api_server_hook
_mark_api_runs_shutdown_requested = gw_mod.GatewayShutdownMixin._mark_api_runs_shutdown_requested
_active_api_run_count = gw_mod.GatewayShutdownMixin._active_api_run_count
_active_deferred_agent_worker_count = gw_mod.GatewayShutdownMixin._active_deferred_agent_worker_count

2
website/docs/user-guide/features/api-server.md
View File

@@ -486,6 +486,8 @@ Poll the current run state. This is useful for dashboards that need status witho
Statuses are retained briefly after terminal states (`completed`, `failed`, `cancelled`, or `interrupted`) for polling and UI reconciliation. When the gateway shuts down while a run is active, the run is persisted as `interrupted` (error `Gateway shutdown interrupted the run.`, terminal event `run.interrupted`) before the agent is asked to stop, so a durable run never survives a restart as `running`; a late result from the interrupted turn cannot overwrite it.
While the gateway is still draining (a `hermes gateway stop`/`restart` or SIGTERM with a turn in flight), every non-terminal run additionally carries `shutdown_requested_at` (Unix seconds) from the moment new turns are refused. `status` stays `running` because the turn is still being served; a poller that sees the field knows the process is on its way out and the run will end `interrupted` at the latest when the drain budget expires. Terminal runs never gain the field.
### GET /v1/runs/\{run_id\}/events
Server-Sent Events stream of the run's tool-call progress, token deltas, and lifecycle events. Designed for dashboards and thick clients that want to attach/detach without losing state.