fix(gateway): include cron drain in restart exit wait

(cherry picked from commit aefbb0741ed3d8491ede6e2d26689022c69a85bc)
This commit is contained in:
Brian Le
2026-09-26 04:41:36 -07:00
committed by kshitij
parent 86f449b172
commit 5aad192995
5 changed files with 27 additions and 10 deletions

View File

@@ -355,7 +355,9 @@ def resolve_systemd_timeout_stop_sec(
return int(max(_seconds(floor_s), max(drain, cron_budget) + _seconds(headroom_s)))
def resolve_restart_exit_wait_budget(drain_timeout: float, after_turn_timeout: float, *, headroom: float = 15.0) -> float:
def resolve_restart_exit_wait_budget(
drain_timeout: float, after_turn_timeout: float, cron_drain_timeout: float = 0.0, *, headroom: float = 15.0,
) -> float:
"""Seconds a CLI should wait for the gateway PID to exit after SIGUSR1: in-band restart may
defer ``stop()`` until turns finish, then spend ``drain_timeout`` inside it — cover both."""
return _seconds(drain_timeout) + _seconds(after_turn_timeout) + _seconds(headroom)
defer ``stop()`` until turns finish, then spend the longer chat or cron drain inside it."""
return max(_seconds(drain_timeout), _seconds(cron_drain_timeout)) + _seconds(after_turn_timeout) + _seconds(headroom)

View File

@@ -3561,14 +3561,11 @@ def _get_cron_drain_timeout() -> float:
def _get_restart_exit_wait_budget() -> float:
"""CLI wait for gateway exit after SIGUSR1 / self-restart (#77184)."""
return resolve_restart_exit_wait_budget(
# TimeoutStopSec must cover the full stop budget, not just restart_drain_timeout. Cron work can
# legally wait cron_drain_timeout plus cleanup reserve before interrupt/teardown, and systemd
# SIGKILLs if the unit's deadline is shorter (#94759). 30s of post-drain headroom is preserved on
# top, with a 60s floor.
_get_restart_drain_timeout(),
_agent_timeout_setting(
"HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout
),
_get_cron_drain_timeout(),
)

View File

@@ -682,8 +682,8 @@ def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool:
print(f" {line}")
if RESTART_PENDING_STATE in states:
print()
print(" ℹ A restart-pending gateway picks up the new code as soon as this update exits;")
print(" verify afterwards with `hermes gateway status`.")
print(" ℹ A restart-pending gateway has not yet been verified on the new code;")
print(" check after this update exits with `hermes gateway status`.")
stale_or_down = sum(1 for entry in fleet if entry.get("state") in ("stale", "down"))
if stale_or_down:
print()

View File

@@ -22,9 +22,25 @@ def test_parse_restart_after_turn_timeout_defaults_and_clamps():
def test_resolve_restart_exit_wait_budget_covers_both_phases():
assert resolve_restart_exit_wait_budget(0, 0, headroom=15) == 15.0
assert resolve_restart_exit_wait_budget(180, 21600, headroom=15) == 180 + 21600 + 15
assert resolve_restart_exit_wait_budget(2, 3, 80, headroom=15) == 3 + 80 + 15
assert resolve_restart_exit_wait_budget(80, 3, 2, headroom=15) == 3 + 80 + 15
assert resolve_restart_exit_wait_budget(2, 3, 0, headroom=15) == 3 + 2 + 15
assert resolve_restart_exit_wait_budget("bad", "bad", headroom="x") == 0.0
def test_cli_restart_wait_covers_configured_cron_drain(tmp_path, monkeypatch):
import hermes_cli.gateway as gateway_cli
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
for key in ("HERMES_RESTART_DRAIN_TIMEOUT", "HERMES_RESTART_AFTER_TURN_TIMEOUT", "HERMES_CRON_DRAIN_TIMEOUT"):
monkeypatch.delenv(key, raising=False)
(tmp_path / "config.yaml").write_text(
"agent:\n restart_drain_timeout: 2\n restart_after_turn_timeout: 3\n cron_drain_timeout: 80\n",
encoding="utf-8",
)
assert gateway_cli._get_restart_exit_wait_budget() >= 3 + 80 + 15
def test_load_restart_after_turn_timeout_preserves_zero(tmp_path, monkeypatch):
"""Config/env ``0`` must disable after-turn wait, not fall back to default."""
import gateway.run as gateway_run

View File

@@ -55,8 +55,10 @@ def test_ancestor_with_accepted_self_restart_is_pending_while_other_stale_rows_s
# Without the sibling, the pending row alone is not a failure — and the same pid NOT recorded
# as pending (the restart phase never reached the ancestor branch) is a plain stale verdict.
only_ancestor = [row for row in fleet if row["profile"] == "default"]
with contextlib.redirect_stdout(io.StringIO()):
with contextlib.redirect_stdout(io.StringIO()) as pending_out:
assert ur.print_fleet_version_matrix(only_ancestor) is False
assert "as soon as this update exits" not in pending_out.getvalue()
assert "restart pending" in pending_out.getvalue()
plain = ur.collect_fleet_versions(pre_restart_pids=[ancestor, sibling])
assert {row["state"] for row in plain} == {"stale"}