fix(gateway): include cron drain in restart exit wait
(cherry picked from commit aefbb0741ed3d8491ede6e2d26689022c69a85bc)
This commit is contained in:
@@ -355,7 +355,9 @@ def resolve_systemd_timeout_stop_sec(
|
||||
return int(max(_seconds(floor_s), max(drain, cron_budget) + _seconds(headroom_s)))
|
||||
|
||||
|
||||
def resolve_restart_exit_wait_budget(drain_timeout: float, after_turn_timeout: float, *, headroom: float = 15.0) -> float:
|
||||
def resolve_restart_exit_wait_budget(
|
||||
drain_timeout: float, after_turn_timeout: float, cron_drain_timeout: float = 0.0, *, headroom: float = 15.0,
|
||||
) -> float:
|
||||
"""Seconds a CLI should wait for the gateway PID to exit after SIGUSR1: in-band restart may
|
||||
defer ``stop()`` until turns finish, then spend ``drain_timeout`` inside it — cover both."""
|
||||
return _seconds(drain_timeout) + _seconds(after_turn_timeout) + _seconds(headroom)
|
||||
defer ``stop()`` until turns finish, then spend the longer chat or cron drain inside it."""
|
||||
return max(_seconds(drain_timeout), _seconds(cron_drain_timeout)) + _seconds(after_turn_timeout) + _seconds(headroom)
|
||||
|
||||
@@ -3561,14 +3561,11 @@ def _get_cron_drain_timeout() -> float:
|
||||
def _get_restart_exit_wait_budget() -> float:
|
||||
"""CLI wait for gateway exit after SIGUSR1 / self-restart (#77184)."""
|
||||
return resolve_restart_exit_wait_budget(
|
||||
# TimeoutStopSec must cover the full stop budget, not just restart_drain_timeout. Cron work can
|
||||
# legally wait cron_drain_timeout plus cleanup reserve before interrupt/teardown, and systemd
|
||||
# SIGKILLs if the unit's deadline is shorter (#94759). 30s of post-drain headroom is preserved on
|
||||
# top, with a 60s floor.
|
||||
_get_restart_drain_timeout(),
|
||||
_agent_timeout_setting(
|
||||
"HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout
|
||||
),
|
||||
_get_cron_drain_timeout(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -682,8 +682,8 @@ def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool:
|
||||
print(f" {line}")
|
||||
if RESTART_PENDING_STATE in states:
|
||||
print()
|
||||
print(" ℹ A restart-pending gateway picks up the new code as soon as this update exits;")
|
||||
print(" verify afterwards with `hermes gateway status`.")
|
||||
print(" ℹ A restart-pending gateway has not yet been verified on the new code;")
|
||||
print(" check after this update exits with `hermes gateway status`.")
|
||||
stale_or_down = sum(1 for entry in fleet if entry.get("state") in ("stale", "down"))
|
||||
if stale_or_down:
|
||||
print()
|
||||
|
||||
@@ -22,9 +22,25 @@ def test_parse_restart_after_turn_timeout_defaults_and_clamps():
|
||||
def test_resolve_restart_exit_wait_budget_covers_both_phases():
|
||||
assert resolve_restart_exit_wait_budget(0, 0, headroom=15) == 15.0
|
||||
assert resolve_restart_exit_wait_budget(180, 21600, headroom=15) == 180 + 21600 + 15
|
||||
assert resolve_restart_exit_wait_budget(2, 3, 80, headroom=15) == 3 + 80 + 15
|
||||
assert resolve_restart_exit_wait_budget(80, 3, 2, headroom=15) == 3 + 80 + 15
|
||||
assert resolve_restart_exit_wait_budget(2, 3, 0, headroom=15) == 3 + 2 + 15
|
||||
assert resolve_restart_exit_wait_budget("bad", "bad", headroom="x") == 0.0
|
||||
|
||||
|
||||
def test_cli_restart_wait_covers_configured_cron_drain(tmp_path, monkeypatch):
|
||||
import hermes_cli.gateway as gateway_cli
|
||||
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
for key in ("HERMES_RESTART_DRAIN_TIMEOUT", "HERMES_RESTART_AFTER_TURN_TIMEOUT", "HERMES_CRON_DRAIN_TIMEOUT"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
"agent:\n restart_drain_timeout: 2\n restart_after_turn_timeout: 3\n cron_drain_timeout: 80\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert gateway_cli._get_restart_exit_wait_budget() >= 3 + 80 + 15
|
||||
|
||||
|
||||
def test_load_restart_after_turn_timeout_preserves_zero(tmp_path, monkeypatch):
|
||||
"""Config/env ``0`` must disable after-turn wait, not fall back to default."""
|
||||
import gateway.run as gateway_run
|
||||
|
||||
@@ -55,8 +55,10 @@ def test_ancestor_with_accepted_self_restart_is_pending_while_other_stale_rows_s
|
||||
# Without the sibling, the pending row alone is not a failure — and the same pid NOT recorded
|
||||
# as pending (the restart phase never reached the ancestor branch) is a plain stale verdict.
|
||||
only_ancestor = [row for row in fleet if row["profile"] == "default"]
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
with contextlib.redirect_stdout(io.StringIO()) as pending_out:
|
||||
assert ur.print_fleet_version_matrix(only_ancestor) is False
|
||||
assert "as soon as this update exits" not in pending_out.getvalue()
|
||||
assert "restart pending" in pending_out.getvalue()
|
||||
plain = ur.collect_fleet_versions(pre_restart_pids=[ancestor, sibling])
|
||||
assert {row["state"] for row in plain} == {"stale"}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user