diff --git a/gateway/restart.py b/gateway/restart.py index 57732284c3..8e31ab713b 100644 --- a/gateway/restart.py +++ b/gateway/restart.py @@ -355,7 +355,9 @@ def resolve_systemd_timeout_stop_sec( return int(max(_seconds(floor_s), max(drain, cron_budget) + _seconds(headroom_s))) -def resolve_restart_exit_wait_budget(drain_timeout: float, after_turn_timeout: float, *, headroom: float = 15.0) -> float: +def resolve_restart_exit_wait_budget( + drain_timeout: float, after_turn_timeout: float, cron_drain_timeout: float = 0.0, *, headroom: float = 15.0, +) -> float: """Seconds a CLI should wait for the gateway PID to exit after SIGUSR1: in-band restart may - defer ``stop()`` until turns finish, then spend ``drain_timeout`` inside it — cover both.""" - return _seconds(drain_timeout) + _seconds(after_turn_timeout) + _seconds(headroom) + defer ``stop()`` until turns finish, then spend the longer chat or cron drain inside it.""" + return max(_seconds(drain_timeout), _seconds(cron_drain_timeout)) + _seconds(after_turn_timeout) + _seconds(headroom) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index a210a213de..1d379af8a5 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -3561,14 +3561,11 @@ def _get_cron_drain_timeout() -> float: def _get_restart_exit_wait_budget() -> float: """CLI wait for gateway exit after SIGUSR1 / self-restart (#77184).""" return resolve_restart_exit_wait_budget( - # TimeoutStopSec must cover the full stop budget, not just restart_drain_timeout. Cron work can - # legally wait cron_drain_timeout plus cleanup reserve before interrupt/teardown, and systemd - # SIGKILLs if the unit's deadline is shorter (#94759). 30s of post-drain headroom is preserved on - # top, with a 60s floor. _get_restart_drain_timeout(), _agent_timeout_setting( "HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout", parse_restart_after_turn_timeout ), + _get_cron_drain_timeout(), ) diff --git a/hermes_cli/update_receipt.py b/hermes_cli/update_receipt.py index 65a163398b..c1d6ddf466 100644 --- a/hermes_cli/update_receipt.py +++ b/hermes_cli/update_receipt.py @@ -682,8 +682,8 @@ def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool: print(f" {line}") if RESTART_PENDING_STATE in states: print() - print(" ℹ A restart-pending gateway picks up the new code as soon as this update exits;") - print(" verify afterwards with `hermes gateway status`.") + print(" ℹ A restart-pending gateway has not yet been verified on the new code;") + print(" check after this update exits with `hermes gateway status`.") stale_or_down = sum(1 for entry in fleet if entry.get("state") in ("stale", "down")) if stale_or_down: print() diff --git a/tests/gateway/test_restart_after_turn.py b/tests/gateway/test_restart_after_turn.py index e0f2e5b1de..dd06e69515 100644 --- a/tests/gateway/test_restart_after_turn.py +++ b/tests/gateway/test_restart_after_turn.py @@ -22,9 +22,25 @@ def test_parse_restart_after_turn_timeout_defaults_and_clamps(): def test_resolve_restart_exit_wait_budget_covers_both_phases(): assert resolve_restart_exit_wait_budget(0, 0, headroom=15) == 15.0 assert resolve_restart_exit_wait_budget(180, 21600, headroom=15) == 180 + 21600 + 15 + assert resolve_restart_exit_wait_budget(2, 3, 80, headroom=15) == 3 + 80 + 15 + assert resolve_restart_exit_wait_budget(80, 3, 2, headroom=15) == 3 + 80 + 15 + assert resolve_restart_exit_wait_budget(2, 3, 0, headroom=15) == 3 + 2 + 15 assert resolve_restart_exit_wait_budget("bad", "bad", headroom="x") == 0.0 +def test_cli_restart_wait_covers_configured_cron_drain(tmp_path, monkeypatch): + import hermes_cli.gateway as gateway_cli + + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + for key in ("HERMES_RESTART_DRAIN_TIMEOUT", "HERMES_RESTART_AFTER_TURN_TIMEOUT", "HERMES_CRON_DRAIN_TIMEOUT"): + monkeypatch.delenv(key, raising=False) + (tmp_path / "config.yaml").write_text( + "agent:\n restart_drain_timeout: 2\n restart_after_turn_timeout: 3\n cron_drain_timeout: 80\n", + encoding="utf-8", + ) + assert gateway_cli._get_restart_exit_wait_budget() >= 3 + 80 + 15 + + def test_load_restart_after_turn_timeout_preserves_zero(tmp_path, monkeypatch): """Config/env ``0`` must disable after-turn wait, not fall back to default.""" import gateway.run as gateway_run diff --git a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py index 1a09291130..7ecbaccd92 100644 --- a/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py +++ b/tests/hermes_cli/test_fleet_matrix_self_restart_pending.py @@ -55,8 +55,10 @@ def test_ancestor_with_accepted_self_restart_is_pending_while_other_stale_rows_s # Without the sibling, the pending row alone is not a failure — and the same pid NOT recorded # as pending (the restart phase never reached the ancestor branch) is a plain stale verdict. only_ancestor = [row for row in fleet if row["profile"] == "default"] - with contextlib.redirect_stdout(io.StringIO()): + with contextlib.redirect_stdout(io.StringIO()) as pending_out: assert ur.print_fleet_version_matrix(only_ancestor) is False + assert "as soon as this update exits" not in pending_out.getvalue() + assert "restart pending" in pending_out.getvalue() plain = ur.collect_fleet_versions(pre_restart_pids=[ancestor, sibling]) assert {row["state"] for row in plain} == {"stale"}