48803623b4
launchctl kickstart -k re-runs whatever definition launchd already holds. After an update that definition was generated by the OLD checkout, so the update's final restart faithfully revived a stale service; when that stale job was wedged, kickstart hung until its 90s timeout and the update reported 'partial' - 'Could not restart the gateway (TimeoutExpired)' - on an install whose code, deps and builds had all succeeded (observed in an update rehearsal on a real macOS host). launchd_restart now calls refresh_launchd_plist_if_needed() first (the same repair 'hermes gateway start' performs): a stale plist is rewritten and bootout/bootstrapped before anything kickstarts. Because refresh_launchd_plist_if_needed returns False for 'nothing to do' as well as 'reload failed', the wedged-launchd revival path is gated on plist-present AND still-stale, so a missing/current plist keeps the ordinary kickstart flow (pinned by the existing unloaded-job noise test). Regression tests cover the refresh-before-kickstart ordering and the failed-refresh -> bounded-bootstrap route.
97 lines
4.1 KiB
Python
97 lines
4.1 KiB
Python
"""``launchd_restart`` must never kickstart a stale service definition.
|
|
|
|
``launchctl kickstart -k`` re-runs whatever definition launchd already holds.
|
|
After an update that definition was generated by the OLD checkout, so the
|
|
update's final restart faithfully revived a stale service; when that stale job
|
|
was also wedged, ``kickstart -k`` hung until its 90s timeout and the update
|
|
reported ``partial`` with "Could not restart the gateway" — on an install
|
|
whose code, deps and builds had all succeeded (observed in an update
|
|
rehearsal on a real macOS host, Sep 2026).
|
|
|
|
``refresh_launchd_plist_if_needed()`` (the ``hermes gateway start`` /
|
|
``gateway install`` repair path) already knows how to rewrite a stale plist
|
|
and bootout/bootstrap it; the contract pinned here is that ``launchd_restart``
|
|
runs it BEFORE any kickstart, and that a refresh which could not re-register
|
|
the job routes into the bounded bootstrap path instead of the unbounded
|
|
kickstart.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import hermes_cli.gateway as gateway_cli
|
|
|
|
|
|
@pytest.fixture
|
|
def launchd_seam(monkeypatch, tmp_path):
|
|
"""Neutralize process-side effects; record every launchctl invocation."""
|
|
calls = []
|
|
plist_path = tmp_path / "ai.hermes.gateway.plist"
|
|
plist_path.write_text("<plist>whatever</plist>", encoding="utf-8")
|
|
|
|
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
|
|
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
|
|
monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: None)
|
|
monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_graceful_restart_via_sigusr1", lambda pid, timeout, **_: False
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_wait_for_launchd_service_pid",
|
|
lambda label, old_pid, timeout=10.0, *, domain: False,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_launchd_unsupported_marker_exists",
|
|
lambda: False,
|
|
)
|
|
|
|
def fake_run(cmd, check=False, timeout=None, **kwargs):
|
|
calls.append(cmd)
|
|
if check and cmd[0] == "launchctl" and cmd[1] == "kickstart" and "-k" in cmd:
|
|
# The wedged-launchctl shape: kickstart never returns.
|
|
raise gateway_cli.subprocess.TimeoutExpired(cmd, 90)
|
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
|
|
|
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
|
|
return calls
|
|
|
|
|
|
def test_stale_plist_is_refreshed_before_any_kickstart(monkeypatch, launchd_seam):
|
|
"""The update-restart path must repair the definition, not revive a stale one."""
|
|
refreshed = []
|
|
monkeypatch.setattr(
|
|
gateway_cli, "refresh_launchd_plist_if_needed", lambda: refreshed.append(1) or True
|
|
)
|
|
# Even when the kickstart later wedges (TimeoutExpired escapes to the
|
|
# caller, exactly as on the broken host), the refresh must already have
|
|
# run — that ordering is the contract.
|
|
with pytest.raises(gateway_cli.subprocess.TimeoutExpired):
|
|
gateway_cli.launchd_restart()
|
|
|
|
assert refreshed == [1], "refresh must run before the restart decision"
|
|
kickstarts = [c for c in launchd_seam if c[:2] == ["launchctl", "kickstart"]]
|
|
assert kickstarts, "a current definition still needs the kickstart relaunch"
|
|
|
|
|
|
def test_failed_refresh_routes_to_bounded_bootstrap_not_the_90s_kickstart(monkeypatch, launchd_seam):
|
|
"""When the refresh couldn't re-register the job, kickstart would hang again.
|
|
|
|
The bootstrap path (30s timeouts, loud failure) is the bounded revival.
|
|
"""
|
|
monkeypatch.setattr(gateway_cli, "refresh_launchd_plist_if_needed", lambda: False)
|
|
|
|
gateway_cli.launchd_restart()
|
|
|
|
kickstarts = [c for c in launchd_seam if "kickstart" in c]
|
|
assert not [c for c in kickstarts if "-k" in c], (
|
|
"kickstart -k after a failed refresh is the exact 90s hang this guards against"
|
|
)
|
|
bootstraps = [c for c in launchd_seam if c[:2] == ["launchctl", "bootstrap"]]
|
|
assert bootstraps, "the bounded bootstrap revival must run instead"
|