diff --git a/gateway/run.py b/gateway/run.py index 2ff17be402..59a9afab97 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -3602,6 +3602,12 @@ class GatewayRunner( if self._session_db is not None: try: from hermes_cli.config import load_config as _load_full_config + from hermes_startup_watchdog import report_startup_progress as _report_startup_progress + # Startup-watchdog lease (#111092): construction-time maintenance is I/O-bound + # with near-zero CPU, which the watchdog misreads as a parked deadlock. The + # maintenance functions renew per long step; this covers the block as a whole. + # No-op when the watchdog is not armed; never raises. + _report_startup_progress(900.0, phase="gateway_startup_state_maintenance") _sess_cfg = (_load_full_config().get("sessions") or {}) if _sess_cfg.get("auto_archive", False): self._session_db._db.maybe_auto_archive( diff --git a/hermes_state_maintenance.py b/hermes_state_maintenance.py index e3e86cab14..680a2d3fd6 100644 --- a/hermes_state_maintenance.py +++ b/hermes_state_maintenance.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple from hermes_state_common import ( AUTO_VACUUM_MIN_FREELIST_RATIO, _id_chunks, _placeholders, _sql_session_last_active, escape_like as _escape_like ) +from hermes_startup_watchdog import report_startup_progress # caplog tests pin the "hermes_state" logger name. logger = logging.getLogger("hermes_state") @@ -397,8 +398,14 @@ class SessionMaintenanceMixin: result["skipped"] = True return result # Prune first: orphans closed below get a full retention window. + # Startup-watchdog leases: each long step is I/O-bound (near-zero CPU), which the + # watchdog's CPU fallback misreads as a parked deadlock. Leases are clamped to + # _MAX_LEASE_S=900 per call, so a multi-minute step renews per step rather than + # once at entry. No-op when the watchdog is not armed; never raises. + report_startup_progress(900.0, phase="state_db_auto_prune") result["pruned"] = pruned = self.prune_sessions( older_than_days=retention_days, sessions_dir=sessions_dir, exclude_active_write_guards=True) + report_startup_progress(900.0, phase="state_db_auto_sweep") closed = self.sweep_orphaned_sessions( max_idle_seconds=float(retention_days) * 86400.0, sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, exclude_pinned=True, @@ -413,6 +420,9 @@ class SessionMaintenanceMixin: result["freelist_ratio"] = ratio = self._freelist_ratio() if ratio is None or ratio > min_vacuum_freelist_ratio: try: + # VACUUM rewrites every page with ~zero CPU: renew the lease here so + # a multi-minute rewrite on a large state.db never outlives the clamp. + report_startup_progress(900.0, phase="state_db_auto_vacuum") self.vacuum() result["vacuumed"] = True self.set_meta("last_vacuum", str(now)) diff --git a/hermes_state_sessions.py b/hermes_state_sessions.py index cfd7811fdf..aee9e0caa0 100644 --- a/hermes_state_sessions.py +++ b/hermes_state_sessions.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple from agent.session_activity import ( ActivityProvenance, bound_activity_description, normalize_activity_provenance, ) +from hermes_startup_watchdog import report_startup_progress from hermes_state_common import ( _LISTABLE_CHILD_SQL, _PREVIEW_ELIGIBLE_SQL, _PREVIEW_RAW_SELECT, _RECOVERABLE_END_REASONS, _RECOVERABLE_END_REASONS_SQL, _RESET_END_REASONS, _legacy_reset_child_sql, _shape_preview, @@ -1614,6 +1615,10 @@ class SessionSessionsMixin: if last and now - last < min_interval_hours * 3600: result["skipped"] = True return result + # Startup-watchdog lease: the archive sweep is I/O-bound (near-zero CPU), + # which the watchdog's CPU fallback misreads as a parked deadlock. + # No-op when the watchdog is not armed; never raises. + report_startup_progress(900.0, phase="state_db_auto_archive") archived = result["archived"] = self.archive_stale_sessions(idle_days, exclude_pinned=exclude_pinned) # Record even a zero-archive run so we don't re-sweep every call. self.set_meta("last_auto_archive", str(now)) diff --git a/tests/hermes_state/test_startup_maintenance_lease.py b/tests/hermes_state/test_startup_maintenance_lease.py new file mode 100644 index 0000000000..419cc999b8 --- /dev/null +++ b/tests/hermes_state/test_startup_maintenance_lease.py @@ -0,0 +1,73 @@ +"""Startup-watchdog progress leases across state.db auto-maintenance. + +Regression for #111092: gateway restart live-lock on large installs. The construction-time +maintenance block (auto-archive, auto-prune, orphan sweep, VACUUM) is I/O-bound with +near-zero CPU, which the startup watchdog misreads as a parked deadlock and kills with +exit 75. Each long synchronous step must hold a ``report_startup_progress`` lease; leases +are clamped to 900 s per call, so the lease is renewed per step rather than once at +entry. + +Invariant (not a change-detector): every long maintenance step reports a progress lease. +""" + +from __future__ import annotations + +from pathlib import Path + +from hermes_state import SessionDB + +import hermes_state_maintenance +import hermes_state_sessions + + +def _make_db(tmp_path: Path) -> SessionDB: + return SessionDB(db_path=tmp_path / "state.db") + + +def test_prune_sweep_and_vacuum_each_hold_a_lease(tmp_path, monkeypatch): + leases: list[str] = [] + monkeypatch.setattr( + hermes_state_maintenance, + "report_startup_progress", + lambda expected_s, phase="": leases.append(phase), + ) + db = _make_db(tmp_path) + try: + # Simulate a large-DB sweep where every step does long I/O-bound work. + monkeypatch.setattr(db, "prune_sessions", lambda **kw: 120) + monkeypatch.setattr(db, "sweep_orphaned_sessions", lambda **kw: []) + monkeypatch.setattr(db, "_freelist_ratio", lambda: 1.0) + monkeypatch.setattr(db, "vacuum", lambda: 0) + + result = db.maybe_auto_prune_and_vacuum( + retention_days=90, + min_interval_hours=0, + vacuum=True, + min_vacuum_interval_days=0, + ) + assert result["pruned"] == 120 + assert result["vacuumed"] is True + # A lease must be held across EACH long step: without them the watchdog + # fires exit 75 mid-sweep on a large state.db (#111092). + assert "state_db_auto_prune" in leases + assert "state_db_auto_sweep" in leases + assert "state_db_auto_vacuum" in leases + finally: + db.close() + + +def test_auto_archive_holds_a_lease(tmp_path, monkeypatch): + leases: list[str] = [] + monkeypatch.setattr( + hermes_state_sessions, + "report_startup_progress", + lambda expected_s, phase="": leases.append(phase), + ) + db = _make_db(tmp_path) + try: + monkeypatch.setattr(db, "archive_stale_sessions", lambda *a, **kw: 7) + result = db.maybe_auto_archive(idle_days=3, min_interval_hours=0) + assert result["archived"] == 7 + assert "state_db_auto_archive" in leases + finally: + db.close()