fix(gateway): hold startup-watchdog progress leases across state.db auto-maintenance

Construction-time maybe_auto_archive / maybe_auto_prune_and_vacuum ran
synchronously with no report_startup_progress lease. A multi-minute
VACUUM of a large state.db accrues near-zero CPU, so the startup
watchdog misread it as a parked deadlock and killed the attempt with
exit 75, live-locking gateway restarts. Renew the lease per long step
(prune, orphan sweep, VACUUM, archive) since leases clamp at 900 s,
plus one lease around the gateway maintenance block.

Fixes #111092
This commit is contained in:
Kevin Rajan
2026-09-14 13:55:24 -05:00
committed by Teknium
parent 5358780264
commit cef69276ae
4 changed files with 94 additions and 0 deletions

View File

@@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
from agent.session_activity import (
ActivityProvenance, bound_activity_description, normalize_activity_provenance,
)
from hermes_startup_watchdog import report_startup_progress
from hermes_state_common import (
_LISTABLE_CHILD_SQL, _PREVIEW_ELIGIBLE_SQL, _PREVIEW_RAW_SELECT, _RECOVERABLE_END_REASONS,
_RECOVERABLE_END_REASONS_SQL, _RESET_END_REASONS, _legacy_reset_child_sql, _shape_preview,
@@ -1614,6 +1615,10 @@ class SessionSessionsMixin:
if last and now - last < min_interval_hours * 3600:
result["skipped"] = True
return result
# Startup-watchdog lease: the archive sweep is I/O-bound (near-zero CPU),
# which the watchdog's CPU fallback misreads as a parked deadlock.
# No-op when the watchdog is not armed; never raises.
report_startup_progress(900.0, phase="state_db_auto_archive")
archived = result["archived"] = self.archive_stale_sessions(idle_days, exclude_pinned=exclude_pinned)
# Record even a zero-archive run so we don't re-sweep every call.
self.set_meta("last_auto_archive", str(now))