1e643524cb
`gateway/run_heartbeat_restore.py::_scope_key` re-implemented the branch condition of `GatewayRunner._profile_scope_for_source` (multiplex check + `_resolve_profile_home_for_source`). If the standalone branch ever became source-dependent the restore grouping would silently merge sessions that need distinct scopes. Add `GatewayRunner._profile_scope_key_for_source` (home-key string, or None for the standalone launch scope) as the single owner of that condition; `_profile_scope_for_source` and the restore grouping both call it. Behaviour-neutral. PROOF: `grep -n _profile_scope_key_for_source gateway/run_turn.py gateway/run_heartbeat_restore.py` shows the definition plus both callers and `_scope_key` is gone (asserted by real import); `scripts/run_tests.sh tests/gateway/test_heartbeat_watch_restore.py` 4 passed (its `_profile_scope_for_source` counter still sees one entry per scope); ruff + check-windows-footguns clean.
80 lines
4.2 KiB
Python
80 lines
4.2 KiB
Python
"""Recover heartbeat watches from the gateway's canonical persisted routing index."""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger("gateway.run")
|
|
|
|
|
|
def _watched_homes(runner, default_home) -> list:
|
|
"""Every home the sweep's ``_profile_scope_for_source`` can resolve an origin to: the gateway home
|
|
plus, under multiplex, the whole served set INCLUDING ``default`` — a ``-p work`` multiplexer's own
|
|
home is not ``~/.hermes``, so dropping ``default`` here would hide its heartbeats forever."""
|
|
from gateway.run import _multiplex_profile_homes
|
|
|
|
homes = [default_home]
|
|
if getattr(getattr(runner, "config", None), "multiplex_profiles", False):
|
|
homes += [home for _name, home in _multiplex_profile_homes(runner.config)]
|
|
return list(dict.fromkeys(Path(home) for home in homes))
|
|
|
|
|
|
async def restore_heartbeat_watches(runner) -> None:
|
|
"""Retryable startup/poll scan; failed reads never prune existing watches.
|
|
|
|
SessionStore owns one routing index across profiles. Its origin and exact key,
|
|
rather than a second heartbeat routing snapshot, also cover pre-upgrade state.
|
|
Run all storage work off-loop so a cold profile DB cannot block adapters.
|
|
"""
|
|
from gateway.run import _profile_runtime_scope
|
|
from gateway.run_idle_gates import profile_has_active_heartbeat
|
|
from hermes_cli.heartbeat import HeartbeatManager
|
|
from hermes_constants import get_hermes_home
|
|
|
|
store = runner.session_store
|
|
|
|
def scan():
|
|
restored = []
|
|
# The poller may have been spawned by a named profile's /heartbeat command.
|
|
# Anchor even default origins to the gateway home, not inherited context.
|
|
home = getattr(store, "_routing_home", None) or get_hermes_home()
|
|
# Cheap gate: with no heartbeat persisted in any served profile there is nothing to
|
|
# restore — skip the per-origin profile-scope re-parse over every routed session.
|
|
if not any(profile_has_active_heartbeat(h) for h in _watched_homes(runner, home)):
|
|
return restored
|
|
with _profile_runtime_scope(home):
|
|
# Enter each profile's scope once per scan, not once per routed session: a scope entry
|
|
# hydrates the secret scope and terminal policy, so N sessions cost N parses otherwise.
|
|
# Sources are read through _restored_source so the persisted receiving bot is re-pinned
|
|
# before the scope key is derived; the same source object is what gets registered.
|
|
by_scope: dict = {}
|
|
for entry in store.list_sessions():
|
|
if entry.origin is None or not entry.session_id or entry.suspended:
|
|
continue
|
|
try:
|
|
source = runner._restored_source(entry)
|
|
by_scope.setdefault(runner._profile_scope_key_for_source(source), []).append((entry, source))
|
|
except Exception:
|
|
logger.debug("heartbeat restore for %s failed", entry.session_key, exc_info=True)
|
|
for group in by_scope.values():
|
|
try:
|
|
with runner._profile_scope_for_source(group[0][1]):
|
|
for entry, source in group:
|
|
try:
|
|
if HeartbeatManager(entry.session_id).is_active():
|
|
restored.append((entry.session_key, source, entry.session_id))
|
|
except Exception:
|
|
logger.debug("heartbeat restore for %s failed", entry.session_key, exc_info=True)
|
|
except Exception:
|
|
logger.debug("heartbeat restore scope for %s failed", group[0][0].session_key, exc_info=True)
|
|
return restored
|
|
|
|
try:
|
|
candidates = await runner._run_in_executor_with_context(scan)
|
|
for key, source, session_id in candidates:
|
|
# A reset/compression may have published a new owner during the executor hop.
|
|
if store.peek_session_id(key) == session_id:
|
|
runner._register_heartbeat_watch(key, source, session_id)
|
|
except Exception:
|
|
logger.debug("heartbeat restore scan failed; retrying on next poll", exc_info=True)
|