940f374a01
The agent cache's dead-session guard resolved the store for the cached snapshot's session id through the routing index. Once the #54878 self-heal has moved the routing key on, that id is in neither the index nor the owner hints, so the lookup fell back to the launch home's store. Under a multiplexed gateway a routed profile's row is not there: the ended session read as live, the stale agent was reused, and every message started an empty session while the reply was appended to the ended one. The cache is keyed by session_key, so the snapshot's row lives in that key's profile store. Resolve the store by the key there. Callers that pass only an id resolve as before. Fixes #118862
252 lines
12 KiB
Python
252 lines
12 KiB
Python
"""SessionStore explicit suspension, crash-recovery markers, pruning and shared clock/id helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import TYPE_CHECKING, Optional
|
|
|
|
from hermes_state_ids import new_session_id
|
|
|
|
if TYPE_CHECKING:
|
|
from gateway.session import SessionEntry, SessionSource
|
|
|
|
# Log-record parity with the origin module.
|
|
logger = logging.getLogger("gateway.session")
|
|
|
|
|
|
def _now() -> datetime:
|
|
"""Return the current local time."""
|
|
return datetime.now()
|
|
|
|
|
|
def _new_session_id(now: datetime) -> str:
|
|
return new_session_id(now, hex_len=8)
|
|
|
|
|
|
def _iso(dt: Optional[datetime]) -> Optional[str]:
|
|
return dt.isoformat() if dt else None
|
|
|
|
|
|
def _parse_iso(value) -> Optional[datetime]:
|
|
"""``datetime.fromisoformat`` that returns None for empty/malformed input."""
|
|
if not value:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
# Auto-continue freshness window (1 hour) after the ``resume_pending`` mark; ``gateway/run.py``
|
|
# bridges config.yaml ``agent.gateway_auto_continue_freshness`` into the env var at startup.
|
|
_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT = 60 * 60
|
|
|
|
|
|
def auto_continue_freshness_window() -> float:
|
|
"""Resume-scheduler freshness window; stale automation never discards the transcript."""
|
|
raw = os.environ.get("HERMES_AUTO_CONTINUE_FRESHNESS")
|
|
try:
|
|
return float(raw) if raw else float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT)
|
|
except (TypeError, ValueError):
|
|
return float(_AUTO_CONTINUE_FRESHNESS_SECS_DEFAULT)
|
|
|
|
|
|
class SessionLifecycleMixin:
|
|
"""SessionStore explicit boundaries and crash-recovery markers."""
|
|
|
|
def _is_session_ended_in_db(self, session_id: str, session_key: Optional[str] = None) -> bool:
|
|
"""True iff state.db says the session is gone: ended (non-null end_reason) or hard-deleted
|
|
(no row in a readable owning DB). No DB or a DB error -> False (same failure mode as
|
|
``_prune_stale_sessions_locked``). Lets routing self-heal a session finalized or deleted
|
|
while the gateway stays alive. Store resolved from the owning profile.
|
|
|
|
Used by ``get_or_create_session`` to self-heal at routing time: ``_prune_stale_sessions_locked``
|
|
only runs at startup, so a session ended in the DB while the gateway stays alive (any path that
|
|
finalizes the row without clearing sessions.json) would otherwise be reused as a live routing key
|
|
and silently swallow every subsequent message until the next restart (#54878 — the live-gateway
|
|
variant of #52804/FM9). A hard delete is the same shape one step further: the row is GONE, not
|
|
merely ended, and reusing the route makes run_agent's INSERT OR IGNORE resurrect the deleted
|
|
session with its old id (#42422) — so a missing row is treated exactly like an ended one. DB
|
|
errors are non-fatal — never block routing on a failed lookup.
|
|
The store is resolved from the row's owning profile rather than the ambient scope: an unscoped
|
|
background writer keeps its own copy of the same session, and comparing against that copy reports a
|
|
live session as ended (#66887).
|
|
|
|
Pass *session_key* when the owning key is known but the id may have left the routing index:
|
|
after the self-heal re-homes the key, the id has no owner and would resolve to the launch
|
|
store, which never holds a routed profile's rows (#118862).
|
|
"""
|
|
db = self._db_for_key(session_key) if session_key else self._db_for_session_id(session_id)
|
|
if not db or not session_id:
|
|
return False
|
|
try:
|
|
row = db.get_session(session_id)
|
|
except Exception:
|
|
return False
|
|
return row is None or row.get("end_reason") is not None
|
|
|
|
def _route_reset_reason(self, entry: SessionEntry) -> Optional[str]:
|
|
"""Only explicit suspension replaces a routed conversation; time never does."""
|
|
return "suspended" if entry.suspended else None
|
|
|
|
def _update_entry(self, session_key: str, mutate) -> bool:
|
|
"""Apply ``mutate(entry)`` under ``_lock`` and full-save; False when the entry is missing
|
|
or *mutate* returned False (nothing to persist)."""
|
|
with self._lock:
|
|
entry = self._entry_locked(session_key)
|
|
if entry is None or mutate(entry) is False:
|
|
return False
|
|
self._save()
|
|
return True
|
|
|
|
def _update_all_entries_locked(self, mutate) -> int:
|
|
"""Apply ``mutate(entry) -> bool`` to every entry under ``_lock``; save once if any
|
|
returned True. Returns the count that did."""
|
|
with self._lock:
|
|
self._ensure_loaded_locked()
|
|
changed = sum(1 for entry in self._entries.values() if mutate(entry))
|
|
if changed:
|
|
self._save()
|
|
return changed
|
|
|
|
def suspend_session(self, session_key: str) -> bool:
|
|
"""Mark a session suspended so it auto-resets on next access (/stop). True if it existed.
|
|
|
|
Used by ``/stop`` to prevent stuck sessions from being resumed after a gateway restart (#7536).
|
|
"""
|
|
return self._update_entry(session_key, lambda e: setattr(e, "suspended", True))
|
|
|
|
def _set_turn_marker_locked(self, session_key: str, entry: SessionEntry, token, started_at) -> None:
|
|
"""Persist the active-turn pair BEFORE publishing it in memory, so a failed write can
|
|
neither leak an unowned token nor drop a live one. Lock held."""
|
|
candidate = entry.to_dict()
|
|
candidate["active_turn_token"] = token
|
|
candidate["active_turn_started_at"] = _iso(started_at)
|
|
touched = _now() if started_at is not None else None
|
|
if touched is not None:
|
|
# Keeps the legacy 120s startup heuristic working for an older binary during a rolling
|
|
# downgrade/upgrade window.
|
|
candidate["updated_at"] = touched.isoformat()
|
|
self._save_entry(session_key, entry_data=candidate, lock_held=True)
|
|
entry.active_turn_token = token
|
|
entry.active_turn_started_at = started_at
|
|
if touched is not None:
|
|
entry.updated_at = touched
|
|
|
|
def mark_turn_active(self, session_key: str) -> Optional[str]:
|
|
"""Persist exact ownership of the running agent turn; returns the opaque token for
|
|
:meth:`clear_turn_active`. Re-marking replaces the previous token so a stale asynchronous
|
|
unwind cannot clear a newer turn."""
|
|
token = uuid.uuid4().hex
|
|
with self._lock:
|
|
entry = self._entry_locked(session_key)
|
|
if entry is None:
|
|
return None
|
|
# Aware UTC, unlike the local wall clock elsewhere: the next process compares it with
|
|
# epoch transcript timestamps and may run in another zone (DST, container vs unit TZ).
|
|
self._set_turn_marker_locked(session_key, entry, token, datetime.now(timezone.utc))
|
|
return token
|
|
|
|
def clear_turn_active(self, session_key: str, token: str) -> bool:
|
|
"""Compare-and-swap clear an active-turn marker; ``False`` when the entry disappeared or a
|
|
newer turn owns it."""
|
|
with self._lock:
|
|
entry = self._entry_locked(session_key)
|
|
if entry is None or entry.active_turn_token != token:
|
|
return False
|
|
self._set_turn_marker_locked(session_key, entry, None, None)
|
|
return True
|
|
|
|
def recover_interrupted_turns(self, max_age_seconds: int = 60 * 60) -> int:
|
|
"""Promote crash-left turn markers into ``resume_pending`` (unclean startup only).
|
|
Old/invalid markers are cleared without resuming; suspended sessions are never re-armed.
|
|
Returns the number of newly promoted sessions."""
|
|
now, epoch_now = _now(), time.time()
|
|
promoted = 0
|
|
|
|
def _promote(entry: SessionEntry) -> bool:
|
|
nonlocal promoted
|
|
if not entry.active_turn_token:
|
|
return False
|
|
started_at = entry.active_turn_started_at
|
|
# Epoch arithmetic: a pre-upgrade naive marker reads as local time, an aware one exactly.
|
|
marker_is_stale = started_at is None or (
|
|
max_age_seconds > 0 and epoch_now - started_at.timestamp() > max_age_seconds
|
|
)
|
|
if not marker_is_stale and not entry.suspended:
|
|
if entry.resume_pending:
|
|
# A drain-timeout marker is more specific; keep it.
|
|
if entry.last_resume_marked_at is None:
|
|
entry.last_resume_marked_at = now
|
|
else:
|
|
entry.resume_pending = True
|
|
entry.resume_reason = "restart_interrupted"
|
|
entry.last_resume_marked_at = now # freshness starts at discovery
|
|
promoted += 1
|
|
entry.active_turn_token = None
|
|
entry.active_turn_started_at = None
|
|
return True
|
|
|
|
self._update_all_entries_locked(_promote)
|
|
return promoted
|
|
|
|
def discard_active_turn_markers(self) -> int:
|
|
"""Clear orphan turn markers after a verified clean shutdown."""
|
|
def _discard(entry: SessionEntry) -> bool:
|
|
if not entry.active_turn_token and entry.active_turn_started_at is None:
|
|
return False
|
|
entry.active_turn_token = None
|
|
entry.active_turn_started_at = None
|
|
return True
|
|
return self._update_all_entries_locked(_discard)
|
|
|
|
def mark_resume_pending(self, session_key: str, reason: str = "restart_timeout") -> bool:
|
|
"""Mark a session resumable after a restart interruption (keeps the session_id/transcript,
|
|
unlike ``suspend_session``). True if marked."""
|
|
def _apply(entry: SessionEntry):
|
|
if entry.suspended: # never override an explicit ``suspended`` (hard forced-wipe)
|
|
return False
|
|
entry.resume_pending = True
|
|
entry.resume_reason = reason
|
|
entry.last_resume_marked_at = _now()
|
|
return self._update_entry(session_key, _apply)
|
|
|
|
def clear_resume_pending(self, session_key: str) -> bool:
|
|
"""Clear the resume-pending flag after a successful resumed turn; True if cleared."""
|
|
def _apply(entry: SessionEntry):
|
|
if not entry.resume_pending:
|
|
return False
|
|
entry.resume_pending = False
|
|
entry.resume_reason = None
|
|
entry.last_resume_marked_at = None
|
|
return self._update_entry(session_key, _apply)
|
|
|
|
def prune_old_entries(self, max_age_days: int) -> int:
|
|
"""Drop routing entries idle (by ``updated_at``) for more than max_age_days; suspended
|
|
entries and entries with active background processes are kept. Only the key -> session_id
|
|
mapping is dropped (the transcript stays). ``max_age_days <= 0`` disables. Returns count."""
|
|
if max_age_days is None or max_age_days <= 0:
|
|
return 0
|
|
cutoff = _now() - timedelta(days=max_age_days)
|
|
with self._lock:
|
|
self._ensure_loaded_locked()
|
|
removed_keys = [
|
|
key for key, entry in list(self._entries.items())
|
|
if not entry.suspended
|
|
# The callback is keyed by session_key, NOT session_id.
|
|
and not self._has_active_processes_safe(entry.session_key, context="prune")
|
|
and entry.updated_at < cutoff
|
|
]
|
|
for key in removed_keys:
|
|
self._entries.pop(key, None)
|
|
if removed_keys:
|
|
self._save()
|
|
if removed_keys:
|
|
logger.info("SessionStore pruned %d entries older than %d days",
|
|
len(removed_keys), max_age_days)
|
|
return len(removed_keys)
|