"""Session lifecycle: active-session slot leases, finalize/teardown/close, turn interrupt, WS-orphan reap scheduling, transport-scoped close. Bodies are rebound onto server.py's globals at install time (method_ctx.bind_module), so they reference server.py globals bare. """ from __future__ import annotations import logging import contextlib from .method_ctx import bind_module @contextlib.contextmanager def _session_turn_admission(session: dict): """Hold process admission until the history-locked running claim is visible to idle probes.""" from hermes_cli.backend_retirement import retirement with retirement.work() as admitted, session["history_lock"]: yield admitted def _start_session_work(target, *, name: str, session: dict | None = None): """Reserve before spawning; release only after the worker (including cleanup) has unwound.""" from agent.memory_provider import spawn_context_thread from hermes_cli.backend_retirement import retirement if not retirement.acquire(): return None def run(): try: target() finally: retirement.release() try: thread = spawn_context_thread(run, name=name) if session is not None: session["_run_thread"] = thread thread.start() return thread except BaseException: retirement.release() raise def _notify_session_boundary(event_type: str, session_id: str | None, platform: str | None = None) -> None: """Fire session lifecycle hooks with CLI parity.""" with contextlib.suppress(Exception): from hermes_cli.lifecycle import finalize_session, invoke_hook if event_type == "on_session_finalize": finalize_session(session_id=session_id, platform=_resolve_agent_platform(platform)) else: invoke_hook(event_type, session_id=session_id, platform=_resolve_agent_platform(platform)) _SESSION_OWNERSHIP_UNAVAILABLE = "Hermes could not safely reserve this session. Try again." _AUTOMATIC_SESSION_END_REASONS = frozenset({"ws_orphan_reap", "ws_disconnect", "idle_timeout", "lru_evict", "tui_shutdown"}) def _lease_metadata(live_session_id: str) -> dict: """Writer identity for a lease: ``live_session_id`` is half of ``_is_same_writer``; the delivery flag is what Bot Chat gates live delivery on (session_notifications).""" return {"live_session_id": live_session_id, "bot_live_delivery_consumer": True} def _claim_active_session_slot( session_key: str, *, live_session_id: str, surface: str = "tui", profile_home: str | Path | None = None ) -> tuple[Any, str | None]: try: from hermes_cli.active_sessions import try_acquire_active_session return try_acquire_active_session( session_id=session_key, surface=surface, config=_load_cfg(), registry_home=profile_home, metadata=_lease_metadata(live_session_id), track_liveness=str(surface or "").strip().lower() == "desktop") except Exception as exc: logger.warning("Failed to claim active session slot: %s", exc) # Fail CLOSED: an errored claim has NOT proven the session unowned; lease-less = silent double-writer hole. # Fail CLOSED regardless of surface: per-session exclusivity is a correctness guarantee (see # PER_SESSION_EXCLUSIVE_SUBMIT), and a claim that errors out has NOT proven the session is unowned. # Proceeding without a lease here is the silent double-writer hole flagged in the #94595 review # (blocker 2). return (None, _SESSION_OWNERSHIP_UNAVAILABLE) def _install_borrowed_lease(sid: str, session: dict, frame: dict) -> None: """Adopt the parent's registry slot as an INERT token on a compute-host child. An isolated turn runs in the compute-host CHILD process. The parent dashboard already claimed the session's active-session lease before routing the turn here, but the child's freshly built session record never carried it — so ``_admit_prompt_turn`` re-claimed from the child's pid and was fenced out by the parent's own entry (``_is_same_writer`` requires same pid AND same live_session_id): every isolated turn failed with "Session ... already has a live owner" (#101416). The slot is real and owned upstream, so the child must neither claim a second one nor be able to release/transfer the parent's: the token is ``enabled=False`` — ``release()`` is a no-op and ``transfer_active_session`` only retargets the token locally, so a compression rotation A->B inside the child never reaches the registry (the parent re-anchors its real lease from the reported ``session_key``, see ``_compute_host_adopt_frame_meta``). NOT ``released=True``: a released token makes the transfer fall through to a real registry claim under the child pid. Only installed when the frame's ``active_session_lease`` vouch names THIS stored session id — a parent lease still keyed on a pre-rotation id must not authorize its continuation. Anything else keeps the legacy behaviour — the child claims for itself and any ownership conflict fails CLOSED with the visible refusal. """ vouch = frame.get("active_session_lease") if not isinstance(vouch, dict) or session.get("active_session_lease") is not None: return key = str(session.get("session_key") or "") if not key or str(vouch.get("session_id") or "") != key: return from hermes_cli.active_sessions import ActiveSessionLease session["active_session_lease"] = ActiveSessionLease( lease_id=f"borrowed:{vouch.get('lease_id') or sid}", session_id=key, surface=str(frame.get("source") or "desktop"), enabled=False) def _ensure_active_session_slot(sid: str, session: dict) -> str | None: """Claim this session's cap slot on its first real turn; None when ok. session.create/resume deliberately do NOT claim: tile paints, reconnect-resumes and abandoned drafts would hold invisible slots (no DB row) that starve the messaging gateway sharing the cap. Anything holding a slot must be user-visible. An inert borrowed token (see _install_borrowed_lease) also lands here: present = slot held upstream.""" if session.get("active_session_lease") is not None: return None key = str(session.get("session_key") or "") lease, limit_message = _claim_active_session_slot( key, live_session_id=sid, surface=_session_source(session), profile_home=session.get("profile_home")) if limit_message is None: _attach_lease(session, lease) return None from hermes_cli.active_sessions import SESSION_NOT_OWNED if getattr(limit_message, "reason", None) == SESSION_NOT_OWNED and _take_over_detached_runtime_lease(sid, session, key): return None return limit_message def _attach_lease(session: dict, lease) -> None: session["active_session_lease"] = lease session.pop("_lease_taken_over", None) # owning again: its finalize ends the row like any owner def _detached_lease_holder(session: dict, key: str) -> tuple[str, dict] | None: """A sibling runtime of ``session`` (same key, same profile) holding an unreleased lease with no client. Caller holds ``_sessions_lock``.""" for other_sid, other in _sessions.items(): if (other is session or other.get("_finalized") or str(other.get("session_key") or "") != key or not _live_profile_matches(other, session.get("profile_home"))): continue lease = other.get("active_session_lease") if lease is not None and not lease.released and _transport_is_dead(other.get("transport")): return other_sid, other return None def _take_over_detached_runtime_lease(sid: str, session: dict, key: str) -> bool: """Hand a detached sibling runtime's lease for ``key`` to ``session``; True when it now owns the slot. Lease identity is (pid, live_session_id), so a client that reconnects under a NEW runtime — Desktop restoring a chat after a renderer crash, a reload during the client-gone settle window (4009 fences ``session.resume``) — is a "different writer" of its own chat and was refused ``SESSION_NOT_OWNED`` until the old runtime's reap released the lease (up to grace + activity-stale + interrupt polls). The user IS the owner: the old runtime is in THIS process and has no client, so the lease moves and its turn is asked to stop (hard interrupt, honoured at the agent's next check — the same best-effort stop the reaper relies on). A live foreign pid, a sibling that still has a client, or a same-id runtime of another profile keeps refusing — cross-process, multi-window and cross-profile (#100029) exclusivity are untouched. See #104691. """ from hermes_cli.active_sessions import transfer_active_session with _session_resume_lock, _sessions_lock: if (found := _detached_lease_holder(session, key)) is None: return False other_sid, other = found lease = other["active_session_lease"] if not transfer_active_session(lease, session_id=key, metadata=_lease_metadata(sid)): return False del other["active_session_lease"] other["_lease_taken_over"] = True _attach_lease(session, lease) logger.info("Session %s took over lease for %s from detached runtime %s", sid, key, other_sid) try: _interrupt_session_turn(other_sid, other, request_id=f"lease-takeover-{sid}") except Exception: logger.exception("lease-takeover interrupt failed sid=%s", other_sid) return True def _lease_retry(attempts: int, fn) -> Exception | None: """Call ``fn`` up to ``attempts`` times (50ms*n backoff: registry writes contend across processes); last exception when every try failed, else None.""" for attempt in range(attempts): try: fn() return None except Exception as exc: last_error = exc if attempt + 1 < attempts: time.sleep(0.05 * (attempt + 1)) return last_error def _release_active_session_slot(session: dict | None) -> bool: """Release the session's lease (liveness-tracked leases get 3 tries); True when released.""" lease = session.get("active_session_lease") if session else None if lease is None: return True if (err := _lease_retry(3 if getattr(lease, "track_liveness", False) else 1, lambda: lease.release())) is not None: logger.warning("Failed to release active session slot", exc_info=err) return False if not (getattr(lease, "released", True) or not getattr(lease, "enabled", True)): return False if session.get("active_session_lease") is lease: session.pop("active_session_lease", None) return True def _release_hosted_room_turn_slot(session: dict) -> None: """End-of-turn release for hosted room member sessions (``source=bot_room``) only. A hosted room turn is serialized by the room driver's lease, not by this process, so the member profile's ``bot_room`` slot is needed only while a turn is in flight. Holding it for the life of the live session (which no reaper ever ends: room sessions have no client transport) locked every other room worker sharing the home — messaging gateway + Desktop ``serve`` — out of that member with ``Refused active session … already held by pid=…`` until this process exited (#106847). Call under ``history_lock`` next to ``running = False`` so the next admission never observes the stale lease and then runs lease-less; ``_admit_prompt_turn`` re-claims on the following turn. """ from tui_gateway.hosted_room_driver import ROOM_SESSION_SOURCE if _session_source(session) == ROOM_SESSION_SOURCE: _release_active_session_slot(session) def _own_live_lease_ids(*, exclude=None) -> set[str]: """Snapshot leases still backed by this process's live session records (plus leases deferred past a close for an unsettled isolated turn — still ours until the child settles).""" with _sessions_lock: return {str(lease.lease_id) for session in _sessions.values() if (lease := session.get("active_session_lease")) is not None and lease is not exclude } | set(_deferred_active_session_leases) @contextlib.contextmanager def _other_runtime_lease_guard(session_id: str, session: dict): """Release this runtime and lock sibling ownership through the DB write. Yields True (another runtime owns the lifecycle -> preserve) when the guard can't be loaded/entered in 3 tries: unknown ownership never ends a row.""" lease = session.get("active_session_lease") try: from hermes_cli.active_sessions import active_session_liveness_guard, release_active_session_liveness_guard except Exception as exc: logger.warning("Failed to load active session ownership guard; preserving session %s: %s", session_id, exc) yield True return stack = contextlib.ExitStack() active: list = [] own_live_lease_ids = _own_live_lease_ids(exclude=lease) def _enter() -> None: stack.close() # drop anything a half-failed previous attempt left behind if lease is not None and getattr(lease, "enabled", False): guard = release_active_session_liveness_guard(lease, session_id, own_live_lease_ids=own_live_lease_ids) else: guard = active_session_liveness_guard( session_id, registry_home=session.get("profile_home"), own_live_lease_ids=own_live_lease_ids) active[:] = [stack.enter_context(guard)] if (last_error := _lease_retry(3, _enter)) is not None: stack.close() logger.warning("Failed to inspect active session leases; preserving session %s: %s", session_id, last_error) yield True return try: yield active[0] finally: stack.close() if lease is not None and getattr(lease, "released", False) and session.get("active_session_lease") is lease: session.pop("active_session_lease", None) def _transfer_active_session_slot(sid: str, session: dict, *, new_session_id: str) -> bool: if not new_session_id: return False lease = session.get("active_session_lease") if lease is None: return True try: from hermes_cli.active_sessions import transfer_active_session if transfer_active_session(lease, session_id=new_session_id, metadata=_lease_metadata(sid)): return True except Exception: logger.debug("Failed to transfer active session slot", exc_info=True) if getattr(lease, "track_liveness", False): return False # Fallback (entry pruned / pid-check transiently failed): reserve the new slot BEFORE releasing the old one so # a gateway at the cap can't grab the freed slot and leave this session lease-less; on failure KEEP the old lease. # See #49041. new_lease, limit_message = _claim_active_session_slot( new_session_id, live_session_id=sid, surface=_session_source(session), profile_home=session.get("profile_home")) if new_lease is None: if limit_message: logger.warning("Compression session lease re-anchor failed (kept old lease): sid=%s new_session_id=%s reason=%s", sid, new_session_id, limit_message) return False if (old := session.pop("active_session_lease", None)) is not None and (err := _lease_retry(1, old.release)): logger.debug("Failed to release stale active session slot", exc_info=err) session["active_session_lease"] = new_lease return True # Sources this backend must never end in state.db: the messaging gateway owns those sessions and the TUI is only # a viewer (ending one causes the Groundhog Day loop, see _finalize_session). Self-created/CLI sources are NOT gateway-owned. # Sources the TUI backend itself creates ("tui", plus whatever a client passes as its own ``source``) and # the CLI's own sessions are NOT gateway-owned. See #60609. _NON_GATEWAY_SOURCES = frozenset({ "", "tui", "cli", "webui", "desktop", "cron", "kanban", "subagent", "test", "local", "acp", "webhook", "api_server", "msgraph_webhook"}) def _is_gateway_owned_source(source: str) -> bool: """True when ``source`` resolves to a gateway ``Platform`` (enum member or plugin via ``Platform._missing_``, so new platforms are covered automatically); self-owned Platform members (local/webhook/api_server) are excluded.""" src = (source or "").strip().lower() if src in _NON_GATEWAY_SOURCES: return False try: from gateway.config import Platform Platform(src) # raises ValueError for arbitrary non-platform strings return True except Exception: return False def _lifecycle_own_sid(session: dict, sid_hint: str = "") -> str: """Live UI sid for ``session``: hint, stamped ``_sid``, else registry scan.""" own_sid = str(sid_hint or session.get("_sid") or "") if not own_sid: with contextlib.suppress(Exception), _sessions_lock: own_sid = next((cand_sid for cand_sid, cand in _sessions.items() if cand is session), "") return own_sid def _lock_vault_managers(session: dict) -> None: """A per-session unlock ends with the session that made it; siblings in the same profile keep theirs.""" try: from agent.vault_backends import unlock if sid := session.get("_sid"): unlock.release_session(sid) except Exception: logging.getLogger(__name__).debug("vault manager lock on session end failed", exc_info=True) def _finalize_session(session: dict | None, end_reason: str = "tui_close") -> None: """Best-effort finalize hook + memory commit; mirrors the CLI exit path so a force-quit mid-turn (double Ctrl-C, terminal close, SIGHUP) loses nothing.""" if not session or session.get("_finalized"): return session["_finalized"] = True _lock_vault_managers(session) if (history_ready := session.get("resume_history_ready")) is not None and not history_ready.is_set(): session["resume_history_error"] = "session resume cancelled" history_ready.set() _desktop_automatic_cleanup = ( end_reason in _AUTOMATIC_SESSION_END_REASONS and _session_source(session).strip().lower() == "desktop") # Automatic Desktop cleanup releases its lease inside the lifecycle guard below; other paths keep force/end semantics. if not _desktop_automatic_cleanup: _release_active_session_slot(session) if (stop_event := session.get("_notif_stop")) is not None: stop_event.set() agent = session.get("agent") with (session.get("history_lock") or contextlib.nullcontext()): history = list(session.get("history", [])) # Persist via ``_persist_session``'s marker-based dedup (gateway-shutdown flush contract). Do NOT pass # ``conversation_history``: ``session["history"]`` and ``_session_messages`` alias the SAME list after a turn, so # the flush would treat every message as durable and skip it — data loss when finalize is the sole persist path. if hasattr(agent, "_persist_session") and (snapshot := getattr(agent, "_session_messages", None)): with contextlib.suppress(Exception): agent._persist_session(snapshot) # interrupted=True so crash-recovery plugins can flush state (mirrors cli.py atexit). The end-of-session # hooks and the memory commit read the provider's config/credentials at call time; every caller of this # chokepoint is an unscoped reaper/Timer/atexit/pool thread, so bind the SESSION's profile here — unscoped # they fail closed under multiplex (tail never committed) or, on the Desktop backend serving a named # profile, commit a secondary's transcript to the launch profile's memory tenant (same class as #110622). with _session_profile_runtime_scope(session): if agent is not None: with contextlib.suppress(Exception): from hermes_cli.lifecycle import invoke_hook invoke_hook( "on_session_end", completed=False, interrupted=True, session_id=getattr(agent, "session_id", None) or session.get("session_key", ""), model=getattr(agent, "model", "unknown"), platform=getattr(agent, "platform", None) or "tui") if agent is not None and history and hasattr(agent, "commit_memory_session"): with contextlib.suppress(Exception): agent.commit_memory_session(history) session_key = session.get("session_key") session_id = getattr(agent, "session_id", None) or session_key _notify_session_boundary("on_session_finalize", session_id, _session_source(session)) # End the state.db row so it doesn't linger as a ghost in /resume. Use session_id (agent.session_id), not # session_key: after compression the key may be the stale ended parent while session_id is the live continuation. # Fix for #20001. if _desktop_automatic_cleanup and not session_id: _release_active_session_slot(session) # ``_lease_taken_over``: a new runtime for the same chat owns the row and its delegations now. _lifecycle_guard = (_other_runtime_lease_guard(session_id, session) if _desktop_automatic_cleanup and session_id else contextlib.nullcontext(bool(session.get("_lease_taken_over")))) with _lifecycle_guard as _other_runtime_owns_lifecycle: _tui_owns_lifecycle = not _other_runtime_owns_lifecycle if _other_runtime_owns_lifecycle: logger.info("Preserving session %s during %s: another backend owns an active lease", session_id, end_reason) if session_id: # The *session's* profile state.db (app-global remote mode), not the launch profile's. with contextlib.suppress(Exception), _session_db(session) as db: if db is not None: # Never end gateway-originated sessions: Groundhog Day loop (gateway self-heals to the parent, # compression splits back to the reaped child, forever). if _is_gateway_owned_source((db.get_session(session_id) or {}).get("source", "")): _tui_owns_lifecycle = False elif _tui_owns_lifecycle and not _desktop_automatic_cleanup: # Automatic Desktop cleanup (ws_orphan_reap, idle_timeout, etc.) reclaims # runtime but must not end the durable row — the conversation stays open # and resumable until the user explicitly closes or archives it. #105588 db.end_session(session_id, end_reason) # ``_teardown_session`` follows with ``agent.close()``, whose ``_finalize_owned_session_row`` ends the row as # ``agent_close`` through the agent's own handle — every spare decision above would be undone one call later. if agent is not None and (_desktop_automatic_cleanup or not _tui_owns_lifecycle): agent._end_session_on_close = False # In-flight async delegations end WITH the session (no return address left). Always interrupt by THIS live UI # sid; by durable session_key only when the TUI owns the lifecycle — a viewer tab must not kill gateway work. with contextlib.suppress(Exception): from tools.async_delegation import interrupt_for_session interrupt_for_session( session_key=str(session_key or "") if _tui_owns_lifecycle else "", origin_ui_session_id=_lifecycle_own_sid(session), reason=end_reason) # Session-persistent code kernels (execute_code) share this owner key and die at the same boundary, like the # gateway's /stop and /new (approval.clear_session); otherwise each finished conversation keeps a live # interpreter until kernel_idle_timeout. Only when the TUI owns the lifecycle: a viewer tab over a # gateway-owned session must not kill the gateway's kernels. if _tui_owns_lifecycle and session_key: with contextlib.suppress(Exception): from tools.approval import clear_session clear_session(str(session_key)) # Close the slash-worker in this single ``_finalized``-guarded chokepoint (a direct caller can't leak it); idempotent. with contextlib.suppress(Exception): if worker := session.get("slash_worker"): worker.close() # End reasons where the BACKEND reclaimed a session the client never asked to close (else its next prompt fails # against a forgotten id). Client-initiated reasons (``tui_close`` etc.) are deliberately absent. _RECLAIM_END_REASONS = frozenset({"idle_timeout", "lru_evict", "ws_orphan_reap"}) def _announce_session_reclaimed(session: dict, end_reason: str) -> None: """Tell connected clients a session was reclaimed out from under them. Broadcast, not session-targeted: reap paths run on timer threads with no contextvar binding and no live transport, so ``_emit`` would hit stdio.""" if end_reason not in _RECLAIM_END_REASONS: return try: _broadcast_global_event("session.reclaimed", { "session_id": str(session.get("_sid") or ""), "stored_session_id": str(session.get("session_key") or ""), "reason": end_reason}) except Exception: logger.debug("session.reclaimed broadcast failed", exc_info=True) def _teardown_session(session: dict | None, *, end_reason: str = "tui_close") -> None: """Fully tear down a session: finalize, unregister notifier, close agent (``session.close`` + WS reaper). The slash-worker is closed in ``_finalize_session`` (the single chokepoint), NOT here. Idempotent via ``_finalized``.""" if not session: return _finalize_session(session, end_reason=end_reason) _announce_session_reclaimed(session, end_reason) with contextlib.suppress(Exception): from tools.approval import unregister_gateway_notify # One approval callback per key: after a takeover it is the new runtime's registration. if (key := session.get("session_key")) and not session.get("_lease_taken_over"): unregister_gateway_notify(key) # agent.close() → shutdown_memory_provider reads the provider's config/credentials at call time; same # scope rule as _finalize_session (every caller here is an unscoped reaper/atexit/pool thread). with contextlib.suppress(Exception), _session_profile_runtime_scope(session): if hasattr(agent := session.get("agent"), "close"): agent.close() def _attach_worker(sid: str, session: dict, worker) -> None: """Store worker on session iff sid still maps to it, else close it (a concurrent teardown popped the session).""" with _sessions_lock: if _sessions.get(sid) is session: session["slash_worker"] = worker return worker.close() # Wall-clock timestamps, like session last_active; retained after close/reap. _closed_session_activity: dict[str, float] = {} def _pop_session_by_id(sid: str) -> dict | None: """Atomically detach one live session from the registry — the ownership claim for teardown (a concurrent close/reaper no-ops). Separate from ``_teardown_session``: slow finalization must not run under the resume lock.""" with _sessions_lock: session = _sessions.pop(sid, None) if session is not None: from hermes_constants import get_hermes_home home = str(Path(session.get("profile_home") or get_hermes_home()).resolve()) last_active = time.time() if session.get("running") else float(session.get("last_active") or 0) _closed_session_activity[home] = max(_closed_session_activity.get(home, 0), last_active) session["_closing"] = True session["_sid"] = sid # out of _sessions now, so teardown can't recover the live id by scanning return session def _teardown_popped_session(session: dict | None, *, end_reason: str = "tui_close") -> bool: """Finish a close after the caller has atomically detached the session.""" if session is None: return False run_thread = session.get("_run_thread") if end_reason != "tui_shutdown" and run_thread is not None and run_thread is not threading.current_thread(): try: if run_thread.is_alive(): run_thread.join(timeout=_TURN_SETTLE_BEFORE_CLOSE_SECONDS) if run_thread.is_alive(): logger.warning( "session turn thread still alive after %.1fs teardown grace", _TURN_SETTLE_BEFORE_CLOSE_SECONDS) except Exception: logger.debug("failed waiting for session turn thread", exc_info=True) if end_reason != "tui_shutdown": _settle_isolated_turn_before_close(session) _teardown_session(session, end_reason=end_reason) return True # lease_id -> REAL lease of a closed session whose isolated child turn has not settled yet. Still live # authority for the orphan sweep (``_own_live_lease_ids``); released by ``_release_deferred_active_session_lease``. _deferred_active_session_leases: dict[str, Any] = {} def _settle_isolated_turn_before_close(session: dict) -> None: """An isolated turn runs in the compute-host child, not on ``_run_thread``: interrupt it and give it the same close grace, and if it still has not settled keep the REAL lease out of finalize's release — the completion callback releases it on the correlated turn.end/turn.error (child death fails pending turns the same way). The RPC close is bounded; ownership ends with the child's last write, never with the grace timer, else a second backend acquires the stored session while the child is still writing.""" if not session.get("_compute_host_turn_id") or not _session_uses_compute_host(session): return with contextlib.suppress(Exception): _interrupt_session_turn(_lifecycle_own_sid(session), session) deadline = time.monotonic() + _TURN_SETTLE_BEFORE_CLOSE_SECONDS while session.get("_compute_host_turn_id") and time.monotonic() < deadline: time.sleep(0.05) with session["history_lock"]: if not session.get("_compute_host_turn_id") or (lease := session.pop("active_session_lease", None)) is None: return session["_deferred_active_session_lease"] = lease _deferred_active_session_leases[str(lease.lease_id)] = lease _deferred_active_session_lease_ages[str(lease.lease_id)] = time.time() logger.warning("isolated turn still live after %.1fs close grace; holding lease for %s until the child settles", _TURN_SETTLE_BEFORE_CLOSE_SECONDS, session.get("session_key")) # A deferred lease is released by the compute-host turn's completion callback # (_on_compute_host_turn_done → _release_deferred_active_session_lease). If that callback # is lost — supervisor restart/reload, a child killed without failing its pending turns, # a dropped completion — the lease sits in the registry FOREVER: _own_live_lease_ids # vouches for it, so the orphan sweep never reclaims it, and the concurrency cap treats # the dead session as active (#62823 zombie slot). A deferred lease past this generous # ceiling (the longest legitimate isolated turn is the compression ceiling, minutes) is # force-released by the reaper tick instead of leaking the slot until process exit. _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS = 1800.0 _deferred_active_session_lease_ages: dict[str, float] = {} def _reap_stale_deferred_leases(now: float | None = None) -> int: """Force-release deferred leases whose settlement callback never arrived. Returns the count.""" now = time.time() if now is None else now stale = [ lease_id for lease_id, deferred_at in list(_deferred_active_session_lease_ages.items()) if now - deferred_at > _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS ] reaped = 0 for lease_id in stale: _deferred_active_session_lease_ages.pop(lease_id, None) lease = _deferred_active_session_leases.pop(lease_id, None) if lease is None: continue if (err := _lease_retry(3, lease.release)) is not None: logger.warning("Failed to force-release stale deferred active session lease %s", lease_id, exc_info=err) continue logger.warning("Force-released deferred active session lease %s held past %.0fs without a " "compute-host settlement (zombie concurrency slot, #62823)", lease_id, _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS) reaped += 1 return reaped def _release_deferred_active_session_lease(session: dict) -> None: """Settlement half of ``_settle_isolated_turn_before_close``; a no-op for sessions that never deferred.""" lease = session.pop("_deferred_active_session_lease", None) if lease is None: return _deferred_active_session_leases.pop(str(lease.lease_id), None) _deferred_active_session_lease_ages.pop(str(lease.lease_id), None) if (err := _lease_retry(3, lease.release)) is not None: logger.warning("Failed to release deferred active session slot", exc_info=err) def _close_session_by_id( sid: str, *, end_reason: str = "tui_close", predicate: Callable[[dict], bool] | None = None) -> bool: """Idempotent teardown funnel for callers with no resume race (resume-sensitive callers pop under ``_session_resume_lock`` and call ``_teardown_popped_session`` after releasing it). Automatic reapers pass ``predicate`` to revalidate under ``_sessions_lock`` right before the claim, so a stale scan can't close a session that reattached.""" with _sessions_lock: # RLock: predicate + claim in one critical section current = _sessions.get(sid) if predicate is not None and (current is None or not predicate(current)): return False session = _pop_session_by_id(sid) return _teardown_popped_session(session, end_reason=end_reason) def _ws_session_is_detached(session: dict | None) -> bool: """True if a live session is still bound to the disconnected-WS sentinel.""" return bool(session and not session.get("_finalized") and session.get("transport") is _detached_ws_transport) def _ws_session_is_orphaned(session: dict | None) -> bool: """True if a WS session sits on ``_detached_ws_transport`` (where ``handle_ws`` parks disconnected clients), idle.""" return bool(_ws_session_is_detached(session) and not session.get("running")) def _interrupt_session_turn(sid: str, session: dict, *, request_id: str | None = None) -> bool: """Apply the shared ``session.interrupt`` contract to one claimed session; returns whether the compute-host control channel was used. The WS orphan reaper reuses this so a dead client gets the same partial-history/queue semantics.""" use_compute_host = _session_uses_compute_host(session) should_interrupt = bool(session.get("running")) run_thread_alive = False if use_compute_host: # The host owns the live turn (parent `running` can lag a blocked tool), so let it decide. Gate on # `_compute_host_active`: HostSupervisor.interrupt() calls start(), so a lazy session would spawn a child to interrupt. if should_interrupt or session.get("_compute_host_active"): _get_compute_host_supervisor().interrupt(sid, request_id=request_id) else: run_thread_alive = (rt := session.get("_run_thread")) is not None and rt.is_alive() with session["history_lock"]: session["_turn_cancel_requested"] = True session["queued_prompt"] = None session.pop("queued_prompts", None) session["_queued_prompt_generation"] = int(session.get("_queued_prompt_generation", 0)) + 1 if should_interrupt: # Sibling of gateway/run_agent_cache.py::_interrupt_and_clear_session: a user-initiated stop of a # live TUI/desktop turn is the same "loop is gone" event for plugins holding per-turn external # resources. Observer-only; dispatch failures never break the interrupt. try: from hermes_cli.plugins import invoke_hook as _invoke_hook _invoke_hook( "agent_loop_stopped", session_key=session.get("session_key", ""), platform="tui", reason="user_stop", invalidation_reason="session_interrupt", ) except Exception: logger.debug("agent_loop_stopped hook dispatch failed", exc_info=True) if not use_compute_host: if should_interrupt: from agent.interrupt_compat import request_hard_interrupt request_hard_interrupt(session.get("agent")) # Background delegations are detached from the turn's interrupt fan-out; a stop ends them too # (own UI sid + spawner id only — a viewer tab must not kill gateway work). Each returns as an # interrupted completion with its partial output. with contextlib.suppress(Exception): from tools.async_delegation import interrupt_for_session interrupt_for_session( origin_ui_session_id=_lifecycle_own_sid(session, sid), reason="user_stop", parent_session_id=str(getattr(session.get("agent"), "session_id", "") or "")) if not run_thread_alive: with session["history_lock"]: if session.get("running"): session["running"] = False _clear_inflight_turn(session) _clear_pending(sid) with contextlib.suppress(Exception): from tools.approval import resolve_gateway_approval resolve_gateway_approval(session["session_key"], "deny", resolve_all=True) return use_compute_host def _session_has_active_delegations(sid: str, session: dict | None = None) -> bool: """True when UI session ``sid`` still owns live background work — by live UI sid AND, when the TUI owns the durable lifecycle (never for gateway-viewer tabs), by session_key so a delegation from an earlier tab keeps it alive. See #60609. """ if session is None: with _sessions_lock: session = _sessions.get(sid) if not session: return False own_sid = _lifecycle_own_sid(session, sid) owned_session_key = session_key = str(session.get("session_key") or "") session_id = getattr(session.get("agent"), "session_id", None) or session_key if session_id: # Only when this session may end its durable row by key — never for gateway-originated sessions (TUI is a # viewer there). Unknown DB state -> assume ownership. # The row lives in the session's OWN store (a named-profile session's row is invisible to # the launch handle, which would leave this guard permanently dead). with contextlib.suppress(Exception), _session_db(session) as db: if db is not None and _is_gateway_owned_source((db.get_session(session_id) or {}).get("source", "")): owned_session_key = "" if not own_sid and not owned_session_key: return False try: from tools.async_delegation import has_live_for_session return has_live_for_session(session_key=owned_session_key, origin_ui_session_id=own_sid) except Exception: logger.debug("Failed to query active delegations for UI session %s", sid, exc_info=True) return True # a transient registry/import failure must not become destructive cleanup # One pending WS-orphan reap Timer per live sid; guarded by _sessions_lock. Cancelled by _cancel_ws_orphan_reap from # every resume/reuse/transport-rebind path — else a reap on a reattached session triggers a reap->broadcast->resume storm. _pending_ws_reaps: dict[str, threading.Timer] = {} def _cancel_ws_orphan_reap(sid: str) -> None: """Cancel a pending WS-orphan reap for ``sid`` (client came back). Called from every path that re-binds a live transport; closes the fired-but-not-run Timer race and stops dead Timers accumulating on flappy clients.""" with _sessions_lock: timer = _pending_ws_reaps.pop(sid, None) if timer is not None: with contextlib.suppress(Exception): timer.cancel() def _reattach_refusal(rid, sid: str, session: dict) -> dict | None: """Under ``_session_resume_lock``: why a reattaching RPC (resume/activate/prompt.submit) must NOT rebind ``session`` — it is stale, or a client-gone interrupt is still settling and the reap Timer must keep polling. None when the reattach may proceed.""" if _sessions.get(sid) is not session: return _err(rid, 4007, "session no longer live; retry resume") if session.get("_client_gone_interrupt_requested"): return _err(rid, 4009, "session disconnect interrupt settling") return None def _rebind_live_transport(sid: str, session: dict, transport: Transport) -> None: """Attach a live peer without displacing existing subscribers (caller holds ``history_lock``). Subagent control authority needs no bookkeeping here: it resolves against ``session["transport"]`` at RPC time (``tools.delegate_tool_registry._subagent_transport_matches``).""" if transport is not _detached_ws_transport and _transport_is_dead(transport): # The rebinding socket already closed: its disconnect cleanup ran before this late RPC (a # resume-then-drop burst), so nothing will detach it again. The client is NOT back — re-arm the # reap the caller cancelled instead of leaving a detached session with no Timer (#116464). with _sessions_lock: if _ws_session_is_detached(session) and sid not in _pending_ws_reaps: _schedule_ws_orphan_reap(sid) return _attach_session_transport(session, transport) # Every transport that showed this session (pop-outs resume the same sid); on disconnect the last # viewer becomes the transport instead of the drop sentinel. session.setdefault("viewers", {})[transport] = time.time() # See #83716. if transport is not _detached_ws_transport: _cancel_ws_orphan_reap(sid) # the client is back — a pending ws-orphan reap must not fire def _ws_orphan_turn_activity_is_fresh(session: dict) -> bool: """Whether a detached RUNNING turn's activity clock (``_touch_activity``) is still fresh — the reaper must NOT interrupt healthy detached work (closed laptop). Conservative: disabled threshold, missing/opaque agent, unreadable summary or never-stamped clock all report NOT fresh (eligible for interrupt-at-grace) to keep the wedged-turn net. Reuses the agent's existing activity summary (``_touch_activity`` is stamped by API waits, stream tokens, and tool heartbeats — the same clock the turn-liveness watchdog samples; see agent/turn_liveness.py). See #100325, #98028. Isolated turns mirror that clock from the child under a unique dispatch token; their monotonic samples keep aging even if the child or its pipe stalls. """ if _WS_ORPHAN_ACTIVITY_STALE_S <= 0: return False if session.get("_compute_host_turn_id"): with session["history_lock"]: stamp = session.get("_compute_host_activity_ns") return (session.get("running", False) and isinstance(stamp, int) and 0 <= (time.perf_counter_ns() - stamp) / 1_000_000_000 < _WS_ORPHAN_ACTIVITY_STALE_S) if not callable(summary_fn := getattr(session.get("agent"), "get_activity_summary", None)): return False try: elapsed = summary_fn().get("seconds_since_activity") return elapsed is not None and float(elapsed) < _WS_ORPHAN_ACTIVITY_STALE_S except Exception: return False def _schedule_ws_orphan_reap( sid: str, *, delay_s: float | None = None, _expected_timer: threading.Timer | None = None, ) -> None: """After a grace window, reap session ``sid`` iff it's still orphaned. Called from the WS-disconnect path; a reconnect or ``session.resume`` cancels the reap by re-binding a live transport. Disabled when grace is 0. The grace is measured in AWAKE (monotonic) time: ``threading.Timer``'s wait elapses in wall-clock time on platforms without a monotonic condvar (macOS lacks ``pthread_condattr_setclock``), so a system sleep makes the timer fire "early" in awake-time terms. Without the sleep check below, closing a laptop lid for longer than the grace reaped the parked session at the instant of wake — before the Desktop's WS reconnect or ``session.resume`` could re-bind a transport — so every sleep/wake cycle 404'd the open chat (#44183).""" if _WS_ORPHAN_REAP_GRACE_S <= 0: return grace_s = _WS_ORPHAN_REAP_GRACE_S if delay_s is None else max(0.0, delay_s) # Sample both clocks at arm time: time.monotonic() (mach_absolute_time / CLOCK_MONOTONIC) # does not advance while the host is asleep, so the divergence between the two clocks' # elapsed times at fire time is exactly the time the host spent asleep. armed_monotonic = time.monotonic() armed_wall = time.time() def _reap() -> None: # The timer fired. If more wall-clock than monotonic time elapsed, the host # slept through the wait: the grace has NOT been granted in awake time, so # re-arm for the remaining awake grace instead of reaping. The slack keeps # ordinary timer jitter and NTP slew from re-arming a legitimately-expired # reap, and a fired-without-elapsed timer (tests, spurious dispatch) shows # zero divergence and reaps normally. slept_s = (time.time() - armed_wall) - (time.monotonic() - armed_monotonic) if slept_s > _WS_ORPHAN_REAP_SLEEP_SLACK_S: rearm_delay = max(0.0, grace_s - (time.monotonic() - armed_monotonic)) if rearm_delay <= 0: pass # no awake grace left — fall through and reap else: # Re-arm through the public scheduler with THIS timer as the expected # one: the fresh closure's identity guard then matches the entry it # installs, so the awake-remainder fire proceeds to the real reap. _schedule_ws_orphan_reap(sid, delay_s=rearm_delay, _expected_timer=timer) return # Serialize the re-check against session.resume (rebinds under _session_resume_lock). Claim teardown by popping # under both locks, then release the resume lock before slow finalization. Order: resume_lock -> sessions_lock. reschedule_delay = interrupt_session = session = None with _session_resume_lock, _sessions_lock: # Keep ownership through interrupt I/O and continuation registration. A cancelled # callback may already be dispatched, but cannot act on a later detachment. if _pending_ws_reaps.get(sid) is not timer: return current = _sessions.get(sid) if current is None: _pending_ws_reaps.pop(sid, None) return if not _ws_session_is_detached(current): # This Timer is abandoning the interrupt claim because another # writer moved the live record off the detached transport. # Do not leave reattach RPCs fenced with 4009, or let this # generation's settlement polls shorten a later detachment. current.pop("_client_gone_interrupt_requested", None) current.pop("_client_gone_interrupt_polls", None) _pending_ws_reaps.pop(sid, None) return if _session_has_active_delegations(sid, current): reschedule_delay = _WS_ORPHAN_REAP_GRACE_S elif not current.get("running"): session = _pop_session_by_id(sid) elif not current.get("_client_gone_interrupt_requested") and _ws_orphan_turn_activity_is_fresh(current): # Client-absent but producing: keep running detached (the sentinel buffers emits), re-check each grace. logger.debug("client_gone sid=%s action=defer (turn activity fresh; stale threshold %.0fs)", sid, _WS_ORPHAN_ACTIVITY_STALE_S) reschedule_delay = _WS_ORPHAN_REAP_GRACE_S else: # Mid-turn detached sessions must never drop the single Timer: interrupt once after grace, then poll # until turn-finalization settles. polls = current["_client_gone_interrupt_polls"] = int(current.get("_client_gone_interrupt_polls") or 0) + 1 # See #85578. if polls > _WS_ORPHAN_INTERRUPT_REAP_MAX_POLLS: # Never settled inside the budget — force-reap rather than park forever. logger.error( "client_gone sid=%s: turn did not settle after %d interrupt polls (%.0fs) — force-reaping detached session", sid, polls - 1, (polls - 1) * _WS_ORPHAN_INTERRUPT_REAP_POLL_S) session = _pop_session_by_id(sid) else: if not current.get("_client_gone_interrupt_requested"): current["_client_gone_interrupt_requested"] = True interrupt_session = current reschedule_delay = _WS_ORPHAN_INTERRUPT_REAP_POLL_S if reschedule_delay is None: _pending_ws_reaps.pop(sid, None) if interrupt_session is not None: try: isolated = _interrupt_session_turn(sid, interrupt_session, request_id=f"client-gone-{sid}") logger.info("client_gone sid=%s action=interrupt turn_isolation=%s", sid, isolated) except Exception: logger.exception("client_gone interrupt failed sid=%s", sid) with _sessions_lock: if (_sessions.get(sid) is interrupt_session and _pending_ws_reaps.get(sid) is timer): interrupt_session.pop("_client_gone_interrupt_requested", None) if reschedule_delay is not None: _schedule_ws_orphan_reap(sid, delay_s=reschedule_delay, _expected_timer=timer) return if session is not None and session.get("_client_gone_interrupt_requested"): logger.info("client_gone sid=%s action=reap", sid) _teardown_popped_session(session, end_reason="ws_orphan_reap") with _sessions_lock: if _expected_timer is not None and _pending_ws_reaps.get(sid) is not _expected_timer: return timer = threading.Timer(_WS_ORPHAN_REAP_GRACE_S if delay_s is None else max(0.0, delay_s), _reap) timer.daemon = True prior = _pending_ws_reaps.pop(sid, None) _pending_ws_reaps[sid] = timer if prior is not None: with contextlib.suppress(Exception): prior.cancel() timer.start() def _close_sessions_for_transport(transport, *, end_reason: str = "ws_disconnect") -> tuple[int, int]: """Single WS-disconnect teardown entry point: reap close_on_disconnect sessions (sidecar/dashboard) immediately; re-point the rest at the detached transport (later emits miss the dead socket) for the grace-windowed WS-orphan reaper. Returns ``(reaped, detached)`` counts. Multi-client fan-out: the departing transport is DETACHED from every session first. A session that still has another client attached keeps streaming and is neither parked nor reaped — a watcher leaving must not end the turn the remaining client is reading. Only the sessions left clientless take the historical close_on_disconnect / park-sentinel path, so a single-client disconnect behaves exactly as it always has.""" clientless = _detach_transport_from_sessions(transport) reaped = detached = 0 for sid, session in clientless: claimed_for_teardown = None should_schedule_reap = False # session.resume fast-path attaches under _session_resume_lock: take it so a reconnect can't attach # between the detach above and the claim. with _session_resume_lock, _sessions_lock: current = _sessions.get(sid) if current is not session: continue # Prune the departing viewer registration in every branch; it must not affect the new owner. (current.get("viewers") or {}).pop(transport, None) # Revalidate before claiming (#77129, kept under fan-out): between _detach_transport_from_sessions # returning this session as clientless and this claim, a concurrent session.resume can attach a NEW # live transport. Tearing the session down, or parking the sentinel over it, would knock an attached # client into detached state and arm an orphan reap against a session that has a live owner. Attach # and detach both serialize on _session_transport_lock, so this check is race-free against them; that # lock is a leaf, so taking it under _sessions_lock is safe and _session_has_live_transport does not # re-acquire it. with _session_transport_lock: if _session_has_live_transport(current, excluding=transport): continue if current.get("close_on_disconnect"): claimed_for_teardown = _pop_session_by_id(sid) else: # Point at the drop sentinel (NOT real stdio) so _ws_session_is_orphaned recognizes it; standalone # `hermes --tui` keeps real _stdio. UNLESS another window (pop-out viewer) still shows the session: # re-bind to the most recent surviving viewer instead. viewers = current.get("viewers") or {} # See #83716. viewers.pop(transport, None) live = [vt for vt, ts in sorted(viewers.items(), key=lambda kv: kv[1]) if not _transport_is_dead(vt)] if live: _rebind_live_transport(sid, current, live[-1]) else: current["transport"] = _detached_ws_transport current.pop("_client_gone_interrupt_requested", None) current.pop("_client_gone_interrupt_polls", None) should_schedule_reap = True # Register before releasing the detachment claim: an old disconnect # must not arm its first timer over a reconnect's newer detachment. with contextlib.suppress(Exception): _schedule_ws_orphan_reap(sid) if claimed_for_teardown is not None: reaped += _teardown_popped_session(claimed_for_teardown, end_reason=end_reason) elif should_schedule_reap: detached += 1 return reaped, detached def register(server) -> None: """Publish this module's helpers onto ``server``, rebound to its globals.""" bind_module(globals(), server, skip=("_",))