From 0c0796bf46c245a6ed191da892e8f286c96dbf13 Mon Sep 17 00:00:00 2001 From: Austin Pickett Date: Fri, 25 Sep 2026 14:44:37 -0400 Subject: [PATCH] fix(tui-gateway): reap stale deferred active-session leases (#62823 zombie slot) When a compute-host turn has not settled by session close, the session's real active-session lease is parked in _deferred_active_session_leases and the turn's completion callback is supposed to release it. If that callback is lost (supervisor restart/reload, child killed without failing its pending turns, dropped completion), the lease sat in the registry forever: _own_live_lease_ids vouches for it so the orphan sweep never reclaims it, and the concurrent-session cap treats the dead session as active - new sessions could not send until a backend restart. Track when each lease was deferred and have the reaper tick force-release deferred leases past a generous 30-minute ceiling (the longest legitimate isolated turn is the compression ceiling, minutes). Normal settlement still releases immediately and clears the age entry. Fixes #62823 (zombie-concurrency-slot half; cross-window queue-sync was fixed by #122953 and visual-merge by #70986 - not touched here) --- .../tui_gateway/test_deferred_lease_reaper.py | 82 +++++++++++++++++++ tui_gateway/session_lifecycle.py | 38 +++++++++ tui_gateway/session_reaper.py | 6 ++ 3 files changed, 126 insertions(+) create mode 100644 tests/tui_gateway/test_deferred_lease_reaper.py diff --git a/tests/tui_gateway/test_deferred_lease_reaper.py b/tests/tui_gateway/test_deferred_lease_reaper.py new file mode 100644 index 0000000000..d099bd5a8d --- /dev/null +++ b/tests/tui_gateway/test_deferred_lease_reaper.py @@ -0,0 +1,82 @@ +"""Stale deferred active-session leases are reaped, not held forever (#62823). + +When a compute-host turn has not settled by session close, the session's real +active-session lease is parked in ``_deferred_active_session_leases`` and the +turn's completion callback is supposed to release it. If that callback is lost +(supervisor restart, child killed without failing pending turns), the lease +previously sat in the registry FOREVER: ``_own_live_lease_ids`` vouches for it, +the orphan sweep skips it, and the concurrent-session cap treats the dead +session as active — new sessions could not send until a backend restart. +""" + +from __future__ import annotations + +import threading +import time +from pathlib import Path + +from hermes_cli.active_sessions import ActiveSessionLease, try_acquire_active_session +from tui_gateway import server + + +def _lease(tmp_path: Path, key: str = "stored-session") -> ActiveSessionLease: + lease, err = try_acquire_active_session( + session_id=key, surface="desktop", config={}, registry_home=tmp_path, track_liveness=True) + assert err is None + assert lease is not None + return lease + + +def _defer(lease: ActiveSessionLease, age_seconds: float) -> None: + lease_id = str(lease.lease_id) + server._deferred_active_session_leases[lease_id] = lease + server._deferred_active_session_lease_ages[lease_id] = time.time() - age_seconds + + +def test_stale_deferred_lease_is_force_released(tmp_path): + lease = _lease(tmp_path) + _defer(lease, age_seconds=server._DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS + 60) + + reaped = server._reap_stale_deferred_leases() + + assert reaped == 1 + assert lease.released is True + assert str(lease.lease_id) not in server._deferred_active_session_leases + assert str(lease.lease_id) not in server._deferred_active_session_lease_ages + + +def test_fresh_deferred_lease_is_kept(tmp_path): + lease = _lease(tmp_path) + _defer(lease, age_seconds=10) + + reaped = server._reap_stale_deferred_leases() + + assert reaped == 0 + assert lease.released is False + assert str(lease.lease_id) in server._deferred_active_session_leases + + # cleanup: settle normally + session = {"history_lock": threading.Lock(), "_deferred_active_session_lease": lease} + server._release_deferred_active_session_lease(session) + assert lease.released is True + + +def test_settlement_clears_the_age_entry(tmp_path): + lease = _lease(tmp_path) + _defer(lease, age_seconds=0) + session = {"history_lock": threading.Lock(), "_deferred_active_session_lease": lease} + + server._release_deferred_active_session_lease(session) + + assert lease.released is True + assert str(lease.lease_id) not in server._deferred_active_session_lease_ages + assert str(lease.lease_id) not in server._deferred_active_session_leases + + +def test_reaped_lease_no_longer_vouched_by_own_live_lease_ids(monkeypatch, tmp_path): + lease = _lease(tmp_path) + _defer(lease, age_seconds=server._DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS + 60) + + assert str(lease.lease_id) in server._own_live_lease_ids() + server._reap_stale_deferred_leases() + assert str(lease.lease_id) not in server._own_live_lease_ids() diff --git a/tui_gateway/session_lifecycle.py b/tui_gateway/session_lifecycle.py index f62429218d..f3b55a5330 100644 --- a/tui_gateway/session_lifecycle.py +++ b/tui_gateway/session_lifecycle.py @@ -549,16 +549,54 @@ def _settle_isolated_turn_before_close(session: dict) -> None: return session["_deferred_active_session_lease"] = lease _deferred_active_session_leases[str(lease.lease_id)] = lease + _deferred_active_session_lease_ages[str(lease.lease_id)] = time.time() logger.warning("isolated turn still live after %.1fs close grace; holding lease for %s until the child settles", _TURN_SETTLE_BEFORE_CLOSE_SECONDS, session.get("session_key")) +# A deferred lease is released by the compute-host turn's completion callback +# (_on_compute_host_turn_done → _release_deferred_active_session_lease). If that callback +# is lost — supervisor restart/reload, a child killed without failing its pending turns, +# a dropped completion — the lease sits in the registry FOREVER: _own_live_lease_ids +# vouches for it, so the orphan sweep never reclaims it, and the concurrency cap treats +# the dead session as active (#62823 zombie slot). A deferred lease past this generous +# ceiling (the longest legitimate isolated turn is the compression ceiling, minutes) is +# force-released by the reaper tick instead of leaking the slot until process exit. +_DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS = 1800.0 +_deferred_active_session_lease_ages: dict[str, float] = {} + + +def _reap_stale_deferred_leases(now: float | None = None) -> int: + """Force-release deferred leases whose settlement callback never arrived. Returns the count.""" + now = time.time() if now is None else now + stale = [ + lease_id for lease_id, deferred_at in list(_deferred_active_session_lease_ages.items()) + if now - deferred_at > _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS + ] + reaped = 0 + for lease_id in stale: + _deferred_active_session_lease_ages.pop(lease_id, None) + lease = _deferred_active_session_leases.pop(lease_id, None) + if lease is None: + continue + if (err := _lease_retry(3, lease.release)) is not None: + logger.warning("Failed to force-release stale deferred active session lease %s", lease_id, + exc_info=err) + continue + logger.warning("Force-released deferred active session lease %s held past %.0fs without a " + "compute-host settlement (zombie concurrency slot, #62823)", + lease_id, _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS) + reaped += 1 + return reaped + + def _release_deferred_active_session_lease(session: dict) -> None: """Settlement half of ``_settle_isolated_turn_before_close``; a no-op for sessions that never deferred.""" lease = session.pop("_deferred_active_session_lease", None) if lease is None: return _deferred_active_session_leases.pop(str(lease.lease_id), None) + _deferred_active_session_lease_ages.pop(str(lease.lease_id), None) if (err := _lease_retry(3, lease.release)) is not None: logger.warning("Failed to release deferred active session slot", exc_info=err) diff --git a/tui_gateway/session_reaper.py b/tui_gateway/session_reaper.py index d4e94441ba..a1955d920a 100644 --- a/tui_gateway/session_reaper.py +++ b/tui_gateway/session_reaper.py @@ -306,6 +306,12 @@ def _repair_missing_ws_orphan_reaps() -> None: def _reclaim_orphaned_leases() -> None: """Hand the registry the lease ids we still own so it can drop the rest.""" + try: + # Stale deferred leases first: a settlement callback that never arrived must not + # keep vouching for a zombie slot (#62823). Released leases leave _own_live_lease_ids. + _reap_stale_deferred_leases() + except Exception: + logger.debug("stale deferred lease sweep failed", exc_info=True) try: from hermes_cli.active_sessions import release_orphaned_leases if dropped := release_orphaned_leases(_own_live_lease_ids()):