fix(tui-gateway): reap stale deferred active-session leases (#62823 zombie slot)

When a compute-host turn has not settled by session close, the session's
real active-session lease is parked in _deferred_active_session_leases and
the turn's completion callback is supposed to release it. If that callback
is lost (supervisor restart/reload, child killed without failing its
pending turns, dropped completion), the lease sat in the registry forever:
_own_live_lease_ids vouches for it so the orphan sweep never reclaims it,
and the concurrent-session cap treats the dead session as active - new
sessions could not send until a backend restart.

Track when each lease was deferred and have the reaper tick force-release
deferred leases past a generous 30-minute ceiling (the longest legitimate
isolated turn is the compression ceiling, minutes). Normal settlement
still releases immediately and clears the age entry.

Fixes #62823 (zombie-concurrency-slot half; cross-window queue-sync was
fixed by #122953 and visual-merge by #70986 - not touched here)

tests/tui_gateway/test_deferred_lease_reaper.py

@@ -0,0 +1,82 @@
+"""Stale deferred active-session leases are reaped, not held forever (#62823).
+
+When a compute-host turn has not settled by session close, the session's real
+active-session lease is parked in ``_deferred_active_session_leases`` and the
+turn's completion callback is supposed to release it. If that callback is lost
+(supervisor restart, child killed without failing pending turns), the lease
+previously sat in the registry FOREVER: ``_own_live_lease_ids`` vouches for it,
+the orphan sweep skips it, and the concurrent-session cap treats the dead
+session as active — new sessions could not send until a backend restart.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from pathlib import Path
+
+from hermes_cli.active_sessions import ActiveSessionLease, try_acquire_active_session
+from tui_gateway import server
+
+
+def _lease(tmp_path: Path, key: str = "stored-session") -> ActiveSessionLease:
+    lease, err = try_acquire_active_session(
+        session_id=key, surface="desktop", config={}, registry_home=tmp_path, track_liveness=True)
+    assert err is None
+    assert lease is not None
+    return lease
+
+
+def _defer(lease: ActiveSessionLease, age_seconds: float) -> None:
+    lease_id = str(lease.lease_id)
+    server._deferred_active_session_leases[lease_id] = lease
+    server._deferred_active_session_lease_ages[lease_id] = time.time() - age_seconds
+
+
+def test_stale_deferred_lease_is_force_released(tmp_path):
+    lease = _lease(tmp_path)
+    _defer(lease, age_seconds=server._DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS + 60)
+
+    reaped = server._reap_stale_deferred_leases()
+
+    assert reaped == 1
+    assert lease.released is True
+    assert str(lease.lease_id) not in server._deferred_active_session_leases
+    assert str(lease.lease_id) not in server._deferred_active_session_lease_ages
+
+
+def test_fresh_deferred_lease_is_kept(tmp_path):
+    lease = _lease(tmp_path)
+    _defer(lease, age_seconds=10)
+
+    reaped = server._reap_stale_deferred_leases()
+
+    assert reaped == 0
+    assert lease.released is False
+    assert str(lease.lease_id) in server._deferred_active_session_leases
+
+    # cleanup: settle normally
+    session = {"history_lock": threading.Lock(), "_deferred_active_session_lease": lease}
+    server._release_deferred_active_session_lease(session)
+    assert lease.released is True
+
+
+def test_settlement_clears_the_age_entry(tmp_path):
+    lease = _lease(tmp_path)
+    _defer(lease, age_seconds=0)
+    session = {"history_lock": threading.Lock(), "_deferred_active_session_lease": lease}
+
+    server._release_deferred_active_session_lease(session)
+
+    assert lease.released is True
+    assert str(lease.lease_id) not in server._deferred_active_session_lease_ages
+    assert str(lease.lease_id) not in server._deferred_active_session_leases
+
+
+def test_reaped_lease_no_longer_vouched_by_own_live_lease_ids(monkeypatch, tmp_path):
+    lease = _lease(tmp_path)
+    _defer(lease, age_seconds=server._DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS + 60)
+
+    assert str(lease.lease_id) in server._own_live_lease_ids()
+    server._reap_stale_deferred_leases()
+    assert str(lease.lease_id) not in server._own_live_lease_ids()

tui_gateway/session_lifecycle.py

@@ -549,16 +549,54 @@ def _settle_isolated_turn_before_close(session: dict) -> None:
             return
         session["_deferred_active_session_lease"] = lease
         _deferred_active_session_leases[str(lease.lease_id)] = lease
+        _deferred_active_session_lease_ages[str(lease.lease_id)] = time.time()
     logger.warning("isolated turn still live after %.1fs close grace; holding lease for %s until the child settles",
                    _TURN_SETTLE_BEFORE_CLOSE_SECONDS, session.get("session_key"))
 
 
+# A deferred lease is released by the compute-host turn's completion callback
+# (_on_compute_host_turn_done → _release_deferred_active_session_lease). If that callback
+# is lost — supervisor restart/reload, a child killed without failing its pending turns,
+# a dropped completion — the lease sits in the registry FOREVER: _own_live_lease_ids
+# vouches for it, so the orphan sweep never reclaims it, and the concurrency cap treats
+# the dead session as active (#62823 zombie slot). A deferred lease past this generous
+# ceiling (the longest legitimate isolated turn is the compression ceiling, minutes) is
+# force-released by the reaper tick instead of leaking the slot until process exit.
+_DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS = 1800.0
+_deferred_active_session_lease_ages: dict[str, float] = {}
+
+
+def _reap_stale_deferred_leases(now: float | None = None) -> int:
+    """Force-release deferred leases whose settlement callback never arrived. Returns the count."""
+    now = time.time() if now is None else now
+    stale = [
+        lease_id for lease_id, deferred_at in list(_deferred_active_session_lease_ages.items())
+        if now - deferred_at > _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS
+    ]
+    reaped = 0
+    for lease_id in stale:
+        _deferred_active_session_lease_ages.pop(lease_id, None)
+        lease = _deferred_active_session_leases.pop(lease_id, None)
+        if lease is None:
+            continue
+        if (err := _lease_retry(3, lease.release)) is not None:
+            logger.warning("Failed to force-release stale deferred active session lease %s", lease_id,
+                           exc_info=err)
+            continue
+        logger.warning("Force-released deferred active session lease %s held past %.0fs without a "
+                       "compute-host settlement (zombie concurrency slot, #62823)",
+                       lease_id, _DEFERRED_ACTIVE_SESSION_LEASE_TTL_SECONDS)
+        reaped += 1
+    return reaped
+
+
 def _release_deferred_active_session_lease(session: dict) -> None:
     """Settlement half of ``_settle_isolated_turn_before_close``; a no-op for sessions that never deferred."""
     lease = session.pop("_deferred_active_session_lease", None)
     if lease is None:
         return
     _deferred_active_session_leases.pop(str(lease.lease_id), None)
+    _deferred_active_session_lease_ages.pop(str(lease.lease_id), None)
     if (err := _lease_retry(3, lease.release)) is not None:
         logger.warning("Failed to release deferred active session slot", exc_info=err)
 

tui_gateway/session_reaper.py

@@ -306,6 +306,12 @@ def _repair_missing_ws_orphan_reaps() -> None:
 
 def _reclaim_orphaned_leases() -> None:
     """Hand the registry the lease ids we still own so it can drop the rest."""
+    try:
+        # Stale deferred leases first: a settlement callback that never arrived must not
+        # keep vouching for a zombie slot (#62823). Released leases leave _own_live_lease_ids.
+        _reap_stale_deferred_leases()
+    except Exception:
+        logger.debug("stale deferred lease sweep failed", exc_info=True)
     try:
         from hermes_cli.active_sessions import release_orphaned_leases
         if dropped := release_orphaned_leases(_own_live_lease_ids()):