fix(gateway): re-check the live store before broadcasting the state.db warning

_send_session_db_warning_notifications() broadcast the error recorded at startup
without asking whether it was still true. A startup `database is locked` routinely
clears while the adapters are still connecting (another profile's open, a `hermes
sessions` one-shot, a slow SMB lock release), so the home channels were told the
store was unavailable when it had already healed — and a warning that is wrong once
is ignored the next time it is right.

The RecoverableHandleCache opener already clears `_session_db_init_error` on
recovery; the broadcast now drives one open attempt through it first and only warns
when the error is still standing. A store that is genuinely still down warns exactly
as before.

Fixes #108031.
This commit is contained in:
teknium1
2026-09-11 02:12:36 -07:00
committed by Teknium
parent 045704ce00
commit 9f9ec647f5
2 changed files with 83 additions and 0 deletions

View File

@@ -823,6 +823,16 @@ class GatewayNotificationsMixin:
error = getattr(self, "_session_db_init_error", None)
if not error:
return
# Re-check the live store before warning: a startup `database is locked` routinely clears while
# the adapters are still connecting, and a borrowed store handle comes back once its owner
# releases it. The cache's opener clears ``_session_db_init_error`` on recovery, so a stale
# startup failure must not be broadcast as current (#108031).
if getattr(self, "_session_db_handle_cache", None) is not None:
self._open_session_db_for_active_scope()
error = self._session_db_init_error
if not error:
logger.info("state.db recovered before the home-channel warning went out; not broadcasting")
return
from hermes_constants import get_default_hermes_root
from hermes_state import _default_db_path, classify_persistence_error, format_session_db_unavailable
if classify_persistence_error(error) == "corrupt":

View File

@@ -0,0 +1,73 @@
"""#108031: the startup state.db warning must not be broadcast once the store has healed."""
from __future__ import annotations
import asyncio
import threading
import pytest
import gateway.run as gateway_run
import hermes_state
import hermes_state_registry
from gateway.run import _SESSION_DB_UNPINNED
from gateway.session_db_recovery import RecoverableHandleCache
class _Clock:
def __init__(self) -> None:
self.now = 0.0
def __call__(self) -> float:
return self.now
def _runner_with_startup_failure(monkeypatch, clock: _Clock, *, heals: bool):
"""A runner whose priming open failed with a lock; the store heals (or not) on the next open."""
runner = object.__new__(gateway_run.GatewayRunner)
runner._session_db_pinned = _SESSION_DB_UNPINNED
runner._session_db_init_error = "database is locked"
runner._session_db_handles = {}
runner._session_db_handles_lock = threading.Lock()
cache = RecoverableHandleCache(
handles=runner._session_db_handles, lock=runner._session_db_handles_lock,
clock=clock, initial_retry_delay=1,
)
runner._session_db_handle_cache = cache
calls: list[str] = []
def acquire(db_path=None):
calls.append("open")
if len(calls) == 1 or not heals: # the priming open always fails: that is the startup lock
raise RuntimeError("database is locked")
return object()
monkeypatch.setattr(hermes_state_registry, "acquire", acquire)
monkeypatch.setattr(hermes_state, "AsyncSessionDB", lambda db: ("async", db))
monkeypatch.setattr(runner, "session_store", None, raising=False)
# Record the startup failure the way __init__ does (one failed open in the cache).
with pytest.raises(RuntimeError):
runner._open_session_db_for_active_scope(raise_on_error=True)
clock.now = 5.0 # past the backoff: the pre-broadcast re-check is allowed to open
sent: list[str] = []
monkeypatch.setattr(runner, "_home_channel_transports", lambda: [("telegram", {}, "home", object())])
async def _capture(_platform, _home, _transport, message, _fmt):
sent.append(message)
monkeypatch.setattr(runner, "_send_home_channel_message", _capture)
return runner, sent
def test_lock_that_cleared_before_connect_is_not_broadcast(monkeypatch):
clock = _Clock()
runner, sent = _runner_with_startup_failure(monkeypatch, clock, heals=True)
asyncio.run(runner._send_session_db_warning_notifications())
assert sent == []
assert runner._session_db_init_error is None
def test_still_unavailable_store_is_still_broadcast(monkeypatch):
clock = _Clock()
runner, sent = _runner_with_startup_failure(monkeypatch, clock, heals=False)
asyncio.run(runner._send_session_db_warning_notifications())
assert len(sent) == 1 and "Session database unavailable" in sent[0]