From 9f9ec647f56832facee068520470ffdff8021bd1 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:12:36 -0700 Subject: [PATCH] fix(gateway): re-check the live store before broadcasting the state.db warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _send_session_db_warning_notifications() broadcast the error recorded at startup without asking whether it was still true. A startup `database is locked` routinely clears while the adapters are still connecting (another profile's open, a `hermes sessions` one-shot, a slow SMB lock release), so the home channels were told the store was unavailable when it had already healed — and a warning that is wrong once is ignored the next time it is right. The RecoverableHandleCache opener already clears `_session_db_init_error` on recovery; the broadcast now drives one open attempt through it first and only warns when the error is still standing. A store that is genuinely still down warns exactly as before. Fixes #108031. --- gateway/run_notifications.py | 10 +++ .../test_session_db_warning_recheck.py | 73 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 tests/gateway/test_session_db_warning_recheck.py diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py index 39bde5a82e..38d69aa422 100644 --- a/gateway/run_notifications.py +++ b/gateway/run_notifications.py @@ -823,6 +823,16 @@ class GatewayNotificationsMixin: error = getattr(self, "_session_db_init_error", None) if not error: return + # Re-check the live store before warning: a startup `database is locked` routinely clears while + # the adapters are still connecting, and a borrowed store handle comes back once its owner + # releases it. The cache's opener clears ``_session_db_init_error`` on recovery, so a stale + # startup failure must not be broadcast as current (#108031). + if getattr(self, "_session_db_handle_cache", None) is not None: + self._open_session_db_for_active_scope() + error = self._session_db_init_error + if not error: + logger.info("state.db recovered before the home-channel warning went out; not broadcasting") + return from hermes_constants import get_default_hermes_root from hermes_state import _default_db_path, classify_persistence_error, format_session_db_unavailable if classify_persistence_error(error) == "corrupt": diff --git a/tests/gateway/test_session_db_warning_recheck.py b/tests/gateway/test_session_db_warning_recheck.py new file mode 100644 index 0000000000..973db27e86 --- /dev/null +++ b/tests/gateway/test_session_db_warning_recheck.py @@ -0,0 +1,73 @@ +"""#108031: the startup state.db warning must not be broadcast once the store has healed.""" +from __future__ import annotations + +import asyncio +import threading + +import pytest + +import gateway.run as gateway_run +import hermes_state +import hermes_state_registry +from gateway.run import _SESSION_DB_UNPINNED +from gateway.session_db_recovery import RecoverableHandleCache + + +class _Clock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +def _runner_with_startup_failure(monkeypatch, clock: _Clock, *, heals: bool): + """A runner whose priming open failed with a lock; the store heals (or not) on the next open.""" + runner = object.__new__(gateway_run.GatewayRunner) + runner._session_db_pinned = _SESSION_DB_UNPINNED + runner._session_db_init_error = "database is locked" + runner._session_db_handles = {} + runner._session_db_handles_lock = threading.Lock() + cache = RecoverableHandleCache( + handles=runner._session_db_handles, lock=runner._session_db_handles_lock, + clock=clock, initial_retry_delay=1, + ) + runner._session_db_handle_cache = cache + calls: list[str] = [] + + def acquire(db_path=None): + calls.append("open") + if len(calls) == 1 or not heals: # the priming open always fails: that is the startup lock + raise RuntimeError("database is locked") + return object() + + monkeypatch.setattr(hermes_state_registry, "acquire", acquire) + monkeypatch.setattr(hermes_state, "AsyncSessionDB", lambda db: ("async", db)) + monkeypatch.setattr(runner, "session_store", None, raising=False) + # Record the startup failure the way __init__ does (one failed open in the cache). + with pytest.raises(RuntimeError): + runner._open_session_db_for_active_scope(raise_on_error=True) + clock.now = 5.0 # past the backoff: the pre-broadcast re-check is allowed to open + sent: list[str] = [] + monkeypatch.setattr(runner, "_home_channel_transports", lambda: [("telegram", {}, "home", object())]) + + async def _capture(_platform, _home, _transport, message, _fmt): + sent.append(message) + + monkeypatch.setattr(runner, "_send_home_channel_message", _capture) + return runner, sent + + +def test_lock_that_cleared_before_connect_is_not_broadcast(monkeypatch): + clock = _Clock() + runner, sent = _runner_with_startup_failure(monkeypatch, clock, heals=True) + asyncio.run(runner._send_session_db_warning_notifications()) + assert sent == [] + assert runner._session_db_init_error is None + + +def test_still_unavailable_store_is_still_broadcast(monkeypatch): + clock = _Clock() + runner, sent = _runner_with_startup_failure(monkeypatch, clock, heals=False) + asyncio.run(runner._send_session_db_warning_notifications()) + assert len(sent) == 1 and "Session database unavailable" in sent[0]