hermes-agent/tests/hermes_state/test_storage_health_latch.py

"""A structurally corrupt state.db is published once and read the same way everywhere (#72046).

Before this, a damaged store showed up as an empty Desktop sidebar (200 + an ``errors`` row),
a 500 from ``/api/sessions``, ``components.storage: ok`` on ``/api/status`` and green
readiness. Each surface guessed; none said the store was damaged, so it read as deleted
history. ``hermes_state_health`` is now the one latch: readers, writers and the readiness
probe publish into it, and every surface below reads it.
"""

import sqlite3

import pytest

from hermes_state import SessionDB, StateDbCorruptError
from hermes_state_errors import classify_persistence_error
from hermes_state_health import (
    is_structural_corruption_error,
    note_storage_error,
    reset_storage_state,
    storage_state,
)


@pytest.fixture(autouse=True)
def _fresh_latch():
    reset_storage_state()
    yield
    reset_storage_state()


def _seed(db_path, count=40):
    db = SessionDB(db_path=db_path)
    try:
        for i in range(count):
            sid = f"s{i:03d}"
            db.create_session(sid, source="cli")
            db.append_message(session_id=sid, role="user", content="hello " * 50)
    finally:
        db.close()


def _corrupt_sessions_btree(db_path):
    """Overwrite the root page of the ``sessions`` table: real B-tree damage, not a mock."""
    conn = sqlite3.connect(db_path)
    try:
        if conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal":
            conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
        page_size = conn.execute("PRAGMA page_size").fetchone()[0]
        root = conn.execute("SELECT rootpage FROM sqlite_master WHERE name = 'sessions'").fetchone()[0]
    finally:
        conn.close()
    with open(db_path, "r+b") as fh:
        fh.seek(page_size * (root - 1))
        fh.write(b"\xde\xad\xbe\xef" * (page_size // 4))


# ── The latch itself ───────────────────────────────────────────────────────


class TestLatch:
    def test_only_structural_corruption_latches(self, tmp_path):
        path = tmp_path / "state.db"
        assert note_storage_error(path, sqlite3.OperationalError("database is locked")) is False
        assert note_storage_error(path, sqlite3.DatabaseError("malformed database schema (messages_fts)")) is False
        fts = sqlite3.DatabaseError("database disk image is malformed")
        fts.sqlite_errorcode = 267  # SQLITE_CORRUPT_VTAB: FTS-scoped, has its own fail-open path
        assert note_storage_error(path, fts) is False
        assert storage_state(path) == "ok"

        assert note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed")) is True
        assert storage_state(path) == "corrupt"

    def test_not_a_database_is_structural(self):
        assert is_structural_corruption_error(sqlite3.DatabaseError("file is not a database"))

    def test_a_real_read_on_a_damaged_btree_latches(self, tmp_path):
        path = tmp_path / "state.db"
        _seed(path)
        _corrupt_sessions_btree(path)
        db = SessionDB(db_path=path, read_only=True)
        try:
            with pytest.raises(sqlite3.DatabaseError):
                db.list_sessions_rich(limit=50, offset=0)
        finally:
            db.close()
        assert storage_state(path) == "corrupt"


class TestPeerWritesAfterLatch:
    """A second handle in the same process must not keep writing to a file another handle
    already found damaged, and its refusal must be the SAME terminal error the gateway and
    agent flush already divert on (not a new class they would retry)."""

    def test_fresh_handle_refuses_writes_with_state_db_corrupt_error(self, tmp_path):
        path = tmp_path / "state.db"
        _seed(path, count=1)
        note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed"))

        peer = SessionDB(db_path=path)
        try:
            with pytest.raises(StateDbCorruptError) as excinfo:
                peer.append_message(session_id="s000", role="user", content="after the latch")
            assert classify_persistence_error(excinfo.value) == "corrupt"
            assert peer._db_corrupt is True
        finally:
            peer.close()

        conn = sqlite3.connect(path)
        try:
            contents = [row[0] for row in conn.execute("SELECT content FROM messages")]
        finally:
            conn.close()
        assert "after the latch" not in contents

    def test_quarantine_on_one_handle_publishes_the_latch(self, tmp_path):
        path = tmp_path / "state.db"
        db = SessionDB(db_path=path)
        real = db._conn

        class _Malformed:
            def execute(self, *a, **k):
                raise sqlite3.DatabaseError("database disk image is malformed")

            def __getattr__(self, name):
                return getattr(real, name)

        try:
            db.create_session("s1", source="cli")
            db._conn = _Malformed()
            with pytest.raises(StateDbCorruptError):
                db.create_session("s2", source="cli")
        finally:
            db._conn = real
            db.close()
        assert storage_state(path) == "corrupt"


# ── Readiness ──────────────────────────────────────────────────────────────


def test_readiness_state_db_and_session_store_agree_on_corrupt(_isolate_hermes_home):
    from gateway.readiness import collect_runtime_readiness
    from hermes_constants import get_hermes_home

    path = get_hermes_home() / "state.db"
    _seed(path, count=1)
    healthy = collect_runtime_readiness(configured_model="m", runtime_status={"session_store": {"status": "ok"}})
    assert healthy["checks"]["state_db"]["status"] == "ok"

    note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed"))
    ready = collect_runtime_readiness(configured_model="m", runtime_status={"session_store": {"status": "ok"}})
    assert ready["status"] == "degraded"
    assert ready["checks"]["state_db"] == {"status": "degraded", "detail": "corrupt"}
    # The gateway's handle cache says "ok"; an open handle on a damaged file is not a working store.
    assert ready["checks"]["session_store"] == {"status": "unavailable", "detail": "corrupt"}


def test_readiness_probe_latches_a_file_that_is_not_a_database(_isolate_hermes_home):
    from gateway.readiness import _probe_state_db
    from hermes_constants import get_hermes_home

    home = get_hermes_home()
    (home / "state.db").write_bytes(b"this is not sqlite" * 512)
    assert _probe_state_db(home) == {"status": "degraded", "detail": "corrupt"}
    assert storage_state(home / "state.db") == "corrupt"


# ── Desktop-facing HTTP surfaces, end to end on a real damaged file ─────────


@pytest.fixture
def client(monkeypatch, _isolate_hermes_home):
    try:
        from starlette.testclient import TestClient
    except ImportError:
        pytest.skip("fastapi/starlette not installed")

    import hermes_state
    from hermes_cli.web_routers import profiles as profiles_routes
    from hermes_cli.web_server import _SESSION_HEADER_NAME, _SESSION_TOKEN, app
    from hermes_constants import get_hermes_home

    monkeypatch.setattr(profiles_routes, "_SIDEBAR_CACHE_TTL_SECONDS", 0.0)
    monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db")
    (get_hermes_home() / "config.yaml").write_text("{}\n", encoding="utf-8")
    c = TestClient(app)
    c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN
    return c


def test_corrupt_state_db_is_published_to_every_desktop_surface(client):
    from hermes_constants import get_hermes_home

    path = get_hermes_home() / "state.db"
    _seed(path)

    healthy = client.get("/api/profiles/sessions/sidebar?recents_profile=all").json()
    assert len(healthy["recents"]["sessions"]) > 0
    assert healthy["storage"] == {}
    assert client.get("/api/sessions").json()["storage"] == {}
    assert client.get("/api/status").json()["components"]["storage"] == {"status": "ok"}

    _corrupt_sessions_btree(path)

    # The sidebar used to be the only surface that noticed, as an empty 200 plus an errors row.
    sidebar = client.get("/api/profiles/sessions/sidebar?recents_profile=all")
    assert sidebar.status_code == 200
    body = sidebar.json()
    assert body["recents"]["sessions"] == []
    assert body["storage"] == {"default": "corrupt"}

    # The flat list says "unavailable", not "internal server error".
    flat = client.get("/api/sessions")
    assert flat.status_code == 503
    assert flat.json()["detail"]["error"] == "state_db_corrupt"

    # Status (what Desktop polls) and readiness now say the same thing.
    status = client.get("/api/status").json()
    assert status["components"]["storage"] == {"status": "degraded", "reason": "corrupt"}
    assert status["overall"] == "degraded"
    assert client.get("/api/profiles/sessions?profile=all").json()["storage"] == {"default": "corrupt"}