diff --git a/apps/desktop/src/api/sessions.test.ts b/apps/desktop/src/api/sessions.test.ts index f3399eab4a..ef9d652b03 100644 --- a/apps/desktop/src/api/sessions.test.ts +++ b/apps/desktop/src/api/sessions.test.ts @@ -218,3 +218,26 @@ describe('listSidebarSessions remote ownership', () => { expect(result.recents.sessions[0]).toMatchObject({ connection_id: 'prometheus', id: 'remote-session' }) }) }) + +describe('listSidebarSessions storage health', () => { + it('passes the backend corrupt-store map through so the sidebar can say why it is empty', async () => { + hermesApi.mockResolvedValue({ + cron: { sessions: [] }, + errors: [{ error: 'database disk image is malformed', profile: 'default' }], + messaging: { sessions: [] }, + recents: { sessions: [] }, + storage: { default: 'corrupt' } + } as never) + + const result = await listSidebarSessions({ + recentsProfile: 'all', + recentsLimit: 40, + recentsExclude: [], + cronLimit: 20, + messagingLimit: 40, + messagingExclude: [] + }) + + expect(result.storage).toEqual({ default: 'corrupt' }) + }) +}) diff --git a/apps/desktop/src/api/sessions.ts b/apps/desktop/src/api/sessions.ts index d983fc1033..ae37d6b55c 100644 --- a/apps/desktop/src/api/sessions.ts +++ b/apps/desktop/src/api/sessions.ts @@ -204,6 +204,9 @@ export interface SidebarSessionsResponse { cron: SidebarSessionSlice messaging: SidebarSessionSlice errors?: Array<{ profile: string; error: string }> + /** `{profile: 'corrupt'}` for each profile whose state.db the backend has found + * structurally damaged. Absent from older backends. */ + storage?: Record } export interface SidebarSessionsRequest { @@ -252,7 +255,10 @@ async function listSidebarSessionsLegacy(req: SidebarSessionsRequest): Promise + + {showSessionSections && (
void }) {
) } + +const SESSION_STORAGE_RECOVERY_URL = + 'https://hermes-agent.nousresearch.com/docs/user-guide/session-storage-recovery#when-the-three-steps-do-not-work' + +// A structurally corrupt state.db empties (or thins out) the list below it, +// which reads as deleted history (#72046). Persistent while the backend +// reports the store corrupt; there is nothing to dismiss until it is recovered. +export function SidebarStorageCorruptNotice() { + const profiles = useStore($corruptSessionStores) + const { t } = useI18n() + const copy = t.sidebar.storageCorrupt + + if (profiles.length === 0) { + return null + } + + return ( +
+ + + {copy.title} + +

{copy.body(profiles.join(', '))}

+

{copy.action}

+ + hermes sessions recover --source <state.db> --inspect-only + + + + +
+ ) +} diff --git a/apps/desktop/src/app/chat/sidebar/storage-corrupt-notice.test.tsx b/apps/desktop/src/app/chat/sidebar/storage-corrupt-notice.test.tsx new file mode 100644 index 0000000000..f9044a1633 --- /dev/null +++ b/apps/desktop/src/app/chat/sidebar/storage-corrupt-notice.test.tsx @@ -0,0 +1,58 @@ +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { $corruptSessionStores, setCorruptSessionStores } from '@/store/session' + +import { SidebarStorageCorruptNotice } from './section-states' + +const openExternalLink = vi.fn() + +vi.mock('@/lib/external-link', () => ({ openExternalLink: (href: string) => openExternalLink(href) })) + +beforeEach(() => { + $corruptSessionStores.set([]) + openExternalLink.mockClear() +}) + +afterEach(cleanup) + +// A structurally corrupt state.db used to render as "No sessions yet" (#72046): +// the list went empty and nothing said the store was damaged. +describe('SidebarStorageCorruptNotice', () => { + it('renders nothing while every store is healthy', () => { + setCorruptSessionStores({}) + render() + + expect(screen.queryByTestId('storage-corrupt-notice')).toBeNull() + }) + + it('names the damaged profile and gives the non-destructive recovery path', () => { + setCorruptSessionStores({ default: 'corrupt' }) + render() + + const notice = screen.getByTestId('storage-corrupt-notice') + + expect(notice.getAttribute('role')).toBe('alert') + expect(notice.textContent).toContain('Session database is damaged') + expect(notice.textContent).toContain('for default') + expect(notice.textContent).toContain('were not deleted') + expect(notice.textContent).toContain('hermes sessions recover --source --inspect-only') + // No blanket "run repair" advice: structural damage goes to inspect/restore first. + expect(notice.textContent).not.toContain('sessions repair') + + screen.getByRole('button', { name: /recovery guide/i }).click() + expect(openExternalLink).toHaveBeenCalledWith(expect.stringContaining('/user-guide/session-storage-recovery')) + }) + + it('keeps atom identity when a refresh reports the same stores', () => { + setCorruptSessionStores({ work: 'corrupt', default: 'corrupt' }) + const first = $corruptSessionStores.get() + + setCorruptSessionStores({ default: 'corrupt', work: 'corrupt' }) + expect($corruptSessionStores.get()).toBe(first) + expect(first).toEqual(['default', 'work']) + + setCorruptSessionStores(undefined) + expect($corruptSessionStores.get()).toEqual([]) + }) +}) diff --git a/apps/desktop/src/app/session/hooks/use-session-list-actions.ts b/apps/desktop/src/app/session/hooks/use-session-list-actions.ts index 3fd19bf20b..fb62d381cb 100644 --- a/apps/desktop/src/app/session/hooks/use-session-list-actions.ts +++ b/apps/desktop/src/app/session/hooks/use-session-list-actions.ts @@ -28,6 +28,7 @@ import { keepFailedProfileMeta, mergeSessionPage, MESSAGING_SECTION_LIMIT, + setCorruptSessionStores, setCronSessions, setMessagingPlatformTotals, setMessagingSessions, @@ -294,6 +295,8 @@ export function useSessionListActions({ profileScope }: UseSessionListActionsArg ) { const recents = result.recents + setCorruptSessionStores(result.storage) + // Drop rows the user just deleted/archived: a refresh can race an // in-flight mutation and the backend page still carries the doomed row. // Honoring the optimistic tombstone keeps the removal from flashing back diff --git a/apps/desktop/src/i18n/en.ts b/apps/desktop/src/i18n/en.ts index 537f49286e..2f2993d5c3 100644 --- a/apps/desktop/src/i18n/en.ts +++ b/apps/desktop/src/i18n/en.ts @@ -3088,6 +3088,13 @@ export const en: Translations = { projectEmpty: 'No sessions yet', projectLoadFailed: 'Could not load sessions', noSessions: 'No sessions yet', + storageCorrupt: { + title: 'Session database is damaged', + body: (profiles: string) => + `Hermes can't read all of the session history for ${profiles}. Chats missing from this list were not deleted; the file they are stored in is damaged.`, + action: 'Quit Hermes on this profile, then inspect the file without changing it, or restore a snapshot:', + guide: 'Recovery guide' + }, noFilterMatches: 'No sessions match these filters', projects: { showAllSessions: 'Show all sessions', diff --git a/apps/desktop/src/i18n/types.ts b/apps/desktop/src/i18n/types.ts index 0757bdf171..4d869b5239 100644 --- a/apps/desktop/src/i18n/types.ts +++ b/apps/desktop/src/i18n/types.ts @@ -2609,6 +2609,12 @@ export interface Translations { projectEmpty: string projectLoadFailed: string noSessions: string + storageCorrupt: { + title: string + body: (profiles: string) => string + action: string + guide: string + } noFilterMatches: string projects: { showAllSessions: string diff --git a/apps/desktop/src/store/session.ts b/apps/desktop/src/store/session.ts index 60dc9a1a9a..f066d51406 100644 --- a/apps/desktop/src/store/session.ts +++ b/apps/desktop/src/store/session.ts @@ -929,6 +929,11 @@ export interface ProfileUsage { } export const $sessionProfilesUsage = atom>({}) + +/** Profiles whose state.db the backend reports as structurally corrupt (the list + * endpoints' `storage` map, #72046). An empty or partial list for one of these + * is a damaged store, not deleted history, and the sidebar says so. */ +export const $corruptSessionStores = atom([]) export const $sessionsLoading = atom(true) export const $activeSessionId = atom(null) export const $selectedStoredSessionId = atom(null) @@ -1296,6 +1301,20 @@ export const setSessionProfilesTruncated = (next: Updater>) => updateAtom($sessionProfilesUsage, next) export const setSessionsLoading = (next: Updater) => updateAtom($sessionsLoading, next) + +/** Publish the corrupt-store profiles from one sidebar refresh; identity-stable when unchanged. */ +export function setCorruptSessionStores(storage: Record | undefined) { + const next = Object.keys(storage ?? {}) + .filter(profile => storage?.[profile] === 'corrupt') + .sort() + + const prev = $corruptSessionStores.get() + + if (prev.length !== next.length || prev.some((profile, i) => profile !== next[i])) { + $corruptSessionStores.set(next) + } +} + export const setActiveSessionId = (next: Updater) => updateAtom($activeSessionId, next) export const setActiveSessionStoredIdRotation = (next: Updater) => updateAtom($activeSessionStoredIdRotation, next) diff --git a/apps/desktop/src/types/hermes.ts b/apps/desktop/src/types/hermes.ts index 6bb632bcea..f1844f3a70 100644 --- a/apps/desktop/src/types/hermes.ts +++ b/apps/desktop/src/types/hermes.ts @@ -496,6 +496,8 @@ export interface PaginatedSessions { /** Per-profile read failures from the cross-profile aggregator (e.g. a locked * or corrupt state.db). Present only on `/api/profiles/sessions`. */ errors?: Array<{ profile: string; error: string }> + /** `{profile: 'corrupt'}` for each listed profile whose state.db is structurally damaged. */ + storage?: Record } export interface SessionCreateResponse { diff --git a/gateway/readiness.py b/gateway/readiness.py index 6c0ca67182..0507b8a743 100644 --- a/gateway/readiness.py +++ b/gateway/readiness.py @@ -22,9 +22,19 @@ def _check(status: str, detail: str | None = None, **extra: Any) -> dict[str, An def _probe_state_db(home: Path) -> dict[str, Any]: + """Read-only schema probe plus the process-wide corruption latch (``hermes_state_health``). + + The schema read only catches an unreadable header or schema; damage deeper in the file + surfaces when a reader or writer touches it, and those publish into the latch. Reporting + the latch here is what makes readiness and ``/api/status`` agree with the session list + (#72046). ``detail="corrupt"`` is the one reason string consumers key off.""" + from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state + path = home / "state.db" if not path.exists(): return _check("ok", "not initialized") + if storage_state(path) == STORAGE_CORRUPT: + return _check("degraded", STORAGE_CORRUPT) try: # Read-only schema query: catches unreadable/corrupt DBs without competing with # writers. ``closing`` is required — sqlite3's context manager only commits/rolls @@ -35,6 +45,8 @@ def _probe_state_db(home: Path) -> dict[str, Any]: conn.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone() return _check("ok") except Exception as exc: + if note_storage_error(path, exc): + return _check("degraded", STORAGE_CORRUPT) return _check("degraded", type(exc).__name__) @@ -71,7 +83,10 @@ def _probe_gateway(runtime_status: dict[str, Any]) -> dict[str, Any]: def _probe_session_store(runtime_status: dict[str, Any], state_db_probe: dict[str, Any]) -> dict[str, Any]: - """Report the running gateway cache state, not an independent reopen.""" + """Report the running gateway cache state, not an independent reopen. A corrupt store is + unavailable whatever the cache says: an open handle on a damaged file is not a working one.""" + if state_db_probe.get("detail") == "corrupt": + return _check("unavailable", "corrupt") runtime_store = runtime_status.get("session_store") state = str(runtime_store.get("status") or "unknown") if isinstance(runtime_store, dict) else "" if state in {"ok", "unavailable", "retrying"}: diff --git a/hermes_cli/web_routers/profiles.py b/hermes_cli/web_routers/profiles.py index 1a0beda234..77d75e0a53 100644 --- a/hermes_cli/web_routers/profiles.py +++ b/hermes_cli/web_routers/profiles.py @@ -39,6 +39,7 @@ from hermes_cli.web_server_profiles import ( _fallback_profile_dicts, _hub_action_name, _write_profile_mcp_servers, ) from hermes_cli.web_server_sessions import _open_session_db_at_path +from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state from starlette.concurrency import run_in_threadpool from hermes_cli.web_models import ( ProfileCreate, ProfileActiveUpdate, ProfileExport, ProfileImport, ProfileRename, @@ -268,6 +269,8 @@ def _read_profile_db(name: str, home, errors: Optional[List[Dict[str, str]]], db = _open_session_db_at_path(db_path, read_only=True) return fn(db) except Exception as exc: + # An open that dies on a damaged file never reaches SessionDB's read helpers. + note_storage_error(db_path, exc) _warn_profile_read_error(name, exc) if errors is not None: errors.append({"profile": name, "error": str(exc)}) @@ -277,6 +280,14 @@ def _read_profile_db(name: str, home, errors: Optional[List[Dict[str, str]]], db.close() +def _corrupt_profile_stores(targets) -> Dict[str, str]: + """``{profile: "corrupt"}`` for every scanned profile whose state.db this process has latched + as structurally corrupt (``hermes_state_health``). Lets Desktop tell an empty or partial list + from a damaged store, including when some reads still succeed (#72046).""" + return {name: STORAGE_CORRUPT for name, home in targets + if storage_state(Path(home) / "state.db") == STORAGE_CORRUPT} + + # Sidebar scan cache TTL: short enough that the UI never shows meaningfully stale data, long # enough to coalesce the desktop's reconnect/focus/change poll bursts into one scan. _SIDEBAR_CACHE_TTL_SECONDS = 5.0 @@ -447,7 +458,8 @@ def get_profiles_sessions( if not full: _strip_session_list_rows(window) return {"sessions": window, "total": sum(totals.values()), "profile_totals": totals, - "limit": limit, "offset": offset, "errors": errors} + "limit": limit, "offset": offset, "errors": errors, + "storage": _corrupt_profile_stores(targets)} @sessions_router.get("/api/profiles/sessions/sidebar") @@ -499,9 +511,11 @@ def get_profiles_sessions_sidebar( _sidebar_profile_cache_put(cache_key, slices) return slices + scanned = [] for name, home in targets: if recents_scope != "all" and name != recents_scope: continue + scanned.append((name, home)) db_path = Path(home) / "state.db" if not db_path.exists(): continue @@ -534,7 +548,7 @@ def get_profiles_sessions_sidebar( "profiles_usage": profile_totals}, "cron": {"sessions": _window("cron")}, "messaging": {"sessions": _window("messaging"), "total": len(rows["messaging"])}, - "errors": errors} + "errors": errors, "storage": _corrupt_profile_stores(scanned)} def _merge_by_id(into: Dict[str, Dict[str, Any]], entries: List[Dict[str, Any]], child_key: str) -> None: diff --git a/hermes_cli/web_routers/sessions.py b/hermes_cli/web_routers/sessions.py index 7cd163ee6c..e406017718 100644 --- a/hermes_cli/web_routers/sessions.py +++ b/hermes_cli/web_routers/sessions.py @@ -22,9 +22,10 @@ from hermes_cli.web_server_gateway import _strip_session_list_rows from hermes_cli.web_server_sessions import _maybe_auto_archive_for_profile, _session_latest_descendant from hermes_cli.web_models import ( BulkDeleteSessions, SessionImport, SessionOwnerBackfill, SessionPrune, SessionRename) -from hermes_cli.web_routers._common import log as _log, destructive_profile, http_failure +from hermes_cli.web_routers._common import CORRUPT_STORE_DETAIL, log as _log, destructive_profile, http_failure from hermes_state import is_malformed_db_error from hermes_state_errors import is_transient_sqlite_error +from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state list_router = APIRouter() search_router = APIRouter() @@ -33,6 +34,7 @@ manage_router = APIRouter() _cron_default_profile = late("_cron_default_profile", "hermes_cli.web_server_cron") _cron_profile_home = late("_cron_profile_home", "hermes_cli.web_server_cron") _open_session_db_for_profile = late("_open_session_db_for_profile", "hermes_cli.web_server_sessions") +_session_db_path_for_profile = late("_session_db_path_for_profile", "hermes_cli.web_server_sessions") _NOT_FOUND = "Session not found" @@ -219,7 +221,11 @@ def get_sessions( s["pinned"] = bool(s.get("pinned")) if not full: _strip_session_list_rows(sessions) - return {"sessions": sessions, "total": total, "limit": limit, "offset": offset} + # ``storage`` tells an empty page apart from an unreadable store (#72046); same + # ``{profile: "corrupt"}`` shape as the /api/profiles/sessions* lists. + storage = {row_profile: STORAGE_CORRUPT} if storage_state(db.db_path) == STORAGE_CORRUPT else {} + return {"sessions": sessions, "total": total, "limit": limit, "offset": offset, + "storage": storage} finally: db.close() except HTTPException: @@ -236,6 +242,14 @@ def get_sessions( if transient else "Internal server error"), ) from exc + except sqlite3.DatabaseError as exc: + # A damaged store is unavailable, not empty and not an internal error (#72046). + db_path = _session_db_path_for_profile(profile) + if not (note_storage_error(db_path, exc) or is_malformed_db_error(exc)): + _log.exception("GET /api/sessions failed") + raise HTTPException(status_code=500, detail="Internal server error") from exc + _log.error("GET /api/sessions: state.db at %s is corrupt: %s", db_path, exc) + raise HTTPException(status_code=503, detail=dict(CORRUPT_STORE_DETAIL)) from exc except Exception: _log.exception("GET /api/sessions failed") raise HTTPException(status_code=500, detail="Internal server error") diff --git a/hermes_cli/web_routers/status.py b/hermes_cli/web_routers/status.py index cbe8399805..88af428de3 100644 --- a/hermes_cli/web_routers/status.py +++ b/hermes_cli/web_routers/status.py @@ -407,6 +407,9 @@ async def _component_health(gateway: Dict[str, Any]) -> Dict[str, Any]: from gateway.readiness import _probe_state_db storage_check = await run_in_threadpool(_probe_state_db, get_hermes_home()) components["storage"] = {"status": storage_check.get("status", "degraded")} + # The one reason enum consumers key off; same latch as readiness and the session lists. + if storage_check.get("detail") == "corrupt": + components["storage"]["reason"] = "corrupt" except Exception: components["storage"] = {"status": "degraded"} # ``disabled`` entries are platforms the multiplexer deliberately does not run for a served profile diff --git a/hermes_state.py b/hermes_state.py index 7e17bd704e..2954eba698 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -33,6 +33,9 @@ from hermes_state_common import ( escape_like as _escape_like, stat_db_file_identity as _stat_db_file_identity, ) from hermes_state_holders import read_only_db_uri +from hermes_state_health import ( + STORAGE_CORRUPT, mark_storage_corrupt, note_storage_error, storage_corrupt_reason, storage_state, +) from hermes_state_errors import ( _DELETED_WAL_GENERATION_MSG, _DISK_IO_ERROR_MARKER, _STATE_DB_CORRUPT_MSG, _STATE_DB_GENERATION_KEY, _STATE_DB_REPLACED_MSG, DeletedWalGenerationError, SessionCompressionInProgressError, StateDbCorruptError, @@ -958,6 +961,13 @@ class SessionDB( ioerr_begin_retried = False while True: self._raise_if_db_corrupt() + if storage_state(self.db_path) == STORAGE_CORRUPT: + # Another handle in this process already saw structural damage on this file. + # Quarantine this one before it touches SQLite; the error type is the same + # StateDbCorruptError, so every transcript-diversion owner handles it unchanged. + self._halt_db_corrupt(sqlite3.DatabaseError( + "database disk image is malformed (reported earlier in this process: " + f"{storage_corrupt_reason(self.db_path)})")) # NOTE: the replaced/generation live probe runs INSIDE the lock below, # not here. close() mutates _conn and _db_sidecar_identity under that # same lock, ending the WAL generation (SQLite unlinks the -wal/-shm @@ -1098,8 +1108,14 @@ class SessionDB( return fn(conn) except sqlite3.OperationalError as exc: if attempt >= _READ_ONLY_IOERR_RETRY_ATTEMPTS or _DISK_IO_ERROR_MARKER not in str(exc).lower(): + note_storage_error(self.db_path, exc) raise time.sleep(_READ_ONLY_IOERR_RETRY_BACKOFF_S) + except sqlite3.DatabaseError as exc: + # A reader is often the only observer (the sidebar poll on a store nobody is + # writing to): publish structural damage so the list is not read as empty. + note_storage_error(self.db_path, exc) + raise def _ensure_db_file_generation(self) -> None: """Mint a once-per-file generation stamp (state_meta + application_id). First opener wins (INSERT @@ -1272,6 +1288,9 @@ class SessionDB( """Quarantine this handle and raise; never run in-file repair here.""" self._db_corrupt = True self._db_corrupt_reason = str(exc) + # Publish the profile-level state first: readiness, /api/status and the session list + # endpoints read it, and every other handle in this process refuses writes on it. + mark_storage_corrupt(self.db_path, exc) self._disable_close_time_checkpoint() logger.error( "state.db %s reported structural corruption outside the FTS " diff --git a/hermes_state_health.py b/hermes_state_health.py new file mode 100644 index 0000000000..9a06e2bec7 --- /dev/null +++ b/hermes_state_health.py @@ -0,0 +1,114 @@ +"""Profile-level session-storage health: one process-wide latch per state.db path. + +A structurally corrupt state.db used to show up as an empty or partial Desktop sidebar, +green readiness and a 500 from ``/api/sessions``: every surface guessed on its own and +none said "the store is damaged" (#72046). This module is the single place that fact is +recorded. Writers (``SessionDB._halt_db_corrupt``), readers (``SessionDB`` read helpers) +and the readiness probe publish into it; ``gateway.readiness`` (``state_db`` / +``session_store`` checks, hence ``/api/status`` ``components.storage``) and the session +list endpoints read from it, so Desktop and readiness cannot disagree. + +Only structural corruption latches: bare ``SQLITE_CORRUPT`` / ``SQLITE_NOTADB`` with no +FTS provenance. FTS-scoped damage has its own fail-open path with canonical rows intact, +and a malformed-schema row is healed by the web open path, so neither is reported here. + +The latch never clears on its own. A corrupt image does not heal, and a store that +flickers between "ok" and "corrupt" is the silent failure this replaces. It resets when +the process restarts, which is the recovery boundary ``StateDbCorruptError`` already +documents (stop Hermes, recover or restore, start again). No marker is written to disk: +the file it would describe is the one that is damaged. +""" + +from __future__ import annotations + +import logging +import sqlite3 +import threading +from pathlib import Path +from typing import Dict, Optional + +from hermes_state_errors import ( + classify_persistence_error, + is_fts_scoped_corruption_error, + is_malformed_schema_error, +) + +logger = logging.getLogger(__name__) + +STORAGE_OK = "ok" +STORAGE_CORRUPT = "corrupt" + +_lock = threading.Lock() +_corrupt: Dict[str, str] = {} # resolved db path -> first error text (log only, never served) + + +def _key(db_path) -> str: + return str(Path(db_path).expanduser().resolve(strict=False)) + + +def is_structural_corruption_error(exc: BaseException) -> bool: + """Canonical B-tree/schema/freelist damage: a corrupt/NOTADB error SQLite does not scope to + the FTS index, and not the malformed-schema case the web open path repairs.""" + return ( + isinstance(exc, sqlite3.DatabaseError) + and not is_fts_scoped_corruption_error(exc) + and not is_malformed_schema_error(exc) + and classify_persistence_error(exc) == "corrupt" + ) + + +def mark_storage_corrupt(db_path, reason: object) -> None: + """Latch *db_path* as corrupt for the life of this process (idempotent).""" + key = _key(db_path) + with _lock: + if key in _corrupt: + return + _corrupt[key] = str(reason) + logger.error( + "state.db at %s is structurally corrupt (%s); session storage is reported as corrupt " + "until Hermes restarts on a recovered or restored file. Stop Hermes, then run " + "`hermes sessions recover --source %s --inspect-only` or restore a snapshot.", + db_path, reason, db_path, + ) + + +def note_storage_error(db_path, exc: BaseException) -> bool: + """Latch *db_path* when *exc* is structural corruption; True when it was.""" + if not is_structural_corruption_error(exc): + return False + mark_storage_corrupt(db_path, exc) + return True + + +def storage_state(db_path) -> str: + """``"corrupt"`` once this process has seen structural corruption on *db_path*, else ``"ok"``.""" + with _lock: + return STORAGE_CORRUPT if _key(db_path) in _corrupt else STORAGE_OK + + +def storage_corrupt_reason(db_path) -> Optional[str]: + """The first error text latched for *db_path* (logs/diagnostics only), or None.""" + with _lock: + return _corrupt.get(_key(db_path)) + + +def reset_storage_state(db_path=None) -> None: + """Forget the latch for *db_path* (all paths when None). For tests and a verified in-process + recovery; nothing in the runtime clears it on its own.""" + with _lock: + if db_path is None: + _corrupt.clear() + else: + _corrupt.pop(_key(db_path), None) + + +__all__ = [ + "STORAGE_CORRUPT", + "STORAGE_OK", + "is_structural_corruption_error", + "mark_storage_corrupt", + "note_storage_error", + "reset_storage_state", + "storage_corrupt_reason", + "storage_state", +] diff --git a/tests/hermes_state/test_storage_health_latch.py b/tests/hermes_state/test_storage_health_latch.py new file mode 100644 index 0000000000..e7d9f3c928 --- /dev/null +++ b/tests/hermes_state/test_storage_health_latch.py @@ -0,0 +1,221 @@ +"""A structurally corrupt state.db is published once and read the same way everywhere (#72046). + +Before this, a damaged store showed up as an empty Desktop sidebar (200 + an ``errors`` row), +a 500 from ``/api/sessions``, ``components.storage: ok`` on ``/api/status`` and green +readiness. Each surface guessed; none said the store was damaged, so it read as deleted +history. ``hermes_state_health`` is now the one latch: readers, writers and the readiness +probe publish into it, and every surface below reads it. +""" + +import sqlite3 + +import pytest + +from hermes_state import SessionDB, StateDbCorruptError +from hermes_state_errors import classify_persistence_error +from hermes_state_health import ( + is_structural_corruption_error, + note_storage_error, + reset_storage_state, + storage_state, +) + + +@pytest.fixture(autouse=True) +def _fresh_latch(): + reset_storage_state() + yield + reset_storage_state() + + +def _seed(db_path, count=40): + db = SessionDB(db_path=db_path) + try: + for i in range(count): + sid = f"s{i:03d}" + db.create_session(sid, source="cli") + db.append_message(session_id=sid, role="user", content="hello " * 50) + finally: + db.close() + + +def _corrupt_sessions_btree(db_path): + """Overwrite the root page of the ``sessions`` table: real B-tree damage, not a mock.""" + conn = sqlite3.connect(db_path) + try: + if conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal": + conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + page_size = conn.execute("PRAGMA page_size").fetchone()[0] + root = conn.execute("SELECT rootpage FROM sqlite_master WHERE name = 'sessions'").fetchone()[0] + finally: + conn.close() + with open(db_path, "r+b") as fh: + fh.seek(page_size * (root - 1)) + fh.write(b"\xde\xad\xbe\xef" * (page_size // 4)) + + +# ── The latch itself ─────────────────────────────────────────────────────── + + +class TestLatch: + def test_only_structural_corruption_latches(self, tmp_path): + path = tmp_path / "state.db" + assert note_storage_error(path, sqlite3.OperationalError("database is locked")) is False + assert note_storage_error(path, sqlite3.DatabaseError("malformed database schema (messages_fts)")) is False + fts = sqlite3.DatabaseError("database disk image is malformed") + fts.sqlite_errorcode = 267 # SQLITE_CORRUPT_VTAB: FTS-scoped, has its own fail-open path + assert note_storage_error(path, fts) is False + assert storage_state(path) == "ok" + + assert note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed")) is True + assert storage_state(path) == "corrupt" + + def test_not_a_database_is_structural(self): + assert is_structural_corruption_error(sqlite3.DatabaseError("file is not a database")) + + def test_a_real_read_on_a_damaged_btree_latches(self, tmp_path): + path = tmp_path / "state.db" + _seed(path) + _corrupt_sessions_btree(path) + db = SessionDB(db_path=path, read_only=True) + try: + with pytest.raises(sqlite3.DatabaseError): + db.list_sessions_rich(limit=50, offset=0) + finally: + db.close() + assert storage_state(path) == "corrupt" + + +class TestPeerWritesAfterLatch: + """A second handle in the same process must not keep writing to a file another handle + already found damaged, and its refusal must be the SAME terminal error the gateway and + agent flush already divert on (not a new class they would retry).""" + + def test_fresh_handle_refuses_writes_with_state_db_corrupt_error(self, tmp_path): + path = tmp_path / "state.db" + _seed(path, count=1) + note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed")) + + peer = SessionDB(db_path=path) + try: + with pytest.raises(StateDbCorruptError) as excinfo: + peer.append_message(session_id="s000", role="user", content="after the latch") + assert classify_persistence_error(excinfo.value) == "corrupt" + assert peer._db_corrupt is True + finally: + peer.close() + + conn = sqlite3.connect(path) + try: + contents = [row[0] for row in conn.execute("SELECT content FROM messages")] + finally: + conn.close() + assert "after the latch" not in contents + + def test_quarantine_on_one_handle_publishes_the_latch(self, tmp_path): + path = tmp_path / "state.db" + db = SessionDB(db_path=path) + real = db._conn + + class _Malformed: + def execute(self, *a, **k): + raise sqlite3.DatabaseError("database disk image is malformed") + + def __getattr__(self, name): + return getattr(real, name) + + try: + db.create_session("s1", source="cli") + db._conn = _Malformed() + with pytest.raises(StateDbCorruptError): + db.create_session("s2", source="cli") + finally: + db._conn = real + db.close() + assert storage_state(path) == "corrupt" + + +# ── Readiness ────────────────────────────────────────────────────────────── + + +def test_readiness_state_db_and_session_store_agree_on_corrupt(_isolate_hermes_home): + from gateway.readiness import collect_runtime_readiness + from hermes_constants import get_hermes_home + + path = get_hermes_home() / "state.db" + _seed(path, count=1) + healthy = collect_runtime_readiness(configured_model="m", runtime_status={"session_store": {"status": "ok"}}) + assert healthy["checks"]["state_db"]["status"] == "ok" + + note_storage_error(path, sqlite3.DatabaseError("database disk image is malformed")) + ready = collect_runtime_readiness(configured_model="m", runtime_status={"session_store": {"status": "ok"}}) + assert ready["status"] == "degraded" + assert ready["checks"]["state_db"] == {"status": "degraded", "detail": "corrupt"} + # The gateway's handle cache says "ok"; an open handle on a damaged file is not a working store. + assert ready["checks"]["session_store"] == {"status": "unavailable", "detail": "corrupt"} + + +def test_readiness_probe_latches_a_file_that_is_not_a_database(_isolate_hermes_home): + from gateway.readiness import _probe_state_db + from hermes_constants import get_hermes_home + + home = get_hermes_home() + (home / "state.db").write_bytes(b"this is not sqlite" * 512) + assert _probe_state_db(home) == {"status": "degraded", "detail": "corrupt"} + assert storage_state(home / "state.db") == "corrupt" + + +# ── Desktop-facing HTTP surfaces, end to end on a real damaged file ───────── + + +@pytest.fixture +def client(monkeypatch, _isolate_hermes_home): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + import hermes_state + from hermes_cli.web_routers import profiles as profiles_routes + from hermes_cli.web_server import _SESSION_HEADER_NAME, _SESSION_TOKEN, app + from hermes_constants import get_hermes_home + + monkeypatch.setattr(profiles_routes, "_SIDEBAR_CACHE_TTL_SECONDS", 0.0) + monkeypatch.setattr(hermes_state, "DEFAULT_DB_PATH", get_hermes_home() / "state.db") + (get_hermes_home() / "config.yaml").write_text("{}\n", encoding="utf-8") + c = TestClient(app) + c.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + return c + + +def test_corrupt_state_db_is_published_to_every_desktop_surface(client): + from hermes_constants import get_hermes_home + + path = get_hermes_home() / "state.db" + _seed(path) + + healthy = client.get("/api/profiles/sessions/sidebar?recents_profile=all").json() + assert len(healthy["recents"]["sessions"]) > 0 + assert healthy["storage"] == {} + assert client.get("/api/sessions").json()["storage"] == {} + assert client.get("/api/status").json()["components"]["storage"] == {"status": "ok"} + + _corrupt_sessions_btree(path) + + # The sidebar used to be the only surface that noticed, as an empty 200 plus an errors row. + sidebar = client.get("/api/profiles/sessions/sidebar?recents_profile=all") + assert sidebar.status_code == 200 + body = sidebar.json() + assert body["recents"]["sessions"] == [] + assert body["storage"] == {"default": "corrupt"} + + # The flat list says "unavailable", not "internal server error". + flat = client.get("/api/sessions") + assert flat.status_code == 503 + assert flat.json()["detail"]["error"] == "state_db_corrupt" + + # Status (what Desktop polls) and readiness now say the same thing. + status = client.get("/api/status").json() + assert status["components"]["storage"] == {"status": "degraded", "reason": "corrupt"} + assert status["overall"] == "degraded" + assert client.get("/api/profiles/sessions?profile=all").json()["storage"] == {"default": "corrupt"}