fix(state): drop orphan FTS5 shadow tables per family on SessionDB open

`sqlite3 state.db .recover` re-emits the FTS5 shadow tables (messages_fts_data,
_idx, _docsize, _config, _content) as ordinary tables but cannot re-emit the
CREATE VIRTUAL TABLE row. The next SessionDB open ran the FTS DDL in
_ensure_fts_schema and died with "fts5: error creating shadow table
messages_fts_data: table 'messages_fts_data' already exists", so a recovered
database was unusable until someone hand-dropped the shadows.

_init_fts now runs _drop_orphan_fts_shadow_tables before any FTS DDL. It is
per-family and exact-name scoped: a family's shadows are dropped only when its
own vtable row is absent from sqlite_master (type='table' AND sql LIKE
'CREATE VIRTUAL TABLE%'), so a healthy messages_fts_trigram survives a
base-family repair untouched. A repaired base/trigram family is then treated
like a missing-trigger repair and rebuilt from the canonical messages table
under the cross-process rebuild admission; the shadows are derived index
state, nothing is lost.

Live repro: real `sqlite3 x.db .recover | sqlite3 y.db` on sqlite 3.50.4
keeps the vtable rows (the shell emits CREATE VIRTUAL TABLE), so the
deterministic fixture removes the vtable row via writable_schema leaving the
shadows behind: BEFORE OperationalError on open; AFTER opens, fts_enabled,
MATCH returns every message, trigram sqlite_master rowids unchanged.

Salvaged from PR #56824 (intent applied onto the current hermes_state_fts /
hermes_state_schema siblings). The ownership-safety point (never touch a live
family's shadows) was raised by @ggoldani in #103868 / #103840.

Refs #103840
Refs #56815
Co-authored-by: ggoldani <ggoldani@users.noreply.github.com>

hermes_state_fts.py

@@ -6,6 +6,7 @@ import logging
 import os
 import sqlite3
 from pathlib import Path
+from typing import Sequence
 
 from hermes_constants import get_hermes_home
 from hermes_state_common import FTS_CJK_STALE_KEY, FTS_STALE_KEY, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS
@@ -122,6 +123,47 @@ def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
         return False
 
 
+# FTS5 shadow tables the virtual-table engine owns. `sqlite3 .recover` re-emits them
+# as ordinary tables but cannot re-emit the CREATE VIRTUAL TABLE row, so every later
+# CREATE VIRTUAL TABLE fails with "fts5: error creating shadow table <name>: table
+# already exists" until the orphans are dropped (#103840).
+_FTS5_SHADOW_SUFFIXES = ("content", "data", "docsize", "idx", "config")
+
+
+def _drop_orphan_fts_shadow_tables(cursor: sqlite3.Cursor, families: Sequence[str]) -> list[str]:
+    """Drop, per family, shadow tables whose virtual table row is absent from sqlite_master.
+
+    Matches exact shadow names only (never a prefix LIKE, so the base family cannot reach
+    ``messages_fts_trigram_*``) and leaves a family alone whenever its vtable is live. The
+    shadows are derived index state; the caller recreates and rebuilds from ``messages``.
+    Returns the families that were repaired.
+    """
+    repaired: list[str] = []
+    for family in families:
+        vtable_live = cursor.execute(
+            "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? "
+            "AND sql LIKE 'CREATE VIRTUAL TABLE%'",
+            (family,),
+        ).fetchone()
+        if vtable_live:
+            continue
+        shadows = [f"{family}_{suffix}" for suffix in _FTS5_SHADOW_SUFFIXES]
+        orphans = [row[0] for row in cursor.execute(
+            f"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ({','.join('?' for _ in shadows)})",
+            shadows,
+        ).fetchall()]
+        if not orphans:
+            continue
+        for name in orphans:
+            cursor.execute(f'DROP TABLE "{name}"')
+        logger.warning(
+            "Dropped orphan FTS5 shadow tables of %s (%s); the index is recreated from messages",
+            family, ", ".join(orphans),
+        )
+        repaired.append(family)
+    return repaired
+
+
 class SessionFtsSetupMixin:
     """FTS table/trigger lifecycle shared by schema init, optimize and the write path."""
 

hermes_state_schema.py

@@ -26,6 +26,7 @@ from hermes_state_common import (
     LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL,
     SCHEMA_VERSION, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS, _ephemeral_child_sql, _sql_json_extract, fts_rebuild_admission,
 )
+from hermes_state_fts import _drop_orphan_fts_shadow_tables
 from hermes_state_holders import _read_proc_argv
 
 # Pre-split logger identity so log filtering/capture is unchanged.
@@ -1129,6 +1130,12 @@ class SessionSchemaMixin:
         OPT-IN v23 boundary: a legacy v22 inline install keeps its inline schema + triggers
         (the v23 DDL would create the trigram source VIEW and leave a mixed state)."""
         legacy_fts = self._db_has_legacy_inline_fts(cursor)
+        # A `.recover`-restored image keeps the shadow tables but not the vtable rows; the DDL
+        # below would fail on the first shadow. Drop only orphaned families, then rebuild the
+        # recreated (empty) index like a missing-trigger repair (#103840).
+        orphan_repaired = _drop_orphan_fts_shadow_tables(
+            cursor, ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"),
+        )
         if not self._fts_stale:
             self._migrate_bounded_tool_fts_triggers(cursor, legacy=legacy_fts)
         if self._fts_stale:
@@ -1142,8 +1149,10 @@ class SessionSchemaMixin:
             # Measure before any DDL. Publishing missing base triggers before rebuild admission lets
             # another process write through an index whose bootstrap/repair has no owner (#105790).
             base_triggers_missing = self._fts_triggers_missing(cursor, _FTS_BASE_TRIGGERS) or getattr(
-                self, "_fts_tool_prefix_migration_requires_rebuild", False)
-            trigram_triggers_missing = self._fts_triggers_missing(cursor, _FTS_TRIGRAM_TRIGGERS)
+                self, "_fts_tool_prefix_migration_requires_rebuild", False) or "messages_fts" in orphan_repaired
+            trigram_triggers_missing = (
+                self._fts_triggers_missing(cursor, _FTS_TRIGRAM_TRIGGERS) or "messages_fts_trigram" in orphan_repaired
+            )
 
             def ensure_and_rebuild() -> None:
                 self._fts_enabled = self._ensure_fts_schema(cursor, "messages_fts", base_sql)

tests/state/test_fts_orphan_shadow_repair.py

@@ -0,0 +1,67 @@
+"""#103840: a ``sqlite3 .recover`` restore re-emits FTS5 shadow tables as ordinary tables
+but cannot re-emit the ``CREATE VIRTUAL TABLE`` row. The next SessionDB open then failed
+in ``_ensure_fts_schema`` with "fts5: error creating shadow table messages_fts_data: table
+already exists". Only families whose vtable row is absent may be repaired; a healthy family's
+shadows must survive untouched.
+"""
+
+import sqlite3
+
+from hermes_state import SessionDB
+
+
+def _orphan_family(db_path, family: str) -> None:
+    """Emulate the ``.recover`` residue for one family: vtable row gone, shadows kept."""
+    raw = sqlite3.connect(db_path)
+    raw.isolation_level = None
+    raw.execute("PRAGMA writable_schema=ON")
+    raw.execute(
+        "DELETE FROM sqlite_master WHERE name = ? AND sql LIKE 'CREATE VIRTUAL TABLE%'", (family,),
+    )
+    version = raw.execute("PRAGMA schema_version").fetchone()[0]
+    raw.execute(f"PRAGMA schema_version={version + 1}")
+    raw.execute("PRAGMA writable_schema=OFF")
+    raw.close()
+
+
+def _fts_master_rows(db_path, prefix: str) -> list:
+    raw = sqlite3.connect(db_path)
+    try:
+        return raw.execute(
+            "SELECT rowid, type, name FROM sqlite_master WHERE name LIKE ? ESCAPE '\\' ORDER BY rowid",
+            (prefix.replace("_", "\\_") + "%",),
+        ).fetchall()
+    finally:
+        raw.close()
+
+
+def test_orphaned_base_family_is_repaired_and_healthy_trigram_untouched(tmp_path):
+    db_path = tmp_path / "state.db"
+    db = SessionDB(db_path=db_path)
+    db.create_session("s1", source="cli", model="m")
+    for i in range(3):
+        db.append_message("s1", "user", f"recovered orphan {i}")
+    db.close()
+
+    _orphan_family(db_path, "messages_fts")
+    trigram_before = _fts_master_rows(db_path, "messages_fts_trigram")
+    assert trigram_before, "fixture needs a live trigram family"
+    raw = sqlite3.connect(db_path)
+    orphan_shadows = raw.execute(
+        "SELECT count(*) FROM sqlite_master WHERE name IN ('messages_fts_data', 'messages_fts_config')"
+    ).fetchone()[0]
+    raw.close()
+    assert orphan_shadows == 2, "fixture must leave the base shadows behind"
+
+    reopened = SessionDB(db_path=db_path)
+    try:
+        assert reopened._fts_enabled is True
+        with reopened._lock:
+            hits = reopened._conn.execute(
+                "SELECT count(*) FROM messages_fts WHERE messages_fts MATCH 'orphan'"
+            ).fetchone()[0]
+        assert hits == 3, "recreated index must be rebuilt from the canonical messages table"
+    finally:
+        reopened.close()
+    # Same rowids: the healthy family was neither dropped nor recreated.
+    assert _fts_master_rows(db_path, "messages_fts_trigram") == trigram_before