fix(state): align messages_fts external content with its indexed projection

`messages_fts` declared `content='messages'` while its triggers indexed only a
bounded prefix of every long tool row, with the boundary held in a
`fts_tool_full_content_high_water` state_meta marker that the migration, the
rebuild seeding and the in-place rebuild path all re-stamped. FTS5's strict
integrity check re-reads the external content source and compares it with the
stored token stream, so the two could never agree: one long tool row is enough to
make

    INSERT INTO messages_fts(messages_fts, rank) VALUES('integrity-check', 1)

fail with `fts5: checksum mismatch for table "messages_fts"`. The delete/update
triggers re-evaluated the *new* marker, so they sent full content for a row whose
index held a prefix and left tokens behind that survived deleting the row.

Fix the class by removing the moving part. `messages_fts` now reads a view,
`messages_fts_src`, that computes exactly what the writers index - tool rows
truncated to FTS_TOOL_CONTENT_PREFIX_CHARS, everything else verbatim - through a
fixed per-row expression with no state_meta lookups, shared by the triggers, the
boundary sweep and the chunked insert.

- `messages_fts_src` view added; `messages_fts` external content points at it
- triggers, boundary sweep and chunked backfill all read that one projection
- `_stamp_fts_tool_high_water`, the marker seeding in `_seed_fts_rebuild_markers`
  and the in-place rebuild stamp are gone, with
  FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY
- FTS_STORAGE_VERSION 2 -> 3: an existing index is re-pointed and rebuilt ONCE by
  `_migrate_misaligned_fts_source` under the shared cross-process rebuild
  admission, because a v2 index holds token streams its old source cannot read
  back; an empty index swaps shape in place, and legacy inline DBs are untouched
- search behaviour is unchanged: tool rows were already prefix-indexed, and
  explicit tool search already used the stored-content LIKE path
This commit is contained in:
Kelly Griffin
2026-09-17 10:31:30 -04:00
committed by Teknium
parent 1b083b85a3
commit 42e97f3808
3 changed files with 118 additions and 71 deletions

View File

@@ -253,23 +253,27 @@ AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25
# layout 0 (marker absent) with a working inline index until the user opts in.
# 1 = v23 external-content layout with a tool-row-excluded trigram
# 2 = trigram also excludes structured tool_calls JSON
FTS_STORAGE_VERSION = 2
# 3 = messages_fts source aligned to a stable projection view
# (``messages_fts_src``): always-truncate tool rows to the prefix, no
# moving high-water boundary. The external-content source now reads
# back EXACTLY what the triggers indexed, so the rank=1
# 'integrity-check' probe cannot drift from the stored index (the
# recurring fts5 "checksum mismatch" / leaked-token failures).
FTS_STORAGE_VERSION = 3
# Tool results are often multi-megabyte machine payloads. Index a useful
# prefix for new tool rows instead of tokenizing the entire body while the
# canonical message write holds SQLite's single writer lock. The high-water
# marker lets upgraded databases retain the exact token stream already stored
# for historical rows, so external-content delete/update commands stay valid
# without an eager full-index rebuild.
# Tool results are often multi-megabyte machine payloads. The base FTS index
# stores only a bounded prefix of every tool row; tool rows are skipped by
# default in search, and explicit tool-only search uses a LIKE fallback over
# the full stored content, so no search capability is lost. The projection
# below is STABLE — it depends only on the row being written, never on
# mutable ``state_meta`` markers — which is what keeps the external-content
# integrity checker and the trigger 'delete'/'update' commands in agreement
# with the stored index forever.
FTS_TOOL_CONTENT_PREFIX_CHARS = 8_192
FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY = "fts_tool_full_content_high_water"
def _fts_indexed_content_sql(alias: str) -> str:
return f"""CASE WHEN {alias}.role = 'tool'
AND {alias}.id > COALESCE((SELECT CAST(value AS INTEGER)
FROM state_meta
WHERE key = '{FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY}'), -1)
THEN substr(COALESCE({alias}.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE {alias}.content END"""
@@ -691,12 +695,32 @@ CREATE INDEX IF NOT EXISTS idx_sessions_effective_activity
# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation.
# The two state_meta PK probes per write are negligible next to the FTS
# insert itself.
#
# messages_fts_src: the base word index no longer reads raw `messages` as its
# external content. Tool rows are indexed as a bounded prefix, so the index
# must read that SAME projection back or FTS5's 'integrity-check' / 'delete'
# commands disagree with the stored tokens and corrupt the index (the
# recurring fts5 checksum-mismatch drift: the projection used to depend on a
# moving state_meta high-water key). The view/trigger/backfill all share the
# one expression in `_fts_indexed_content_sql` — a fixed per-row function
# with no marker lookups — so the boundary can never move again.
FTS_SQL = f"""
-- Stable projection the base word index reads and writes through: the view
-- computes EXACTLY what the triggers/backfill insert, so 'rebuild' and the
-- integrity checker always agree with the stored index.
CREATE VIEW IF NOT EXISTS messages_fts_src AS
SELECT id,
CASE WHEN role = 'tool'
THEN substr(COALESCE(content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE content END AS content,
tool_name, tool_calls
FROM messages;
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
content,
tool_name,
tool_calls,
content='messages',
content='messages_fts_src',
content_rowid='id'
);
@@ -902,7 +926,9 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN
INSERT INTO messages_fts(rowid, content) VALUES (
new.id,
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
COALESCE(CASE WHEN new.role = 'tool'
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE new.content END, '')
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
);
END;
@@ -916,7 +942,9 @@ AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN
DELETE FROM messages_fts WHERE rowid = old.id;
INSERT INTO messages_fts(rowid, content) VALUES (
new.id,
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
COALESCE(CASE WHEN new.role = 'tool'
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE new.content END, '')
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
);
END;
@@ -932,7 +960,9 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(
CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN
INSERT INTO messages_fts_trigram(rowid, content) VALUES (
new.id,
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
COALESCE(CASE WHEN new.role = 'tool'
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE new.content END, '')
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
);
END;
@@ -946,7 +976,9 @@ AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN
DELETE FROM messages_fts_trigram WHERE rowid = old.id;
INSERT INTO messages_fts_trigram(rowid, content) VALUES (
new.id,
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
COALESCE(CASE WHEN new.role = 'tool'
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
ELSE new.content END, '')
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
);
END;