fix(state): align messages_fts external content with its indexed projection
`messages_fts` declared `content='messages'` while its triggers indexed only a
bounded prefix of every long tool row, with the boundary held in a
`fts_tool_full_content_high_water` state_meta marker that the migration, the
rebuild seeding and the in-place rebuild path all re-stamped. FTS5's strict
integrity check re-reads the external content source and compares it with the
stored token stream, so the two could never agree: one long tool row is enough to
make
INSERT INTO messages_fts(messages_fts, rank) VALUES('integrity-check', 1)
fail with `fts5: checksum mismatch for table "messages_fts"`. The delete/update
triggers re-evaluated the *new* marker, so they sent full content for a row whose
index held a prefix and left tokens behind that survived deleting the row.
Fix the class by removing the moving part. `messages_fts` now reads a view,
`messages_fts_src`, that computes exactly what the writers index - tool rows
truncated to FTS_TOOL_CONTENT_PREFIX_CHARS, everything else verbatim - through a
fixed per-row expression with no state_meta lookups, shared by the triggers, the
boundary sweep and the chunked insert.
- `messages_fts_src` view added; `messages_fts` external content points at it
- triggers, boundary sweep and chunked backfill all read that one projection
- `_stamp_fts_tool_high_water`, the marker seeding in `_seed_fts_rebuild_markers`
and the in-place rebuild stamp are gone, with
FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY
- FTS_STORAGE_VERSION 2 -> 3: an existing index is re-pointed and rebuilt ONCE by
`_migrate_misaligned_fts_source` under the shared cross-process rebuild
admission, because a v2 index holds token streams its old source cannot read
back; an empty index swaps shape in place, and legacy inline DBs are untouched
- search behaviour is unchanged: tool rows were already prefix-indexed, and
explicit tool search already used the stored-content LIKE path
This commit is contained in:
@@ -253,23 +253,27 @@ AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25
|
||||
# layout 0 (marker absent) with a working inline index until the user opts in.
|
||||
# 1 = v23 external-content layout with a tool-row-excluded trigram
|
||||
# 2 = trigram also excludes structured tool_calls JSON
|
||||
FTS_STORAGE_VERSION = 2
|
||||
# 3 = messages_fts source aligned to a stable projection view
|
||||
# (``messages_fts_src``): always-truncate tool rows to the prefix, no
|
||||
# moving high-water boundary. The external-content source now reads
|
||||
# back EXACTLY what the triggers indexed, so the rank=1
|
||||
# 'integrity-check' probe cannot drift from the stored index (the
|
||||
# recurring fts5 "checksum mismatch" / leaked-token failures).
|
||||
FTS_STORAGE_VERSION = 3
|
||||
|
||||
# Tool results are often multi-megabyte machine payloads. Index a useful
|
||||
# prefix for new tool rows instead of tokenizing the entire body while the
|
||||
# canonical message write holds SQLite's single writer lock. The high-water
|
||||
# marker lets upgraded databases retain the exact token stream already stored
|
||||
# for historical rows, so external-content delete/update commands stay valid
|
||||
# without an eager full-index rebuild.
|
||||
# Tool results are often multi-megabyte machine payloads. The base FTS index
|
||||
# stores only a bounded prefix of every tool row; tool rows are skipped by
|
||||
# default in search, and explicit tool-only search uses a LIKE fallback over
|
||||
# the full stored content, so no search capability is lost. The projection
|
||||
# below is STABLE — it depends only on the row being written, never on
|
||||
# mutable ``state_meta`` markers — which is what keeps the external-content
|
||||
# integrity checker and the trigger 'delete'/'update' commands in agreement
|
||||
# with the stored index forever.
|
||||
FTS_TOOL_CONTENT_PREFIX_CHARS = 8_192
|
||||
FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY = "fts_tool_full_content_high_water"
|
||||
|
||||
|
||||
def _fts_indexed_content_sql(alias: str) -> str:
|
||||
return f"""CASE WHEN {alias}.role = 'tool'
|
||||
AND {alias}.id > COALESCE((SELECT CAST(value AS INTEGER)
|
||||
FROM state_meta
|
||||
WHERE key = '{FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY}'), -1)
|
||||
THEN substr(COALESCE({alias}.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE {alias}.content END"""
|
||||
|
||||
@@ -691,12 +695,32 @@ CREATE INDEX IF NOT EXISTS idx_sessions_effective_activity
|
||||
# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation.
|
||||
# The two state_meta PK probes per write are negligible next to the FTS
|
||||
# insert itself.
|
||||
#
|
||||
# messages_fts_src: the base word index no longer reads raw `messages` as its
|
||||
# external content. Tool rows are indexed as a bounded prefix, so the index
|
||||
# must read that SAME projection back or FTS5's 'integrity-check' / 'delete'
|
||||
# commands disagree with the stored tokens and corrupt the index (the
|
||||
# recurring fts5 checksum-mismatch drift: the projection used to depend on a
|
||||
# moving state_meta high-water key). The view/trigger/backfill all share the
|
||||
# one expression in `_fts_indexed_content_sql` — a fixed per-row function
|
||||
# with no marker lookups — so the boundary can never move again.
|
||||
FTS_SQL = f"""
|
||||
-- Stable projection the base word index reads and writes through: the view
|
||||
-- computes EXACTLY what the triggers/backfill insert, so 'rebuild' and the
|
||||
-- integrity checker always agree with the stored index.
|
||||
CREATE VIEW IF NOT EXISTS messages_fts_src AS
|
||||
SELECT id,
|
||||
CASE WHEN role = 'tool'
|
||||
THEN substr(COALESCE(content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE content END AS content,
|
||||
tool_name, tool_calls
|
||||
FROM messages;
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
content,
|
||||
tool_name,
|
||||
tool_calls,
|
||||
content='messages',
|
||||
content='messages_fts_src',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
@@ -902,7 +926,9 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, content) VALUES (
|
||||
new.id,
|
||||
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
|
||||
COALESCE(CASE WHEN new.role = 'tool'
|
||||
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE new.content END, '')
|
||||
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
|
||||
);
|
||||
END;
|
||||
@@ -916,7 +942,9 @@ AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN
|
||||
DELETE FROM messages_fts WHERE rowid = old.id;
|
||||
INSERT INTO messages_fts(rowid, content) VALUES (
|
||||
new.id,
|
||||
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
|
||||
COALESCE(CASE WHEN new.role = 'tool'
|
||||
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE new.content END, '')
|
||||
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
|
||||
);
|
||||
END;
|
||||
@@ -932,7 +960,9 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts_trigram(rowid, content) VALUES (
|
||||
new.id,
|
||||
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
|
||||
COALESCE(CASE WHEN new.role = 'tool'
|
||||
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE new.content END, '')
|
||||
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
|
||||
);
|
||||
END;
|
||||
@@ -946,7 +976,9 @@ AFTER UPDATE OF content, tool_name, tool_calls, role ON messages BEGIN
|
||||
DELETE FROM messages_fts_trigram WHERE rowid = old.id;
|
||||
INSERT INTO messages_fts_trigram(rowid, content) VALUES (
|
||||
new.id,
|
||||
COALESCE({_FTS_NEW_INDEXED_CONTENT_SQL}, '')
|
||||
COALESCE(CASE WHEN new.role = 'tool'
|
||||
THEN substr(COALESCE(new.content, ''), 1, {FTS_TOOL_CONTENT_PREFIX_CHARS})
|
||||
ELSE new.content END, '')
|
||||
|| ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')
|
||||
);
|
||||
END;
|
||||
|
||||
Reference in New Issue
Block a user