diff --git a/hermes_state_common.py b/hermes_state_common.py index e1347b0fac..74054a7f65 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -280,10 +280,10 @@ SCHEMA_VERSION = 28 AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 # FTS storage layout, tracked INDEPENDENTLY of SCHEMA_VERSION (state_meta -# ``fts_storage_version``): the schema version advances freely on open, but -# the FTS layout only changes when a DB is born fresh or explicitly optimized -# via ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker -# absent) with a working inline index. 1 = v23 external-content layout. +# ``fts_storage_version``): schema version advances freely on open, the FTS +# layout only changes when a DB is born fresh or explicitly optimized via +# ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker absent) +# with a working inline index; 1 = v23 external-content layout. FTS_STORAGE_VERSION = 1 # Cap on user-controlled FTS5 query input before sanitizer processing. @@ -573,13 +573,11 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash # While a background rebuild is pending, two state_meta keys define which rows # are IN the FTS indexes: H = fts_rebuild_high_water (MAX(messages.id) when the # old indexes were dropped), P = fts_rebuild_progress (highest backfilled id). -# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids, so post-drop rows -# are indexed live by the insert triggers); rows in (P, H] are not. -# -# Every trigger gates on that predicate: an FTS5 external-content 'delete' for -# a row NOT in the index corrupts it, and skipping one for an indexed row -# leaves a stale entry. With no rebuild pending both keys are absent and -# COALESCE makes the predicate a tautology. +# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids: post-drop rows are +# indexed live by the insert triggers); rows in (P, H] are not. Every trigger +# gates on that predicate: an external-content 'delete' for a row NOT in the +# index corrupts it, and skipping one for an indexed row leaves a stale entry. +# With no rebuild pending both keys are absent and COALESCE makes it a tautology. FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content, @@ -769,26 +767,22 @@ END; """ # ── Cross-process full-FTS-rebuild admission (single authority) ────────────── -# -# Several Hermes processes share one state.db. A full structural FTS rebuild -# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in -# ONE of them at a time: concurrent rebuilds have structurally corrupted -# state.db in production. This is the single admission authority for -# `rebuild_fts()`, `_rebuild_fts_indexes()` and `_recover_stale_fts()`. The -# chunked backfill (`fts_rebuild_step`) is deliberately NOT routed through it: -# it claims progress under SQLite transaction authority and is multi-process. -# +# Several Hermes processes share one state.db; a full structural FTS rebuild +# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in ONE +# of them at a time — concurrent rebuilds structurally corrupted state.db in +# production. Single authority for `rebuild_fts()`, `_rebuild_fts_indexes()` +# and `_recover_stale_fts()`; the chunked backfill (`fts_rebuild_step`) is +# deliberately NOT routed through it (it claims progress under SQLite +# transaction authority and is multi-process). # Semantics mirror `hermes_state._cross_process_repair_lock`: portable (msvcrt -# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. The kernel drops -# the lock when the holder dies UNLESS a forked child inherited the fd (flock -# rides the open file description), which holds it forever; so the holder's -# pid + start time are recorded under the lock and a provably-dead holder's -# lock is broken by unlinking and retaking on a fresh inode. Indeterminate -# liveness still defers. Lives here because the mixins cannot import +# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. flock rides the open +# file description, so a forked child that inherited the fd holds it forever +# after the holder dies; the holder's pid + start time are recorded under the +# lock and a provably-dead holder's lock is broken by unlinking and retaking on +# a fresh inode. Indeterminate liveness still defers. `.fts_rebuild.lock` +# is distinct from `.repair.lock` (schema surgery on an EXCLUSIVE offline +# connection, minutes in VACUUM). Lives here because mixins cannot import # hermes_state (cycle). -# -# `.fts_rebuild.lock` is distinct from `.repair.lock`: schema surgery -# runs on an EXCLUSIVE offline connection and may take minutes in VACUUM. logger = logging.getLogger("hermes_state") diff --git a/hermes_state_schema.py b/hermes_state_schema.py index 4796f21181..4f20e51a2c 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -33,9 +33,8 @@ _FTS_HOLDER_ESCALATE_ATTEMPTS = 3 _FTS_HOLDER_ESCALATE_SECONDS = 60.0 # In-process retry cadence for a deferred stale-FTS rebuild # (``retry_deferred_fts_recovery``): startup paid the full admission wait once; -# later retries are non-blocking probes so a live holder never stalls a -# long-lived writer. Each failed retry doubles the spacing up to the cap, so a -# permanent holder costs one deferral warning per hour, not per minute. +# later retries are non-blocking probes. Each failed retry doubles the spacing +# up to the cap, so a permanent holder costs one warning per hour, not per minute. _FTS_STALE_RETRY_SECONDS = 60.0 _FTS_STALE_RETRY_MAX_SECONDS = 3600.0 @@ -43,12 +42,11 @@ _FTS_STALE_RETRY_MAX_SECONDS = 3600.0 # in-memory SQLite database, so do it once per process. _READ_PROBE_STATEMENTS: Optional[tuple] = None -# The trigram triggers come ONLY from FTS_TRIGRAM_SQL / LEGACY_FTS_TRIGRAM_SQL, -# whose CREATE VIRTUAL TABLE needs the trigram tokenizer (SQLite >= 3.34); -# without it _ensure_fts_schema soft-fails that DDL and "all six present" is -# permanently unsatisfiable. Split the set so a trigger's absence is only -# measured against the DDL that can create it. Exhaustive and disjoint by -# construction; pinned by test_fts_trigger_subsets_match_the_ddl. +# Trigram triggers come ONLY from FTS_TRIGRAM_SQL / LEGACY_FTS_TRIGRAM_SQL, whose +# CREATE VIRTUAL TABLE needs the trigram tokenizer (SQLite >= 3.34); without it +# _ensure_fts_schema soft-fails that DDL and "all six present" is unsatisfiable. +# Split so a trigger's absence is measured only against the DDL that can create +# it. Exhaustive and disjoint by construction (test_fts_trigger_subsets_match_the_ddl). _FTS_TRIGRAM_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if "_trigram_" in n) _FTS_BASE_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if n not in _FTS_TRIGRAM_TRIGGERS) @@ -811,11 +809,10 @@ class SessionSchemaMixin: (row transforms) that cannot be expressed declaratively. """ # Startup-watchdog progress lease: on multi-GB state.db files the - # reconciliation + data migrations are legitimately slow and I/O-bound - # (near-zero CPU), which the watchdog's CPU fallback would misread as - # a parked deadlock. Single lease is deliberate (clamped to - # _MAX_LEASE_S=900): a genuinely wedged init delays supervisor respawn - # by up to the lease; per-chunk renewal isn't worth the complexity. + # reconciliation + data migrations are I/O-bound (near-zero CPU), which + # the watchdog's CPU fallback would misread as a parked deadlock. Single + # lease (clamped to _MAX_LEASE_S=900) is deliberate: a wedged init delays + # supervisor respawn by up to the lease; per-chunk renewal isn't worth it. report_startup_progress(600.0, phase="state_db_init_schema") cursor = self._conn.cursor() cursor.executescript(SCHEMA_SQL) @@ -953,15 +950,13 @@ class SessionSchemaMixin: pass if current_version < 22: self._migrate_v22_session_model_usage(cursor) - # v23: FTS storage redesign (external-content tables; inline v11 - # tables were ~75% of state.db on heavy installs). OPT-IN, NOT - # AUTOMATIC: the transition is disk-heavy (~2x transient) and long - # (hours on a 25 GB DB), so an existing install only gets a flag - # advertising it; `hermes sessions optimize-storage` performs it as - # a deliberate foreground operation. DECOUPLED VERSIONING: the FTS - # layout is tracked by the independent `fts_storage_version` - # marker, so schema_version still advances here and future - # migrations land for legacy-FTS users too. + # v23: FTS storage redesign (external-content tables; inline v11 tables + # were ~75% of state.db on heavy installs). OPT-IN, NOT AUTOMATIC: the + # transition is disk-heavy (~2x transient) and long (hours on 25 GB), so + # an existing install only gets a flag; `hermes sessions optimize-storage` + # performs it in the foreground. The FTS layout is tracked by the + # independent `fts_storage_version` marker, so schema_version still + # advances here and future migrations land for legacy-FTS users too. if current_version < 23 and fts5_available and self._db_has_legacy_inline_fts(cursor): self.set_meta("fts_optimize_available", "1", cursor=cursor) if current_version < 25: @@ -970,12 +965,11 @@ class SessionSchemaMixin: # for partially migrated or externally written rows. self._dedupe_legacy_system_prompts(cursor) - # Stamp the FTS layout version (fresh/optimized DBs) so the main - # version can always advance; a legacy DB keeps its absent/0 marker - # until optimize-storage runs. An INTERRUPTED optimize (rebuild - # markers, trash tables, or an empty external index against - # non-empty messages) is NOT stamped: the marker is the source of - # truth for "fully optimized" and keeps the resume offer alive. + # Stamp the FTS layout version (fresh/optimized DBs); a legacy DB keeps + # its absent/0 marker until optimize-storage runs. An INTERRUPTED optimize + # (rebuild markers, trash tables, or an empty external index against + # non-empty messages) is NOT stamped: the marker is the source of truth + # for "fully optimized" and keeps the resume offer alive. if ( fts5_available and not self._db_has_legacy_inline_fts(cursor) @@ -987,10 +981,9 @@ class SessionSchemaMixin: ): self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor) - # Advance schema_version — deliberately NOT gated on the FTS opt-in - # (that would block every future migration for a user who never - # optimizes). FTS5 unavailable is the one skip: we can't have - # created the current FTS objects, so claiming current would lie. + # Advance schema_version — deliberately NOT gated on the FTS opt-in (that + # would block every future migration for a user who never optimizes). + # FTS5 unavailable is the one skip: claiming current would lie. if current_version < SCHEMA_VERSION and fts_migrations_complete and fts5_available: cursor.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION,)) diff --git a/hermes_state_search.py b/hermes_state_search.py index e534f01265..0b6faaa78c 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -1201,13 +1201,12 @@ class SessionSearchMixin: except sqlite3.OperationalError as exc: logger.debug("Unindexed-gap supplement skipped: %s", exc) - # unicode61 puts no boundary between Latin and adjacent CJK - # ("修改youer服务端" is one token, so MATCH "youer" misses). On a - # zero-result Latin miss retry the substring-capable indexes: cjk - # first (splits Latin off CJK: exact ranked match), then trigram - # (needs >=3-char tokens). Gated on a miss so successful searches keep - # their ranking; trade-off: "cat" may then match "concatenate". - # Skipped for role='tool' (both indexes exclude tool rows). + # unicode61 puts no boundary between Latin and adjacent CJK ("修改youer服务端" + # is one token, so MATCH "youer" misses). On a zero-result Latin miss retry + # the substring-capable indexes: cjk first (exact ranked match), then + # trigram (>=3-char tokens). Gated on a miss so hits keep their ranking + # ("cat" may then match "concatenate"). Skipped for role='tool' (both + # indexes exclude tool rows). if not matches and not is_cjk and not wants_tool_rows: fb_query = _quote_fts_tokens(query.strip('"').strip()) if self._fts_cjk_available: