From d15c61b5dcfdf008ed181d86cc05000a64df9695 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:09:00 -0700 Subject: [PATCH] refactor(state): split SessionDB into domain mixins and free-function modules; unify SQL boilerplate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hermes_state.py 17,220 -> 6,442 LOC. Behavior-neutral: every moved body is AST-identical to the original, verified per extraction. SessionDB core - _write_sql / _write_rowcount / _read_one / _read_all replace ~120 copies of the `def _do(conn): conn.execute(...)` + `_execute_write(_do)` and `with self._read_ctx() as conn: row = conn.execute(...).fetchone()` shapes. - _set_lineage_column replaces four copies of the recursive compression-lineage UPDATE (archived / pinned / hidden / last_read_at). - _read_session_number unifies the three compression counter readers. - Dead (zero refs repo-wide): restore_rewound, delete_gateway_routing_entries, _is_duplicate_replayed_user_message, SessionPortabilityMixin.get_first_assistant_text. New mixins bound onto SessionDB via the MRO (logger name stays "hermes_state"): hermes_state_messages SessionMessagesMixin 48 methods hermes_state_compression SessionCompressionMixin 30 hermes_state_gateway SessionGatewayMixin 26 hermes_state_maintenance SessionMaintenanceMixin 13 hermes_state_usage SessionUsageMixin 12 hermes_state_titles SessionTitlesMixin 13 hermes_state_telegram SessionTelegramTopicsMixin 11 Origin-internal symbols resolve through a lazy `from hermes_state import ...` inside the few methods that need them (no import cycle). New free-function modules, every name re-imported into hermes_state so `hermes_state.` (and test monkeypatches on it) keep working; intra-module calls to patched helpers go through the lazy origin import: hermes_state_repair repair/backup/preflight (43 defs) hermes_state_wal journal-mode / PRAGMA policy (33 defs) hermes_state_dbfile header probes, zeroed-db quarantine, stats, holders (21 defs) Existing mixins: search — shared FTS MATCH/LIKE builders, unified rebuild status/step/finish engines, state_meta helpers; schema — one legacy/v23 FTS init branch, shared _live_pk_columns, Row/tuple dual access dropped; portability — shared _PREVIEW_RAW_SUBQUERY_SQL and _rich_row; common — single stat_db_file_identity (was 3 copies), AUTO_VACUUM_MIN_FREELIST_RATIO. Docstrings/comments hand-compacted (AST-identical) keeping every invariant, ordering rule, failure mode and WHY. Schema SQL, migration order and PRAGMAs untouched. test_repair_path_has_no_bare_connects repointed to hermes_state_repair. --- hermes_state.py | 11569 +--------------------- hermes_state_common.py | 461 +- hermes_state_compression.py | 1030 ++ hermes_state_dbfile.py | 674 ++ hermes_state_gateway.py | 1005 ++ hermes_state_maintenance.py | 699 ++ hermes_state_messages.py | 2642 +++++ hermes_state_portability.py | 419 +- hermes_state_registry.py | 186 +- hermes_state_repair.py | 1639 +++ hermes_state_schema.py | 886 +- hermes_state_search.py | 1922 ++-- hermes_state_telegram.py | 529 + hermes_state_titles.py | 305 + hermes_state_usage.py | 597 ++ hermes_state_wal.py | 821 ++ tests/test_state_db_write_durability.py | 8 +- 17 files changed, 11604 insertions(+), 13788 deletions(-) create mode 100644 hermes_state_compression.py create mode 100644 hermes_state_dbfile.py create mode 100644 hermes_state_gateway.py create mode 100644 hermes_state_maintenance.py create mode 100644 hermes_state_messages.py create mode 100644 hermes_state_repair.py create mode 100644 hermes_state_telegram.py create mode 100644 hermes_state_titles.py create mode 100644 hermes_state_usage.py create mode 100644 hermes_state_wal.py diff --git a/hermes_state.py b/hermes_state.py index e374fea4e0..d43979fea2 100644 --- a/hermes_state.py +++ b/hermes_state.py @@ -16,7 +16,6 @@ Key design decisions: import asyncio import atexit -import contextlib import errno import hashlib import json @@ -26,7 +25,6 @@ import queue import random import re import sqlite3 -import struct import sys import threading import time @@ -36,7 +34,6 @@ from collections import deque from contextlib import contextmanager from pathlib import Path -from agent.memory_manager import sanitize_context from agent.session_activity import ActivityProvenance from agent.message_sanitization import _sanitize_surrogates # Intrinsic persistence marker stamped on message dicts that are known-durable @@ -45,22 +42,14 @@ from agent.message_sanitization import _sanitize_surrogates # transitive dependency via hermes_state_common). run_agent keeps its own # predating copy — hermes_state cannot import run_agent (circular) — guarded # by test_marker_constant_in_sync. -from agent.context_compressor import ( +from agent.context_compressor import ( # noqa: F401 (re-exported; tests import it from here) _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY, ) -from agent.skill_commands import ( - SKILL_EXCERPT_JOINT, - SKILL_SCAFFOLD_SQL_LIKE, - describe_skill_invocation, -) from hermes_constants import get_hermes_home -from hermes_startup_watchdog import report_startup_progress -from hermes_cli.sqlite_runtime import ( - is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable, -) -from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, TypeVar, cast +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, TypeVar, cast from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) + AUTO_VACUUM_MIN_FREELIST_RATIO, _BRANCH_CHILD_SQL, _COMPRESSION_CHILD_SQL, _FTS_CJK_TRIGGERS, @@ -101,9 +90,120 @@ from hermes_state_common import ( # noqa: F401 (re-exported for back-compat) _describe_lock_holder, _read_lock_holder_record, is_advisory_lock_contention, + stat_db_file_identity as _stat_db_file_identity, ) from hermes_state_portability import SessionPortabilityMixin +from hermes_state_telegram import SessionTelegramTopicsMixin, _normalize_telegram_topic_profile_name # noqa: F401 (re-exported for back-compat) from hermes_state_schema import SessionSchemaMixin +from hermes_state_dbfile import ( # noqa: F401 (re-exported; tests patch hermes_state.) + _HEADER_PROBE_FDS, + _HEADER_PROBE_LOCK, + _HERMES_CMDLINE_MARKERS, + _RETIRED_HEADER_PROBE_FDS, + _canonical_sqlite_path, + _concrete_state_db_holder_pids, + _connect_tracked_db, + _is_inactive_orphan_desktop_holder, + _looks_like_hermes, + _pread_db_header, + _read_proc_cmdline, + _read_sqlite_application_id, + _stat_sqlite_sidecar_identity, + _watched_sqlite_sidecar_paths, + collect_state_db_stats, + count_db_holders, + is_zeroed_state_db, + iter_deleted_sqlite_sidecar_holders, + quarantine_cross_process_lock, + quarantine_zeroed_state_db, + refuse_deleted_wal_generation, +) +from hermes_state_messages import SessionMessagesMixin +from hermes_state_wal import ( # noqa: F401 (re-exported; tests patch hermes_state.) + WalUnsupportedError, + _SYNCHRONOUS_FULL, + _SYNCHRONOUS_LEVELS, + _SYNCHRONOUS_NAMES, + _WAL_INCOMPAT_MARKERS, + _WAL_SIZE_LIMIT_BYTES, + _apply_delete_for_wal_reset_bug, + _apply_macos_checkpoint_barrier, + _apply_synchronous_pragma, + _apply_wal_size_limit, + _database_has_content, + _delete_overridden_warned_lock, + _delete_overridden_warned_paths, + _enforce_macos_synchronous_full, + _journal_upgrade_warned_lock, + _journal_upgrade_warned_paths, + _log_configured_delete_overridden_once, + _log_journal_mode_upgrade_once, + _log_wal_fallback_once, + _log_wal_reset_bug_once, + _on_disk_journal_mode, + _set_journal_mode_no_wait, + _wal_fallback_warned_lock, + _wal_fallback_warned_paths, + _wal_reset_bug_warned_lock, + _wal_reset_bug_warned_paths, + _wal_reset_repair_hint, + apply_database_pragmas, + apply_wal_with_fallback, + is_sqlite_wal_reset_vulnerable, + resolve_journal_mode, + resolve_synchronous_level, + sqlite_source_id, +) +from hermes_state_repair import ( # noqa: F401 (re-exported; tests patch hermes_state.) + _DB_SIDECAR_SUFFIXES, + _FINGERPRINT_SAMPLE_BYTES, + _FINGERPRINT_VOLATILE_HEADER_RANGES, + _MAX_MALFORMED_BACKUPS, + _MAX_PERSISTENT_REPAIR_ATTEMPTS, + _REPAIR_BACKUP_FREE_FRACTION, + _REPAIR_BACKUP_MIN_FREE_BYTES, + _REPAIR_LOCK_POLL_SECONDS, + _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, + _backup_content_identity, + _backup_db_file, + _bump_schema_cookie, + _claim_repair_attempt, + _connect_repair_durable, + _copy_database_snapshot, + _cross_process_repair_lock, + _db_fingerprint, + _db_opens_cleanly, + _exclusive_repair_db_guard, + _existing_malformed_backups, + _live_writer_holds_db, + _mask_volatile_header, + _persistent_repair_attempts_exhausted, + _persistent_repair_exhausted_error, + _probe_journal_mode_for_repair, + _prune_malformed_backups, + _read_repair_ledger, + _reapply_durability_barriers, + _record_repair_outcome, + _release_auto_maintenance_lock, + _repair_backup_headroom_bytes, + _repair_failure_consumes_attempt, + _repair_ledger_path, + _repair_scratch_space_error, + _repair_snapshot_timeout_seconds, + _repair_state_db_schema_locked, + _restore_journal_mode_after_repair, + _run_repair_strategies, + _try_acquire_auto_maintenance_lock, + _unlink_db_triple, + apply_durability_barriers, + preflight_db_writability, + repair_state_db_schema, +) +from hermes_state_titles import SessionTitlesMixin +from hermes_state_usage import SessionUsageMixin +from hermes_state_maintenance import SessionMaintenanceMixin +from hermes_state_gateway import SessionGatewayMixin +from hermes_state_compression import SessionCompressionMixin from hermes_state_search import SessionSearchMixin try: # Hard dependency, but tolerate scaffold-phase imports before pip install. @@ -116,13 +216,6 @@ logger = logging.getLogger(__name__) MAX_SAFE_RESUME_MESSAGES = 20_000 MAX_SAFE_EXPORT_MESSAGES = 20_000 -# Auto-maintenance only VACUUMs when at least this fraction of the database -# file is reclaimable (``PRAGMA freelist_count / PRAGMA page_count``). Below -# it a full rewrite costs more I/O than it returns — pruning a handful of small -# sessions on a dense multi-GB state.db should never rewrite the whole file to -# reclaim a few MB (#54189). Composes with ``min_vacuum_interval_days``. -AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 - def _configured_transcript_limit(key: str, fallback: int) -> int: """Resolve a transcript safety limit from config at call time. @@ -943,45 +1036,6 @@ def _ensure_test_isolation(db_path: Path) -> None: "its environment." ) -# --------------------------------------------------------------------------- -# WAL-compatibility fallback -# --------------------------------------------------------------------------- -# SQLite's WAL mode requires shared-memory (mmap) coordination and fcntl -# byte-range locks that don't reliably work on network filesystems (NFS, -# SMB/CIFS, some FUSE mounts, WSL1). Upstream documents this explicitly: -# https://www.sqlite.org/wal.html#sometimes_queries_return_sqlite_busy_in_wal_mode -# -# On those filesystems ``PRAGMA journal_mode=WAL`` raises -# ``sqlite3.OperationalError: locking protocol`` (SQLITE_PROTOCOL). If we -# propagate that, every feature backed by state.db / kanban.db breaks -# silently — /resume, /title, /history, /branch, kanban dispatcher, etc. -# -# ZFS is a separate case: its COW + mmap semantics can corrupt the WAL -# shared-memory (-shm) file under concurrent connection bursts, presenting -# as ``disk I/O error`` rather than ``locking protocol``. -# -# Instead, fall back to ``journal_mode=DELETE`` (the pre-WAL default) which -# works on NFS and ZFS. Concurrency drops — concurrent readers are blocked -# during a write — but the feature works. -# -# Separately, SQLite's WAL-reset bug can corrupt multi-process WAL databases -# on unfixed library builds (issue #69784). See: -# https://sqlite.org/wal.html#walresetbug -# Fixed in 3.51.3+ with backports 3.50.7 and 3.44.6. On vulnerable builds we -# refuse to *enable* WAL for fresh / non-WAL databases (prefer DELETE). We do -# NOT live-downgrade an on-disk WAL database — other gateway/cron/worker -# connections may still hold it open, and flipping journal_mode under them is -# unsafe (same invariant as the NFS path below). -_WAL_INCOMPAT_MARKERS = ( - "locking protocol", # SQLITE_PROTOCOL on NFS/SMB - "not authorized", # Some FUSE mounts block WAL pragma outright - "disk i/o error", # ZFS SHM corruption under concurrent connections -) - -# Upper bound for the write-ahead log. SQLite defaults to -1 (unlimited), -# which lets state.db-wal keep the high-water mark of the largest-ever -# transaction forever. See _apply_wal_size_limit(). -_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB # Last SessionDB() init error, per-process. Surfaced in /resume and # related slash-command error strings so users know WHY the DB is @@ -992,20 +1046,6 @@ _WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB _last_init_error: Optional[str] = None _last_init_error_lock = threading.Lock() -# Paths for which we've already logged a WAL-fallback WARNING. Without -# this, kanban_db.connect() (called on every kanban operation — see -# hermes_cli/kanban_db.py for ~30 call sites) would re-log the same -# filesystem-incompat warning on every connection, filling errors.log. -_wal_fallback_warned_paths: set[str] = set() -_wal_fallback_warned_lock = threading.Lock() - -# Dedup WARNING for the WAL-reset vulnerability fallback (issue #69784). -_wal_reset_bug_warned_paths: set[str] = set() -_wal_reset_bug_warned_lock = threading.Lock() - -# Dedup ERROR for the "configured delete overridden by on-disk WAL" warning. -_delete_overridden_warned_paths: set[str] = set() -_delete_overridden_warned_lock = threading.Lock() def _set_last_init_error(msg: Optional[str]) -> None: """Record (or clear) the most recent state.db init failure. @@ -1141,17 +1181,6 @@ def _strip_stale_tool_call_markers( return messages -def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str: - """Normalize profile namespace for Telegram topic-mode tables. - - Empty / missing values map to ``\"default\"`` so non-multiplexed gateways - keep a single namespace. Multiplexed callers must pass the *routed* - profile (``source.profile``), never the process-global active profile. - """ - name = str(profile_name or "").strip() - return name if name else "default" - - def format_session_db_unavailable(prefix: str = "Session database not available") -> str: """Format a user-facing 'session DB unavailable' message with cause. @@ -1175,900 +1204,6 @@ def format_session_db_unavailable(prefix: str = "Session database not available" return f"{prefix}: {cause}{hint}." -def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]: - """Read the journal mode from the SQLite DB header on disk. - - Returns the mode string (e.g. ``"wal"``, ``"delete"``), or ``None`` - if the value cannot be determined (new DB, or PRAGMA read failed). - - A PRAGMA read can fail transiently with ``disk i/o error`` on - virtualized block devices (XFS on cloud hosts). Treating that as - "mode unknown" pushes callers onto their fail-closed unknown-mode - branch even though the on-disk mode is perfectly readable a few - milliseconds later. Retry the read a few times before giving up: - transient EIO clears, deterministic unsupported-filesystem errors do - not. ``None`` is still returned on final failure so the caller's - existing "unknown → refuse to downgrade" logic applies. - """ - last_exc: Optional[Exception] = None - for _ in range(4): - try: - row = conn.execute("PRAGMA journal_mode").fetchone() - except sqlite3.OperationalError as exc: - last_exc = exc - if "disk i/o error" not in str(exc).lower(): - return None - time.sleep(0.05) - continue - if row is None: - return None - mode = row[0] - if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes - try: - mode = mode.decode("ascii") - except UnicodeDecodeError: - return None - return str(mode).strip().lower() if mode is not None else None - if last_exc is not None: - logger.debug( - "_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc - ) - return None - - -def _apply_wal_size_limit(conn: sqlite3.Connection) -> None: - """Bound the WAL so it returns space to the OS after big transactions. - - SQLite's default ``journal_size_limit`` is -1 (unlimited): after a - checkpoint the WAL file is *reused in place* and never truncated, so - ``state.db-wal`` permanently retains the high-water mark of the largest - transaction ever run against it. - - A single bulk operation is enough to strand gigabytes. Observed on a - 3.0 GB ``state.db``: ``hermes sessions optimize`` (FTS merge + VACUUM) - rewrites every page through the WAL, leaving a **3.07 GB** - ``state.db-wal`` sitting next to the database indefinitely — the host - went from 6.9 GB free to 772 MB (100% full) and stayed there, because - nothing shrinks the WAL back down. An explicit - ``PRAGMA wal_checkpoint(TRUNCATE)`` reclaimed the full 3.07 GB, which - confirms the space was pure slack rather than live data. - - That also makes the maintenance command self-defeating on exactly the - databases that need it most: the larger the DB, the larger the WAL it - strands, so ``optimize`` can consume more disk than it frees. - - ``journal_size_limit`` makes SQLite truncate the WAL back to the limit - at each checkpoint. 64 MiB is comfortably above normal transaction - sizes (so steady-state commits never pay a truncate) while capping the - stranded slack at a bounded, predictable figure. - - ``hermes_cli/kanban_db.py`` already bounds its WAL growth with - ``wal_autocheckpoint=100``; the session store — by far the larger - database — had no equivalent. - - Best-effort: never raises. A failure here only costs disk slack, and - must not prevent the database from opening. - """ - try: - conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}") - except sqlite3.OperationalError as exc: # pragma: no cover - defensive - logger.debug("journal_size_limit not applied: %s", exc) - - -def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None: - """Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere). - - On Darwin, ``synchronous=FULL`` (the WAL default) issues a plain - ``fsync()``, which Apple documents does *not* guarantee that data - has reached stable storage or that writes are not reordered — see - the ``fsync(2)`` man page. SQLite's WAL corruption-safety guarantee - assumes the OS honors the fsync write barrier; macOS does not unless - the app uses ``F_FULLFSYNC``. - - During a launchd *system* shutdown/reboot the OS page cache is - dropped (effectively a power-loss event for in-flight pages), so a - WAL checkpoint whose ``fsync()`` "reported" durable may never have - hit the platter — corrupting ``state.db`` with a malformed image. - This is the trigger in issue #30636 ("SIGTERM during launchd - shutdown under high load"), distinct from a plain in-session kill - (which the page cache survives and SQLite recovers from). - - ``checkpoint_fullfsync=1`` forces an ``F_FULLFSYNC`` barrier only at - checkpoint boundaries — where WAL frames land in the main DB — so the - cost amortizes to roughly +0.1 ms/commit (vs ~+4 ms for the broader - ``fullfsync=1`` that flushes on every commit's WAL sync). Guarded by - ``sys.platform == "darwin"`` because ``F_FULLFSYNC`` is macOS-only; - on other platforms the PRAGMA is a no-op, so we skip it entirely. - - Best-effort: never raises. - """ - if sys.platform != "darwin": - return - try: - conn.execute("PRAGMA checkpoint_fullfsync=1") - except sqlite3.OperationalError: - pass - - -def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: - """Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption. - - On Darwin, the default ``synchronous=NORMAL`` only calls ``fsync()``, - which Apple's fsync(2) man page explicitly states does *not* guarantee - data-on-platter or write-ordering. During a WAL checkpoint race with - process termination (e.g., launchd shutdown), this can leave the main - DB with half-written btree pages → ``btreeInitPage error 11``. - - WAL mode's durability guarantee assumes the OS honors fsync barriers; - macOS does not unless we explicitly set ``synchronous=FULL``, which issues - a real ``fsync()`` on every transaction commit. The ``F_FULLFSYNC`` - barrier at checkpoint boundaries is handled separately by - :func:`_apply_macos_checkpoint_barrier`. - - This function is called after any successful WAL activation (either - from ``apply_wal_with_fallback()`` setting a fresh WAL or when probing - an existing WAL mode). It ensures macOS connections always use FULL - synchronous mode, even if a prior connection set ``synchronous=NORMAL``. - - Best-effort: never raises. - """ - if sys.platform != "darwin": - return - try: - conn.execute("PRAGMA synchronous=FULL") - except sqlite3.OperationalError: - pass - - -def is_sqlite_wal_reset_vulnerable( - version_info: Optional[tuple] = None, -) -> bool: - """Return True when the linked SQLite library has the WAL-reset bug. - - Upstream documents the bug in versions 3.7.0 through 3.51.2, fixed in - 3.51.3+, with backports 3.50.7 and 3.44.6: - https://sqlite.org/wal.html#walresetbug - - Pre-WAL libraries (< 3.7.0) cannot hit the race and are treated as safe. - """ - info = version_info if version_info is not None else sqlite3.sqlite_version_info - return _is_sqlite_wal_reset_vulnerable(info) - - -def sqlite_source_id() -> str: - """Return ``sqlite_source_id()``, or an empty string when unavailable.""" - try: - conn = sqlite3.connect(":memory:") - try: - row = conn.execute("SELECT sqlite_source_id()").fetchone() - finally: - conn.close() - except sqlite3.Error: - return "" - if not row or row[0] is None: - return "" - return str(row[0]) - - -def _database_has_content(conn: sqlite3.Connection) -> bool: - """Return whether the database file already holds pages. - - Used to tell an EXISTING database apart from a brand-new one before - rewriting its journal mode. ``PRAGMA page_count`` is a header read, so - this costs nothing and takes no lock. - - Fail-quiet: any error, or a database we cannot measure, answers False. - The only caller uses this to decide whether to emit a warning, and a - warning that fires when the answer is unknown would fire on every fresh - database -- precisely the case where there is provably no operator choice - being overwritten. - """ - try: - row = conn.execute("PRAGMA page_count").fetchone() - except sqlite3.Error: - return False - if not row or row[0] is None: - return False - try: - return int(row[0]) > 0 - except (TypeError, ValueError): - return False - - -def resolve_journal_mode() -> str: - """Return the configured journal mode (``wal`` or ``delete``). - - ``database.journal_mode`` in config.yaml is the canonical operator - setting. ``wal`` remains the default; use ``delete`` when the backing - filesystem does not provide WAL-safe durability (for example macOS - virtiofs, NFS, or SMB). Invalid or malformed values fail safely to the - existing default. - """ - try: - from hermes_cli.config import load_config_readonly - - config = load_config_readonly() or {} - database = config.get("database", {}) - if not isinstance(database, dict): - return "wal" - raw = database.get("journal_mode", "wal") - except Exception: - return "wal" - - if not isinstance(raw, str): - return "wal" - mode = raw.strip().lower() - return mode if mode in ("wal", "delete") else "wal" - - -class WalUnsupportedError(sqlite3.OperationalError): - """Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and - the filesystem cannot provide WAL journal mode. - - Covers both shapes of WAL refusal on network filesystems (NFS / SMB / FUSE - / the AgentFS NFS overlay): SQLite *raising* ``SQLITE_PROTOCOL`` ("locking - protocol"), and the quieter macOS-NFS case where ``PRAGMA journal_mode=WAL`` - silently returns the still-effective mode without raising. Subclasses - ``sqlite3.OperationalError`` so existing ``except sqlite3.OperationalError`` - DB-init handling still catches it, while callers that specifically mandate - WAL can catch this narrower type. - """ - - -def apply_wal_with_fallback( - conn: sqlite3.Connection, - *, - db_label: str = "state.db", - require_wal: bool = False, -) -> str: - """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure. - - Returns the journal mode actually set (``"wal"`` or ``"delete"``). - - On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS), SQLite either - raises ``OperationalError("locking protocol")`` / - ``OperationalError("disk I/O error")`` or — on macOS NFS / SMB / - the AgentFS NFS overlay — silently refuses the switch and leaves the DB in - DELETE. Either way the degradation is logged at ERROR level (it is a real - loss of concurrency — a write blocks concurrent readers — not a cosmetic - warning) and, by default, the function falls back to DELETE (the pre-WAL - default, which works on NFS and ZFS) so the feature keeps working. - - On SQLite builds that still contain the WAL-reset corruption bug - (issue #69784), refuse to enable WAL on fresh / non-WAL databases - (prefer DELETE). If the on-disk DB is already WAL, keep WAL and warn - — never live-downgrade under possible concurrent openers. - - This gate (#70055) is deliberately RETAINED. An earlier revision of the - lock-cancellation fix (#71724) reverted it on the theory that DELETE was - "the mode that corrupts", but that comparison was confounded: the clean - WAL result came from SQLite 3.53.1, which carries BOTH the WAL-reset fix - AND 3.51.0's defenses against close()-broken POSIX locks, so it says - nothing about 3.50.4. Re-measured on the actually-bundled 3.50.4 with - the lock fix in place, WAL and DELETE are both clean (0/3 each) — i.e. - there is no evidence that WAL is safer here, and upstream still documents - the WAL-reset bug as real through 3.51.2 with serious consequences. Until - a fixed runtime is delivered, keep new databases out of WAL. - - Callers that genuinely require WAL concurrency (and would rather fail loudly - than run silently degraded) pass ``require_wal=True``; the function then - raises :class:`WalUnsupportedError` instead of returning ``"delete"``. All - current callers deliberately keep the default ``require_wal=False`` so - NFS-homed installs keep working. - - The ERROR is deduplicated per ``db_label``: repeated connections to the - same underlying DB (e.g. kanban_db.connect() which is called on every - kanban operation) log once per process, not once per call. Different - db_labels log independently, so state.db and kanban.db each get one error - on the same NFS mount. - - Shared by :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` so - both databases get identical fallback behavior. - - Never downgrades to DELETE if the on-disk DB header reports WAL — see - _on_disk_journal_mode. That holds for both the NFS path and the - WAL-reset vulnerability path. - """ - configured = resolve_journal_mode() - - # Vulnerable SQLite: do not enable WAL on new/non-WAL files. Resolve the - # operator setting first so an explicit DELETE request still verifies that - # SQLite actually accepted DELETE rather than silently returning MEMORY or - # another connection-specific mode. - if is_sqlite_wal_reset_vulnerable(): - return _apply_delete_for_wal_reset_bug( - conn, - db_label=db_label, - require_delete=configured == "delete", - ) - - # Read-only probe — no flock, no checkpoint, no WAL/SHM unlink. - # Skipping the set-pragma prevents WAL-init from unlinking files other connections hold open. - current_mode = _on_disk_journal_mode(conn) - if current_mode == "wal": - if configured == "delete": - # Never-live-downgrade keeps this WAL; tell the operator their - # configured delete did not apply (see _log_configured_delete_overridden_once). - _log_configured_delete_overridden_once(db_label) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - - # #68545: honor the canonical database.journal_mode setting. Existing - # on-disk WAL databases were returned above and are never live-downgraded. - if configured == "delete": - if current_mode is None: - # The mode probe failed (database locked / busy): another - # process may hold this DB open in WAL. Ownership is not - # provably exclusive, so flipping journal modes here could - # destroy committed-but-uncheckpointed WAL transactions of a - # concurrent writer. Fail loudly instead of downgrading — the - # operator explicitly requested DELETE and we cannot verify it. - raise sqlite3.OperationalError( - "could not verify journal mode before applying configured " - "journal_mode=delete (database is locked — possible " - "concurrent openers); refusing to downgrade a database " - "this process does not exclusively own" - ) - actual = _set_journal_mode_no_wait(conn, "DELETE") - if actual != "delete": - raise sqlite3.OperationalError( - f"could not set configured journal_mode=delete (got {actual or 'no result'})" - ) - return actual - - # Decide BEFORE the flip whether it would silently overwrite a mode - # somebody chose. Both inputs are only readable while the file is still - # in its original state: `current_mode` is the probe above, and - # page_count distinguishes an existing database from a fresh one. - # - # A 0-page database has no prior choice to overwrite, and every caller - # reaches this before creating any schema (SessionDB._connect_and_init - # applies WAL, then _init_schema), so brand-new databases land here empty - # and stay quiet. - _upgrading_existing_db = ( - current_mode is not None - and current_mode != "wal" - and _database_has_content(conn) - ) - - try: - # ``PRAGMA journal_mode=WAL`` is a query-that-sets: it RETURNS the - # resulting journal mode. Network filesystems that refuse WAL by - # *raising* SQLITE_PROTOCOL ("locking protocol") are handled in the - # except branch below. But macOS NFS — and SMB/CIFS, and the AgentFS - # NFS overlay — refuse the switch WITHOUT raising: the pragma simply - # returns the still-effective mode (e.g. ``delete``). Trust the - # returned row, not the mere absence of an exception; otherwise we - # report a false ``"wal"`` AND skip the fallback WARNING, leaving the - # DB silently in DELETE (reader-blocks-writer) with no signal. - row = conn.execute("PRAGMA journal_mode=WAL").fetchone() - mode = str(row[0]).strip().lower() if row and row[0] is not None else "" - if mode == "wal": - if _upgrading_existing_db: - _log_journal_mode_upgrade_once(db_label, current_mode) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - # Silent refusal (macOS NFS / SMB / AgentFS overlay): WAL was not - # honored, but nothing raised. - silent_exc = WalUnsupportedError( - f"journal_mode=WAL refused without raising (still {mode!r})" - ) - if require_wal: - raise silent_exc - _log_wal_fallback_once(db_label, silent_exc) - return mode or "delete" - except sqlite3.OperationalError as exc: - # The require_wal silent-refusal raise above is a WalUnsupportedError - # (an OperationalError subclass) and lands here — propagate it - # unchanged rather than re-running it through the marker logic. - if isinstance(exc, WalUnsupportedError): - raise - msg = str(exc).lower() - if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS): - # Unrelated OperationalError — don't silently swallow. - raise - # ``disk i/o error`` is ambiguous: on ZFS / APFS-CoW it is a - # deterministic WAL-incompatibility (SHM corruption under concurrent - # connection bursts — #55305, #71498), but it can also be a one-shot - # transient EIO (page-cache pressure, brief lock contention). - # Treating a transient EIO as a permanent downgrade signal produced - # the mixed-journal-mode corruption pattern fixed in 5c49cd0ed0 - # (process A downgrades to DELETE while sibling processes set WAL). - # Disambiguate by retrying the pragma a couple of times: transient - # EIO clears and we return "wal"; the deterministic filesystem cases - # keep failing and fall through to the guarded DELETE fallback. - if "disk i/o error" in msg: - for _ in range(2): - time.sleep(0.05) - try: - row = conn.execute("PRAGMA journal_mode=WAL").fetchone() - except sqlite3.OperationalError as retry_exc: - if "disk i/o error" not in str(retry_exc).lower(): - raise - exc = retry_exc - continue - mode = ( - str(row[0]).strip().lower() - if row and row[0] is not None - else "" - ) - if mode == "wal": - # Same flip, later door: a transient EIO cleared and the - # switch went through. The header rewrite is identical, so - # the signal must be too. - if _upgrading_existing_db: - _log_journal_mode_upgrade_once(db_label, current_mode) - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - break - # Don't downgrade if another process already set WAL on disk, or if - # the mode cannot be verified at all (probe blocked by a concurrent - # opener's locks) — ownership is not provably exclusive either way. - existing = _on_disk_journal_mode(conn) - if existing == "wal" or existing is None: - raise - if require_wal: - # Caller mandates WAL — fail loudly instead of degrading to DELETE. - raise WalUnsupportedError(str(exc)) from exc - _log_wal_fallback_once(db_label, exc) - _set_journal_mode_no_wait(conn, "DELETE") - return "delete" - - -def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str: - """Execute ``PRAGMA journal_mode=`` without waiting on other openers. - - This is the ONLY place a journal-mode switch pragma may be issued for a - non-WAL target. It temporarily forces ``busy_timeout=0`` so SQLite's own - exclusivity requirement becomes a concurrent-opener detector: leaving WAL - mode requires exclusive access to the database, so if ANY other connection - (this process or another) holds the DB open, the pragma fails immediately - with ``database is locked`` instead of waiting out a busy timeout and - sneaking the flip in between a concurrent writer's transactions — which is - exactly how committed-but-uncheckpointed WAL transactions get destroyed. - - Callers must treat a raised ``OperationalError`` as "not exclusively - owned: leave the journal mode alone", never as a retryable condition. - - Returns the resulting journal mode as reported by SQLite (lowercase), or - ``""`` when SQLite returned no row. - """ - previous_timeout = 0 - try: - row = conn.execute("PRAGMA busy_timeout").fetchone() - if row and row[0] is not None: - previous_timeout = int(row[0]) - except (sqlite3.OperationalError, TypeError, ValueError): - previous_timeout = 0 - conn.execute("PRAGMA busy_timeout=0") - try: - row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone() - return str(row[0]).strip().lower() if row and row[0] is not None else "" - finally: - try: - conn.execute(f"PRAGMA busy_timeout={previous_timeout}") - except sqlite3.OperationalError: - pass - - -def _apply_delete_for_wal_reset_bug( - conn: sqlite3.Connection, - *, - db_label: str, - require_delete: bool = False, -) -> str: - """Avoid enabling WAL when the linked SQLite has the WAL-reset bug. - - - Already-WAL on disk: leave WAL alone (no live downgrade) and warn. - - Mode unreadable (probe blocked by a concurrent opener's locks): - ownership is not provably exclusive — leave the journal mode alone - and warn. Never treat "could not read the mode" as "not WAL": that - exact confusion let a vulnerable-SQLite process flip a live WAL - state.db to DELETE under a concurrent WAL writer, destroying its - committed-but-uncheckpointed transactions. - - Otherwise: set DELETE (refusing to wait out concurrent openers) and - warn. - - For an explicit operator request, verify SQLite accepted DELETE. - """ - current = _on_disk_journal_mode(conn) - - if current == "wal": - _log_wal_reset_bug_once(db_label, kept_wal=True) - if require_delete: - # The vulnerability warning above suggests upgrading SQLite, which - # does not help on a WAL-incompatible filesystem; surface that the - # configured delete is not in effect (see _log_configured_delete_overridden_once). - # Emitted last so the actionable message is the final one in the log. - _log_configured_delete_overridden_once(db_label) - # Do not TRUNCATE / journal_mode=DELETE while other processes may - # still hold this WAL DB open; same safety rule as the NFS path. - _apply_wal_size_limit(conn) - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return "wal" - - if current is None: - # The mode probe itself failed — another opener's locks are the - # most likely cause, and the DB may well be in WAL under a live - # writer. Never flip a journal mode we cannot even read. - if require_delete: - raise sqlite3.OperationalError( - "could not verify journal mode before applying configured " - "journal_mode=delete (database is locked — possible " - "concurrent openers); refusing to downgrade a database " - "this process does not exclusively own" - ) - _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) - return "wal" - - actual = "" - try: - actual = _set_journal_mode_no_wait(conn, "DELETE") - except sqlite3.OperationalError as exc: - if require_delete: - raise - lowered = str(exc).lower() - if "locked" in lowered or "busy" in lowered: - # A concurrent opener appeared between the probe and the flip - # (or already held the DB): SQLite refused the exclusive lock. - # Leave the journal mode exactly as it is. - _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) - return current or "delete" - # Best-effort for the automatic vulnerable-runtime fallback: DELETE is - # normally already the default for new file-backed databases. - if require_delete and actual != "delete": - raise sqlite3.OperationalError( - "could not set configured journal_mode=delete " - f"(got {actual or 'no result'})" - ) - _log_wal_reset_bug_once(db_label, kept_wal=False) - return "delete" - - -def _wal_reset_repair_hint() -> str: - """Return a context-appropriate hint for repairing the SQLite runtime. - - Uses the codebase's install-type detection so the hint matches what - ``hermes update`` can actually do for this install (#75153). - """ - try: - from hermes_cli.config import ( - detect_install_method, - recommended_update_command_for_method, - get_project_root, - ) - method = detect_install_method(get_project_root()) - cmd = recommended_update_command_for_method(method) - if method in {"git", "unknown"}: - return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`" - if method == "docker": - return f"update the container image with `{cmd}`" - # nix/nixos - return cmd - except Exception: - pass - return ( - "install a Python build bundled with SQLite 3.51.3+ " - "(or backports 3.50.7 / 3.44.6) and restart Hermes" - ) - - -# Dedup state for _log_journal_mode_upgrade_once, mirroring the -# _wal_fallback_warned_* pair below it. -_journal_upgrade_warned_paths: set = set() -_journal_upgrade_warned_lock = threading.Lock() - - -def _log_wal_reset_bug_once( - db_label: str, - *, - kept_wal: bool, - indeterminate: bool = False, -) -> None: - """Log once per (process, db_label) about the WAL-reset vulnerability path.""" - with _wal_reset_bug_warned_lock: - if db_label in _wal_reset_bug_warned_paths: - return - _wal_reset_bug_warned_paths.add(db_label) - if indeterminate: - action = ( - "journal mode could not be verified or exclusively switched " - "(database is locked — possible concurrent openers); leaving the " - "journal mode untouched (no live downgrade under concurrent " - "openers)" - ) - elif kept_wal: - action = ( - "is already in WAL mode — leaving WAL in place (no live " - "downgrade under concurrent openers)" - ) - else: - action = "using journal_mode=DELETE instead of enabling WAL" - # Check whether this is a Hermes-managed install (uv-managed venv) - # so the warning doesn't promise a repair path that doesn't exist - # for git/pip/system Python installs (#75153). - repair_hint = _wal_reset_repair_hint() - logger.warning( - "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset " - "corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. " - "Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); " - "%s. See `hermes doctor`. This warning fires once per " - "process per database.", - db_label, - sqlite3.sqlite_version, - sys.executable, - action, - repair_hint, - ) - - -def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None: - """Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip. - - ``PRAGMA journal_mode`` is a property of the FILE, not of the connection: - switching an existing database to WAL rewrites its header and outlives the - process that did it. Operators do set it directly on the file -- that was - the documented mitigation for the SQLite 3.50.4 WAL-reset bug -- and - nothing here told them the next open would silently put it back. - - WARNING, not ERROR, and deliberately so. The reverse move is logged at - ERROR by ``_log_wal_fallback_once`` because dropping to DELETE is a real - loss of concurrency; this direction is normally the desirable one (see - ``hermes_cli/managed_uv._default_live_venv``, which treats a database - stuck on DELETE as a bug worth repairing on update). The problem is not - the change, it is that the change was invisible: an operator who chose - DELETE deliberately had no way to learn their choice had been overwritten, - or which lever makes it stick. So this says what happened and names the - durable setting, without claiming a degradation that is not there. - - Deduped per process per ``db_label`` like its siblings: kanban opens a - fresh connection per operation, so an undeduped line here would be a log - flood rather than a signal. - """ - with _journal_upgrade_warned_lock: - if db_label in _journal_upgrade_warned_paths: - return - _journal_upgrade_warned_paths.add(db_label) - logger.warning( - "%s: on-disk journal_mode was %s and has been switched to WAL. This " - "rewrites the database header and persists after this process exits. " - "If %s was a deliberate choice (for example the mitigation for the " - "SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with " - "PRAGMA on the file will not survive -- every open re-applies the " - "configured mode. Set `database.journal_mode: delete` in config.yaml " - "to make it stick. This message fires once per process per database.", - db_label, - previous_mode, - previous_mode, - ) - - -def _log_wal_fallback_once(db_label: str, exc: Exception) -> None: - """Log a single ERROR per (process, db_label) about WAL fallback. - - ERROR (not WARNING): a DB silently dropped to DELETE means a real loss of - concurrency — under the kanban dispatcher + workers a write blocks readers, - surfacing as SQLITE_BUSY/lock contention — so it must be loud, not cosmetic. - - Without this dedup, NFS users running kanban (which opens a fresh - connection on every operation — see hermes_cli/kanban_db.py) would - fill errors.log with hundreds of identical errors per hour. - """ - with _wal_fallback_warned_lock: - if db_label in _wal_fallback_warned_paths: - return - _wal_fallback_warned_paths.add(db_label) - logger.error( - "%s: WAL journal_mode unsupported on this filesystem (%s) — " - "falling back to journal_mode=DELETE (slower rollback-journal " - "mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See " - "https://www.sqlite.org/wal.html for details. This message " - "fires once per process per database.", - db_label, - exc, - ) - - -def _log_configured_delete_overridden_once(db_label: str) -> None: - """Log a single ERROR per (process, db_label) when the operator configured - ``journal_mode=delete`` but the on-disk DB is already WAL, so the configured - mode is not in effect. - - Counterpart to :func:`_log_wal_fallback_once` for the opposite direction: - there WAL was refused by the filesystem and we silently fell back to DELETE; - here the operator asked for DELETE but an inherited on-disk WAL header means - we keep WAL (the never-live-downgrade rule prevents a live downgrade, which - causes mixed-mode corruption). The signal matters because otherwise the - operator has no indication that ``database.journal_mode: delete`` had no - effect and the DB still requires a one-time offline ``PRAGMA - journal_mode=DELETE`` (with no open connections) to apply. - - Fires once per process per database. - """ - with _delete_overridden_warned_lock: - if db_label in _delete_overridden_warned_paths: - return - _delete_overridden_warned_paths.add(db_label) - logger.error( - "%s: database.journal_mode=delete is configured but the on-disk " - "database is already WAL; keeping WAL (a live downgrade under open " - "connections can corrupt the DB). To apply journal_mode=DELETE, stop " - "all connections to this DB and run a one-time offline " - "'PRAGMA journal_mode=DELETE' on the file. This message fires once " - "per process per database.", - db_label, - ) - - -# --------------------------------------------------------------------------- -# Config-driven database pragmas -# --------------------------------------------------------------------------- -# PRAGMA synchronous accepts either an integer or a symbolic name, and operators -# write the names. Mapped here rather than passed through so a typo becomes a -# warning instead of a silently different durability level. -_SYNCHRONOUS_LEVELS: Dict[str, int] = { - "OFF": 0, - "NORMAL": 1, - "FULL": 2, - "EXTRA": 3, -} -_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()} -_SYNCHRONOUS_FULL = 2 - - -def resolve_synchronous_level(raw_value: Any) -> Optional[int]: - """Map a configured ``database.synchronous`` value to its PRAGMA integer. - - Accepts the symbolic names SQLite documents (``OFF``/``NORMAL``/``FULL``/ - ``EXTRA``, any case) and the equivalent integers ``0``-``3``. Returns None - for anything else so the caller can warn and leave the level untouched — - guessing at a malformed durability setting is worse than ignoring it. - """ - if isinstance(raw_value, bool): - # bool is an int subclass, and YAML turns a bare `on`/`off` into one. - # "off" as a durability level is a real choice; True is meaningless. - return 0 if raw_value is False else None - if isinstance(raw_value, int): - return raw_value if raw_value in _SYNCHRONOUS_NAMES else None - text = str(raw_value).strip() - if not text: - return None - upper = text.upper() - if upper in _SYNCHRONOUS_LEVELS: - return _SYNCHRONOUS_LEVELS[upper] - try: - value = int(text) - except (TypeError, ValueError): - return None - return value if value in _SYNCHRONOUS_NAMES else None - - -def _apply_synchronous_pragma( - conn: sqlite3.Connection, - raw_value: Any, - *, - db_label: str, -) -> None: - """Set ``PRAGMA synchronous`` from config, never below FULL on macOS. - - Split out of the integer loop in :func:`apply_database_pragmas` because - this PRAGMA is not interchangeable with the sizing ones around it. Those - trade memory or disk for speed; this one decides whether a commit is on - the platter, so an unrecognised value must not fall through to "SQLite - default" the way a bad ``cache_size`` harmlessly can. - - The Darwin floor exists because :func:`_enforce_macos_synchronous_full` - runs during ``apply_wal_with_fallback()`` and this function runs after it, - so a configured ``NORMAL`` would otherwise silently undo the macOS btree - protection that fix put there deliberately. Raising the level on macOS is - allowed; lowering it is refused out loud. - """ - level = resolve_synchronous_level(raw_value) - if level is None: - logger.warning( - "%s: ignoring unrecognized database.synchronous=%r " - "(expected OFF, NORMAL, FULL, EXTRA, or 0-3)", - db_label, - raw_value, - ) - return - if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL: - logger.warning( - "%s: refusing database.synchronous=%s on macOS; keeping FULL. " - "Darwin's fsync() does not guarantee write ordering, so a lower " - "level readmits the half-written btree pages FULL exists to " - "prevent.", - db_label, - _SYNCHRONOUS_NAMES[level], - ) - return - try: - conn.execute(f"PRAGMA synchronous={level}") - except sqlite3.OperationalError: - pass - - -def apply_database_pragmas( - conn: sqlite3.Connection, - *, - db_label: str = "state.db", -) -> None: - """Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``. - - Reads the ``database:`` section and applies configurable PRAGMAs when set - to integer values. The journal mode itself is NOT handled here — - ``database.journal_mode`` is owned by :func:`resolve_journal_mode` inside - :func:`apply_wal_with_fallback`, which layers the operator setting under - all the safety guards (never live-downgrading an on-disk WAL DB, - filesystem fallback, WAL-reset-bug gating). - - Supported keys under ``database:`` in config.yaml: - - * ``cache_size`` — negative value = KiB, positive = pages - (e.g. ``-262144`` = 256 MB page cache) - * ``mmap_size`` — max bytes for memory-mapped I/O (0 = disabled) - * ``temp_store`` — 0=DEFAULT(file), 1=FILE, 2=MEMORY, 3=ALWAYS - * ``wal_autocheckpoint`` — WAL auto-checkpoint threshold in pages - * ``journal_size_limit`` — max journal/WAL size in bytes - * ``synchronous`` — durability level: ``OFF``/``NORMAL``/``FULL``/``EXTRA`` - or ``0``-``3``. Unset leaves SQLite's own default, which is a - *compile-time* constant (``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) and so - differs between the bundled, distro and Homebrew builds an operator - might be running. Setting it explicitly is the only way to know. - - Best-effort: config load or pragma failures are ignored so DB init - never breaks on a malformed ``database:`` section. - """ - try: - # Local import avoids a circular import with hermes_cli.config. - from hermes_cli.config import cfg_get, load_config_readonly - - cfg = load_config_readonly() - except Exception: - return - - # Performance PRAGMAs (applied to ALL connection types: writer, read_only, - # and WAL per-thread readers). - for pragma_name in ( - "cache_size", - "mmap_size", - "temp_store", - "wal_autocheckpoint", - "journal_size_limit", - ): - raw_value = cfg_get(cfg, "database", pragma_name, default=None) - if raw_value is None: - continue - try: - value = int(str(raw_value).strip()) - except (TypeError, ValueError): - logger.warning( - "%s: ignoring non-integer database.%s=%r", - db_label, - pragma_name, - raw_value, - ) - continue - try: - conn.execute(f"PRAGMA {pragma_name}={value}") - except sqlite3.OperationalError: - pass - - # Last, so it wins over nothing and loses to nothing: the sizing pragmas - # above cannot change durability, and the macOS enforcement ran earlier - # during WAL activation (see _apply_synchronous_pragma for why that - # ordering needs an explicit floor rather than an explicit override). - raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) - if raw_synchronous is not None: - _apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label) - - # --------------------------------------------------------------------------- # Malformed-schema recovery # --------------------------------------------------------------------------- @@ -2318,21 +1453,6 @@ def classify_persistence_error(exc_or_str) -> str: return "unknown" -def _claim_repair_attempt(db_path: Path) -> bool: - """Claim the one-shot repair attempt for *db_path* in this process. - - Returns True for the first caller, False afterwards. Keeps a malformed - DB from triggering an unbounded repair/reopen loop and stops concurrent - callers from racing surgery on the same file. - """ - key = str(db_path) - with _repair_attempt_lock: - if key in _repair_attempted_paths: - return False - _repair_attempted_paths.add(key) - return True - - # Cross-process serialisation for the schema-surgery paths below. The # ``_repair_attempt_lock`` above is a ``threading.Lock`` — it only covers # threads inside ONE interpreter, yet a normal Hermes host runs several @@ -2347,208 +1467,9 @@ def _claim_repair_attempt(db_path: Path) -> bool: # lock the losing caller spent the same minutes running its own surgery, it # just did so on top of the winner's. _REPAIR_LOCK_TIMEOUT_SECONDS = 120.0 -_REPAIR_LOCK_POLL_SECONDS = 0.1 -# Copying a multi-GB database through SQLite's online-backup API is a data -# transfer, not an inter-process locking operation. Keep the repair-lock -# bound separate and give a conservative 10 MiB/s budget to each snapshot or -# promotion, with the historical two-minute floor for ordinary state.db files. -_REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024 _IS_WINDOWS = sys.platform == "win32" -@contextlib.contextmanager -def _cross_process_repair_lock(db_path: Path): - """Serialize state.db schema surgery across processes. - - Yields True when this process holds the repair lock for *db_path*, False - when the bounded acquire timed out or the lock file could not be opened at - all. Unlike the kanban init lock — whose critical section is idempotent, - so proceeding without the lock is merely redundant work — proceeding here - would be exactly the unsafe interleaving we are trying to prevent, so a - caller that gets False must NOT do surgery. - - ``flock`` is the right primitive for this: the kernel drops the lock when - the holding process dies, so a crashed repairer cannot leave a stale lock - that wedges every future repair (a pidfile would). One exception exists - (issue #100108): a forked child that inherited the lock fd keeps the - flock alive after the acquirer dies, so the acquire path records the - holder's pid + start time and breaks the lock when that holder is - provably dead (see ``_acquire_db_flock``). The acquire is still - bounded because a *live* repairer can legitimately sit in ``VACUUM`` for - minutes on a large DB, and an unbounded wait would hang the caller's open - with no traceback (the failure shape of #36644). - """ - lock_path = db_path.with_name(db_path.name + ".repair.lock") - try: - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - except OSError as exc: - # Fail closed, exactly as a timed-out acquire does. A lock file we - # cannot even open means the filesystem is out of space, inodes or - # descriptors — and a sibling that opened ITS handle before the disk - # filled is still inside writable_schema surgery or VACUUM. Yielding - # True here let two processes run schema surgery on the same live - # state.db concurrently, which is itself the corruption source this - # lock exists to remove (#100368: the disk-full trigger, then a fresh - # corruption on every boot with other writers alive). Callers already - # handle False by re-probing and reporting, and on a read-only - # directory no repair strategy could have written anyway. - logger.warning( - "Could not open state.db repair lock %s (%s) — skipping schema " - "surgery rather than running it without cross-process authority.", - lock_path, exc, - ) - yield False - return - - acquired = False - try: - if _IS_WINDOWS: - deadline = time.monotonic() + _REPAIR_LOCK_TIMEOUT_SECONDS - while True: - try: - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - acquired = True - break - except (BlockingIOError, OSError) as exc: - if not is_advisory_lock_contention(exc): - logger.warning( - "Could not acquire state.db repair lock %s (%s) — " - "skipping schema surgery on a non-contention error.", - lock_path, exc, - ) - acquired = None - break - if time.monotonic() >= deadline: - break - time.sleep(_REPAIR_LOCK_POLL_SECONDS) - else: - acquired, handle = _acquire_db_flock( - str(lock_path), - handle, - _REPAIR_LOCK_TIMEOUT_SECONDS, - _REPAIR_LOCK_POLL_SECONDS, - "state.db repair lock", - ) - if acquired is None: - # Non-contention failure already logged with its errno. - acquired = False - elif not acquired: - record = None if _IS_WINDOWS else _read_lock_holder_record(handle) - logger.warning( - "state.db repair lock %s held by another process for more " - "than %.0fs — skipping schema surgery in this process to " - "avoid racing the repairer. Recorded holder: %s.", - lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS, - _describe_lock_holder(record), - ) - yield acquired - finally: - try: - if acquired: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - - _clear_lock_holder_record(handle) - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except OSError: # pragma: no cover - best effort release - pass - finally: - handle.close() - - -def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]: - """Non-blocking cross-process lock for one auto-maintenance pass. - - The kernel releases this advisory lock if the holder exits, unlike a - durable pid/meta marker. A caller that cannot acquire it must skip the - pass: otherwise two startups can both pass the interval check and the - second can prune a row the first has only just closed recoverably. - """ - lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock") - try: - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - except OSError as exc: - logger.warning( - "Could not open state.db auto-maintenance lock %s (%s) — skipping " - "automatic maintenance.", - lock_path, - exc, - ) - return None - - try: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking( # type: ignore[attr-defined] - handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined] - ) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - except (BlockingIOError, OSError): - handle.close() - return None - return handle - - -def _release_auto_maintenance_lock(handle: Any) -> None: - """Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`.""" - try: - if _IS_WINDOWS: - import msvcrt - - handle.seek(0) - msvcrt.locking( # type: ignore[attr-defined] - handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined] - ) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except OSError: # pragma: no cover - best effort release - pass - finally: - handle.close() - - -def _bump_schema_cookie(conn: sqlite3.Connection) -> None: - """Increment the schema cookie after direct ``sqlite_master`` surgery. - - Ordinary DDL bumps this counter for free, and every other connection - compares it before running a prepared statement — that is how they learn - to discard a cached schema. Editing ``sqlite_master`` under - ``PRAGMA writable_schema=ON`` does NOT bump it, so live connections in - other processes keep compiling statements against the schema we just - deleted objects from — e.g. writing ``messages`` rows through triggers - into ``messages_fts*`` shadow tables that no longer exist. SQLite's - writable_schema documentation calls out incrementing ``schema_version`` - as the required companion to such an edit. - - Best-effort and never raises: a failed bump leaves exactly the - pre-existing behaviour, and the repair itself is still worth completing. - """ - try: - current = conn.execute("PRAGMA schema_version").fetchone()[0] - # Wraps within the 32-bit signed range SQLite stores this in; the - # comparison other connections make is equality, not ordering. - conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}") - except (sqlite3.DatabaseError, TypeError, IndexError) as exc: - logger.warning("Could not bump state.db schema cookie: %s", exc) - - # ── Repair-loop bounding + dead-backup hygiene (#86747) ───────────────────── # # ``_claim_repair_attempt`` above is an in-memory set: it bounds the loop @@ -2565,1629 +1486,6 @@ def _bump_schema_cookie(conn: sqlite3.Connection) -> None: # damaged file is never copied twice, and only the newest # ``_MAX_MALFORMED_BACKUPS`` forensic copies are kept. -_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3 -_MAX_MALFORMED_BACKUPS = 3 - -# Sidecars copied alongside a damaged DB and pruned with it. ``-journal`` is -# included because rollback-journal (DELETE) mode — Hermes's fallback on -# NFS/SMB/FUSE/ZFS and on WAL-reset-vulnerable SQLite builds — leaves a hot -# journal on disk whenever a transaction was open, and that file is what -# interprets the damaged bytes. Omitting it from the forensic copy means the -# backup cannot be rolled back to a consistent state by hand. -_DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") - -# Head/tail bytes sampled by ``_db_fingerprint``. Enough to change whenever -# the DB is genuinely repaired, truncated or restored (SQLite rewrites the -# header on any real recovery), while staying O(1) on a multi-GB file. -_FINGERPRINT_SAMPLE_BYTES = 65536 - -# Byte ranges inside SQLite's 100-byte database header that move on ordinary -# commits rather than on repair, and are therefore masked out of the content -# sample. In rollback-journal (DELETE) mode a commit writes the main file -# directly, bumping the file change counter (24-27) and version-valid-for -# (92-95); a malformed-SCHEMA DB still accepts those writes, so without the -# mask any live session write re-keys the ledger and the repair budget resets -# to 1 forever — the exact unbounded loop this ledger exists to stop. (WAL mode -# routes commits to the -wal sidecar, so the main file's header only moves on -# checkpoint; masking is harmless there and correct for both.) Everything that -# matters for repair identity — the page-1 sqlite_master b-tree — sits after -# byte 100 and stays in the sample. -_FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96)) - - -def _mask_volatile_header(head: bytes) -> bytes: - """Zero the commit-counter fields so ordinary writes don't re-key the ledger.""" - if len(head) < 96: - return head - buf = bytearray(head) - for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES: - buf[start:end] = b"\x00" * (end - start) - return bytes(buf) - -# Free-space headroom for the pre-repair forensic backup. The backup is a -# full raw copy of the damaged DB (plus its -wal/-shm sidecars), so a repair -# loop on a large state.db is a disk amplifier: the reporting incident wrote -# ~98MB every ~10s until the volume was nearly full, which would have taken -# down every agent on the host. -# -# Proportional, not a flat floor: an absolute multi-GB reserve would refuse -# backups that fit comfortably on small container/VM volumes, and because a -# refused backup is a HARD STOP (#69603) that would silently convert "repair -# loops" into "repair never runs" for those deployments. Require the copy -# itself plus a small slice of the volume, clamped to a modest floor. -_REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor -_REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume - - -def _repair_backup_headroom_bytes(total_bytes: int) -> int: - """Free space required *beyond* the copy itself, for a volume of *total_bytes*.""" - return max( - _REPAIR_BACKUP_MIN_FREE_BYTES, - int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION), - ) - - -def _repair_scratch_space_error(db_path: Path) -> Optional[str]: - """Return an error unless snapshot, VACUUM and promotion can fit safely.""" - import shutil - - try: - main_bytes = db_path.stat().st_size - snapshot_bytes = main_bytes - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - snapshot_bytes += sidecar.stat().st_size - usage = shutil.disk_usage(db_path.parent) - headroom = _repair_backup_headroom_bytes(usage.total) - # Strategy 2 runs VACUUM on the staged database. SQLite documents that - # VACUUM may need up to twice the database size in additional free - # space while it builds the replacement and journals the overwrite. - # Reserve that beyond the snapshot itself; after VACUUM releases its - # temporary files, the same reserve also covers transactional - # promotion into the live database. - required = snapshot_bytes + (2 * snapshot_bytes) + headroom - if usage.free >= required: - return None - return ( - f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; the " - f"repair snapshot needs up to {snapshot_bytes / 1e9:.2f}GB, " - f"VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and " - f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, " - "then retry." - ) - except OSError as exc: - return ( - f"could not determine free space on {db_path.parent} ({exc}); " - "refusing the repair snapshot rather than risk filling the volume" - ) - - -def _repair_snapshot_timeout_seconds(source_path: Path) -> float: - """Bound one SQLite snapshot by source size, including live sidecars. - - A WAL can contain committed canonical rows which are not yet present in - the main database file. Count it (and the rollback journal where - present), both to describe the work honestly and to avoid applying the - repair-lock timeout to an otherwise healthy large-database copy. - """ - source_bytes = 0 - for suffix in ("", *_DB_SIDECAR_SUFFIXES): - candidate = ( - source_path - if not suffix - else source_path.with_name(source_path.name + suffix) - ) - try: - source_bytes += candidate.stat().st_size - except FileNotFoundError: - continue - return max( - _REPAIR_LOCK_TIMEOUT_SECONDS, - source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, - ) - - -def _repair_failure_consumes_attempt(exc: BaseException) -> bool: - """Whether a pre-strategy SQLite failure proves deterministic corruption. - - Lock contention, timeouts, disk-full, I/O and filesystem failures are - environmental aborts: retrying later may succeed and must not exhaust the - repair ledger. Only SQLite's corruption/image result codes prove the - deterministic damage the bounded ledger exists to stop from retrying - forever, even when SQLite cannot stage a snapshot far enough to run a - named strategy. - """ - if not isinstance(exc, sqlite3.DatabaseError): - return False - error_code = getattr(exc, "sqlite_errorcode", None) - if isinstance(error_code, int): - # Extended result codes retain the primary code in the low byte. - primary_code = error_code & 0xFF - return primary_code in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB) - - # sqlite3 versions before exception result-code attributes need a narrow, - # conservative compatibility path. Do not turn generic DatabaseError - # messages such as "disk is full" or "readonly" into permanent failures. - message = str(exc).lower() - return ( - "file is not a database" in message - or "database disk image is malformed" in message - ) - - -def _repair_ledger_path(db_path: Path) -> Path: - return db_path.with_name(db_path.name + ".repair-attempts.json") - - -def _db_fingerprint(db_path: Path) -> "Optional[str]": - """Cheap identity for a damaged DB file: size + a bounded content sample. - - Deliberately EXCLUDES mtime. The original ledger keyed on - ``size:mtime_ns`` on the assumption that "nothing can successfully write - to a damaged file", but that does not hold for the malformed-schema - class: the DB still opens and accepts writes (only ``sqlite_master`` is - unreadable), so live writers, WAL checkpoints and the in-place repair - strategies themselves all move mtime between passes. Every pass then - looked like a NEW file — the attempt counter reset to 1 forever, never - reaching ``_MAX_PERSISTENT_REPAIR_ATTEMPTS``, and the ``_backup_db_file`` - dedupe (which compares mtime too) never matched, so each pass wrote - another full-size forensic copy. Observed: a repair every ~10s, a fresh - 98MB copy each time, 2.3GB in 20 minutes, disk heading to zero. - - Hashing a multi-GB corrupt file on every open is the repeated cost this - ledger exists to avoid, so sample instead of digesting the whole file: - size plus the head/tail slices that any real repair, truncation or - restore necessarily changes. Stable across passes that merely touch - mtime; still resets the attempt count after genuine recovery. - - The content read runs under ``offline_file_access`` because it takes a raw - descriptor, and ``close()`` on ANY descriptor cancels every POSIX advisory - lock this process holds on the file — including a peer connection's - RESERVED lock (see ``hermes_cli.sqlite_safe_read`` rule 1). This function - is reached from ``repair_state_db_schema``'s exhaustion probe BEFORE - ``_backup_db_file``'s ``has_live_connection`` guard, and the repair path is - entered by one SessionDB while the gateway holds others, so a live peer is - the expected case rather than a theoretical one. - - Returns ``None`` when a live connection makes the read unsafe. Callers MUST - NOT substitute a differently-shaped key (an earlier revision fell back to - ``size:mtime_ns``): the ledger compares keys for equality, so alternating - between a content key and an mtime key across passes never matches, the - counter resets to 1 every time and the unbounded repair loop this ledger - exists to stop comes straight back. ``None`` means "identity unavailable", - and the ledger helpers below keep using the key already on record. - """ - try: - st = db_path.stat() - try: - from hermes_cli.sqlite_safe_read import ( - LiveConnectionError, - offline_file_access, - ) - except ImportError: - # Scaffold/embed installs ship hermes_state without hermes_cli. No - # tracked connections exist there, so the raw read is safe. - @contextmanager - def offline_file_access(_path, **_kw): - yield - - class LiveConnectionError(Exception): - pass - - try: - with offline_file_access(db_path, what="fingerprint"): - with open(db_path, "rb") as fh: - head = fh.read(_FINGERPRINT_SAMPLE_BYTES) - if st.st_size > _FINGERPRINT_SAMPLE_BYTES: - fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES)) - tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) - else: - tail = b"" - except LiveConnectionError: - return None - digest = hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32] - return f"{st.st_size}:{digest}" - except OSError: - return None - - -def _backup_content_identity(db_path: Path) -> "Optional[str]": - """Recovery-image identity for forensic-backup dedupe: whole-file + sidecars. - - This is a DIFFERENT equivalence relation from :func:`_db_fingerprint`, and - the two MUST NOT be conflated. ``_db_fingerprint`` answers "same repair - epoch?" — it masks SQLite's commit counters and samples only the head/tail - so an ordinary write does not mint a fresh repair budget. That is exactly - the wrong predicate for "may I reuse an existing forensic copy?": a live - writer can commit new transcript/session rows into an *interior* page while - preserving file size and leaving the first/last 64 KiB untouched, so two - materially different recovery images share one ``_db_fingerprint``. Reusing - a backup on that basis hands the operator a snapshot that predates real - user data (and #87409 shows a failed in-place repair can still VACUUM - canonical tables away), so the forensic copy must claim byte identity, not - epoch identity. - - So this digests the ENTIRE main file plus every present sidecar - (``-wal``/``-shm``/``-journal``) — the WAL can hold committed frames not yet - checkpointed, so it is part of the recovery image. The cost is an O(n) read; - on a miss the caller is about to do an O(n) *write* (the full raw copy), so - the read is the cheaper half and never the dominant cost. Runs under - ``offline_file_access`` for the same POSIX-advisory-lock reason as - ``_db_fingerprint``; returns ``None`` when a live connection makes the read - unsafe (caller then declines to dedupe and takes a fresh backup — the safe - side, never a false reuse). - """ - try: - from hermes_cli.sqlite_safe_read import ( - LiveConnectionError, - offline_file_access, - ) - except ImportError: - @contextmanager - def offline_file_access(_path, **_kw): - yield - - class LiveConnectionError(Exception): - pass - - def _hash_whole(path: Path, hasher: "Any") -> None: - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1024 * 1024), b""): - hasher.update(chunk) - - try: - hasher = hashlib.sha256() - with offline_file_access(db_path, what="backup-identity"): - # Length-delimit every member (main file included) so the - # concatenation is prefix-free — otherwise a main-file tail could - # coincide with a main+sidecar split and dedupe two different - # recovery images together. - hasher.update(f"\0main:{db_path.stat().st_size}\0".encode()) - _hash_whole(db_path, hasher) - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - hasher.update(f"\0{suffix}:{sidecar.stat().st_size}\0".encode()) - _hash_whole(sidecar, hasher) - return hasher.hexdigest() - except LiveConnectionError: - return None - except OSError: - return None - - -def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]": - try: - raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8")) - if isinstance(raw, dict): - return raw - except (OSError, ValueError): - pass - return {} - - -def _persistent_repair_attempts_exhausted(db_path: Path) -> bool: - """Whether *db_path* has already burned its cross-restart repair budget. - - True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` - failed attempts against the CURRENT file fingerprint. Never raises; a - missing/corrupt ledger or unstatable DB reads as "not exhausted" (the - in-process claim and cross-process lock still bound a single run). - - When the fingerprint is unavailable because a live connection makes the - content read unsafe, fall back to the SIZE the ledger recorded rather than - reading as "not exhausted". Otherwise a peer connection is enough to hide - an exhausted budget on every pass, which is the unbounded loop again. - """ - ledger = _read_repair_ledger(db_path) - recorded = ledger.get("fingerprint") - fp = _db_fingerprint(db_path) - if fp is None: - # Size is the one component both key shapes share and that a raw read - # is not needed for; an unchanged size means the damaged file is very - # likely the same one the budget was burned on. - try: - size_prefix = f"{db_path.stat().st_size}:" - except OSError: - return False - if not isinstance(recorded, str) or not recorded.startswith(size_prefix): - return False - elif recorded != fp: - return False - return int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS - - -def _persistent_repair_exhausted_error(db_path: Path) -> str: - """The stable operator-facing diagnostic for an exhausted repair budget.""" - return ( - f"automatic repair has already failed " - f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — " - "the corruption is beyond the schema/FTS repair strategies " - "(likely b-tree page damage). Manual recovery required: restore " - f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. " - f"Delete {_repair_ledger_path(db_path).name} to force another " - "automatic attempt." - ) - - -def _record_repair_outcome( - db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None -) -> None: - """Update the persistent attempt ledger after a repair pass. Never raises. - - Defaults to the post-attempt fingerprint — the file state the NEXT - attempt's exhaustion probe will observe. - - When the fingerprint is unavailable (a live connection makes the content - read unsafe), keep the key already on record and still increment: dropping - the pass would let a peer connection reset the budget every time, which is - the unbounded loop this ledger exists to stop. Never write a differently - shaped key — the probe compares for equality, so mixing key shapes across - passes never matches. - """ - ledger_path = _repair_ledger_path(db_path) - try: - if repaired: - ledger_path.unlink(missing_ok=True) - return - ledger = _read_repair_ledger(db_path) - recorded = ledger.get("fingerprint") - fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path) - if fp is None: - if not isinstance(recorded, str): - # No prior key to extend and no way to mint one safely: the - # in-process claim and cross-process lock still bound this run. - return - fp = recorded - attempts = ( - int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1 - ) - import datetime - - ledger_path.write_text( - json.dumps( - { - "fingerprint": fp, - "failed_attempts": attempts, - "last_attempt": datetime.datetime.now().isoformat( - timespec="seconds" - ), - } - ), - encoding="utf-8", - ) - except Exception as exc: # pragma: no cover - best effort - logger.warning("Could not update state.db repair ledger: %s", exc) - - -def _existing_malformed_backups(db_path: Path) -> "List[Path]": - """Timestamped forensic backups of *db_path*, newest first.""" - prefix = f"{db_path.name}.malformed-backup-" - try: - found = [ - p - for p in db_path.parent.iterdir() - if p.name.startswith(prefix) - and not p.name.endswith(_DB_SIDECAR_SUFFIXES) - ] - except OSError: - return [] - return sorted(found, key=lambda p: p.name, reverse=True) - - -def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None: - """Delete all but the *keep* newest forensic backups (and sidecars).""" - for stale in _existing_malformed_backups(db_path)[keep:]: - for victim in ( - stale, - *(stale.with_name(stale.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES), - ): - try: - victim.unlink(missing_ok=True) - except OSError as exc: # pragma: no cover - best effort - logger.warning("Could not prune stale DB backup %s: %s", victim, exc) - - -def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": - """Copy a (possibly malformed) DB file to a timestamped backup beside it. - Raw file copy on purpose: the DB won't open cleanly, so we preserve the - bytes exactly for forensics / manual restore. WAL, SHM and rollback-journal - sidecars are copied too when present. Returns ``(backup_path, None)`` on success or - ``(None, reason)`` on failure — callers on the repair path treat a - refused backup as a HARD STOP (see #69603). Repair strategies run on a - scratch snapshot, but the forensic bundle remains the recovery path when - corruption defeats them. - - Refuses when a connection to this database is still live in the process: - reading the file would ``close()`` a descriptor for it and cancel that - connection's POSIX advisory locks (see ``hermes_cli.sqlite_safe_read``). - The repair path can be entered by one SessionDB while the gateway holds - others, so this is a real possibility rather than a theoretical one. - """ - import datetime - import shutil - - try: - from hermes_cli.sqlite_safe_read import has_live_connection - except ImportError: - has_live_connection = None # type: ignore[assignment] - - if has_live_connection is not None and has_live_connection(db_path): - reason = ( - f"a connection to {db_path} is still open in this process; " - "raw-copying it would cancel that connection's POSIX advisory " - "locks. Close all SessionDB handles first." - ) - logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason) - return None, reason - - stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}") - # Same-second collision (two distinct damaged states within one second) - # must not silently overwrite the earlier forensic copy. - seq = 1 - while backup_path.exists(): - backup_path = db_path.with_name( - f"{db_path.name}.malformed-backup-{stamp}_{seq}" - ) - seq += 1 - try: - # Sweep staging debris from an earlier interrupted pass (kill mid-copy) - # BEFORE the dedupe below. A leftover staging file is a byte-identical - # copy of the damaged DB, so its fingerprint MATCHES and the dedupe - # would otherwise hand it back as a legitimate forensic backup. - # Matches sidecar staging names (``.backup-staging--wal``) too. - # The second pattern is the pre-merge ``.incomplete`` spelling, swept so - # a host that ran that build does not keep prefix-matching debris that - # sorts NEWEST and survives prune forever. - for pattern in ( - f"{db_path.name}.backup-staging-*", - f"{db_path.name}.malformed-backup-*.incomplete*", - ): - for old in db_path.parent.glob(pattern): - try: - old.unlink(missing_ok=True) - except OSError: # pragma: no cover - best effort - pass - # Dedupe (#86747): a repair loop used to copy the SAME damaged bytes - # on every restart — ~900MB a pass, 89GB over 11 days in the - # reporting install. If the newest existing backup is byte-identical to - # the current recovery image, reuse it. - # - # Matching on mtime made this dedupe miss exactly when it mattered - # most: the malformed-SCHEMA class still accepts writes, so live - # writers and the in-place repair strategies move mtime between - # passes and every pass wrote another full-size copy (2.3GB in 20 - # minutes). - # - # Use ``_backup_content_identity`` (whole file + sidecars), NOT the - # repair-epoch ``_db_fingerprint``. They are different equivalence - # relations: the fingerprint masks commit counters and samples only - # head/tail so an ordinary interior-page write does not re-key the - # repair budget — but that same write DOES change the recovery image, - # and deduping on the fingerprint would hand back a stale backup that - # predates the write. A forensic copy must prove byte identity, so it - # pays the O(n) read (cheaper than the O(n) write it avoids on a hit). - try: - # Only hash the source when there is actually a candidate to dedupe - # against — on the common first-corruption pass there is no prior - # backup, and hashing the (possibly multi-GB) source then would be - # pure waste right before the copy reads it again anyway. - existing_backups = _existing_malformed_backups(db_path)[:1] - if existing_backups: - src_id = _backup_content_identity(db_path) - for existing in existing_backups: - if src_id is not None and _backup_content_identity(existing) == src_id: - logger.info( - "Reusing existing forensic backup %s (identical to the " - "damaged DB).", existing, - ) - return existing, None - except OSError: - pass - # Disk guard: this is a full raw copy of a possibly multi-GB DB plus - # its sidecars. On a host whose volume is already nearly full — which - # a preceding repair loop may itself have caused — taking it can - # finish off the disk and take down every process on the machine. - # Refuse while there is still room to refuse in. - try: - need = db_path.stat().st_size - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - need += sidecar.stat().st_size - usage = shutil.disk_usage(db_path.parent) - headroom = _repair_backup_headroom_bytes(usage.total) - if usage.free - need < headroom: - reason = ( - f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; " - f"copying the damaged DB needs {need / 1e9:.2f}GB and must " - f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, " - f"then retry (or recover manually with `sqlite3 {db_path} " - '".recover"`).' - ) - logger.error("Refusing forensic backup of %s: %s", db_path, reason) - return None, reason - except OSError as exc: - # Fail CLOSED. This guard exists for the nearly-full volume, which - # is exactly where stat()/disk_usage() is most likely to fail — and - # proceeding would take the multi-GB copy that finishes off the - # disk. A refused backup is a HARD STOP (#69603), so repair simply - # does not run until a human frees space, which is the safe side. - reason = ( - f"could not determine free space on {db_path.parent} ({exc}); " - "refusing the forensic copy rather than risk filling the " - f"volume. Free disk space, then retry (or recover manually " - f'with `sqlite3 {db_path} ".recover"`).' - ) - logger.error("Refusing forensic backup of %s: %s", db_path, reason) - return None, reason - # Copy to a staging name OUTSIDE the ``.malformed-backup-`` prefix, then - # rename into place only once every copy has succeeded. The prefix - # matters: ``_existing_malformed_backups`` matches on - # ``startswith(f"{db}.malformed-backup-")`` and excludes only ``-wal``/ - # ``-shm`` suffixes, so a staging name derived from the backup name (e.g. - # ``…malformed-backup-.incomplete``) still counts as a backup — - # it sorts NEWEST (``.incomplete`` > the bare stamp), so prune's - # keep-3-newest slice retained partials and deleted intact copies, and - # the dedupe could hand a partial back as the official ``backup_path``, - # passing the #69603 hard-stop gate with no real forensic copy on disk. - staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}") - # (staging_src, final_dst) pairs. ORDER MATTERS for publication: the - # main-DB backup name is the bundle's commit marker — - # ``_existing_malformed_backups`` matches ``{db}.malformed-backup-*`` - # and excludes only the ``-wal``/``-shm``/``-journal`` suffixes, so the - # main file appearing is what makes the bundle "count". Sidecars are - # therefore staged/published FIRST and the main DB LAST, so a failure - # partway through never leaves a countable main backup standing over a - # missing sidecar (an incomplete recovery image that would pass the - # #69603 hard stop and dedupe as legitimate on the next pass). - staged_sidecars: "List[Tuple[Path, Path, Path]]" = [] - for suffix in _DB_SIDECAR_SUFFIXES: - sidecar = db_path.with_name(db_path.name + suffix) - if sidecar.exists(): - side_staging = staging.with_name(staging.name + suffix) - side_dst = backup_path.with_name(backup_path.name + suffix) - staged_sidecars.append((sidecar, side_staging, side_dst)) - main_pair = (staging, backup_path) - published: "List[Path]" = [] - all_staging_srcs = [staging] + [s for _src, s, _d in staged_sidecars] - try: - shutil.copy2(db_path, staging) - for sidecar, side_staging, _side_dst in staged_sidecars: - shutil.copy2(sidecar, side_staging) - # Publish sidecars first, main DB LAST (the commit marker), so a - # mid-publish failure never leaves a countable-but-incomplete bundle. - publish_order = [ - (s, d) for _src, s, d in staged_sidecars - ] + [main_pair] - for src, dst in publish_order: - os.replace(src, dst) - published.append(dst) - except Exception: - # Roll back BOTH unpublished staging files AND anything already - # promoted — the old code unlinked only staging srcs, so a failure - # after the main os.replace left the official backup_path on disk. - for src in all_staging_srcs: - try: - src.unlink(missing_ok=True) - except OSError: - pass - for dst in published: - try: - dst.unlink(missing_ok=True) - except OSError: - pass - try: - staging.unlink(missing_ok=True) - except OSError: - pass - raise - # Retention cap (#86747): keep only the newest few forensic copies. - _prune_malformed_backups(db_path) - return backup_path, None - except Exception as exc: # pragma: no cover - best effort - logger.warning("Could not back up malformed DB %s: %s", db_path, exc) - return None, f"backup copy failed: {exc}" - - -def preflight_db_writability( - db_path: Path, - *, - db_label: str = "state.db", -) -> None: - """Refuse-or-repair read-only DB files BEFORE the first connection opens. - - Port of Kilo-Org/kilocode#12508's startup preflight. A stray read-only - ``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored backup, copied - dotfiles) previously surfaced as an opaque - ``sqlite3.OperationalError: attempt to write a readonly database`` raised - from deep inside ``_init_schema`` — naming no file and no fix — and the - obvious wrong "fix" (deleting the ``-wal``) silently loses committed - transactions. This preflight: - - - **Repairs** permissions with ``chmod u+rw`` when the file lives inside - the Hermes home tree (``get_hermes_home()``) — the safe repair scope: - Hermes owns those files, and the OS makes ``chmod`` fail on files the - user doesn't own, which bounds the repair exactly. - - **Fails fast with an actionable error** naming the exact file and the - exact ``chmod`` command for anything else (root-owned files, read-only - mounts, custom paths outside the home tree). - - Never deletes or truncates a WAL sidecar — once writable, the normal - open path checkpoints its committed frames into the DB as intended. - - ``:memory:`` and ``file:`` URI paths are skipped (no plain on-disk files - to check). Shared by :class:`SessionDB` and ``hermes_cli.kanban_db``. - """ - raw = str(db_path) - if raw == ":memory:" or raw.startswith("file:"): - return - - try: - home: Optional[Path] = Path(get_hermes_home()).resolve() - except Exception: # pragma: no cover - defensive - home = None - - def _in_repair_scope(p: Path) -> bool: - if home is None: - return False - try: - return p.resolve().is_relative_to(home) - except (OSError, ValueError): - return False - - def _ensure_writable(p: Path, *, is_dir: bool = False) -> None: - import stat as _stat - - if os.access(p, os.R_OK | os.W_OK): - return - if _in_repair_scope(p): - try: - add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0) - os.chmod(p, p.stat().st_mode | add) - except OSError: - pass - if os.access(p, os.R_OK | os.W_OK): - logger.info( - "%s preflight: repaired read-only %s (chmod u+rw%s)", - db_label, - p, - "x" if is_dir else "", - ) - return - kind = "directory" if is_dir else "file" - wal_note = ( - " Do NOT delete the -wal file — it contains committed data that " - "will be merged into the database once it is writable." - if p.name.endswith("-wal") - else "" - ) - raise sqlite3.OperationalError( - f"{db_label} is not writable: {kind} {p} is read-only for this " - f"user. Hermes needs read-write access to open the database. " - f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'" - f" (files owned by another user may need sudo/chown).{wal_note}" - ) - - parent = db_path.parent - if parent.is_dir(): - # SQLite needs a writable directory in every journal mode (WAL and - # SHM sidecars in WAL mode; the rollback journal in DELETE mode). - _ensure_writable(parent, is_dir=True) - - for suffix in ("", "-wal", "-shm"): - p = db_path.with_name(db_path.name + suffix) if suffix else db_path - if p.is_file(): - _ensure_writable(p) - - -def _connect_repair_durable( - db_path: Path, *, timeout: float = 5.0 -) -> sqlite3.Connection: - """``sqlite3.connect`` for the repair/probe paths, with macOS write barriers. - - These paths open ``state.db`` directly rather than through ``SessionDB`` - (which routes via :func:`apply_wal_with_fallback`), so they inherited - SQLite's ``synchronous=NORMAL`` default and no ``checkpoint_fullfsync``. - On Darwin that is exactly the combination :func:`_enforce_macos_synchronous_full` - exists to prevent: ``fsync()`` there guarantees neither data-on-platter nor - write ordering, so a rewrite interrupted by process or OS termination can - leave half-written b-tree pages behind. - - That matters more here than anywhere else in the module, because what runs - through these connections is ``REINDEX``, ``VACUUM`` and ``writable_schema`` - surgery — the operations that rewrite nearly every page of the file. The - 2026-08-19 recurrence tore ``messages`` (root page 5) and - ``idx_messages_session``, reporting the unmistakable signature: repeated - "2nd reference to page", a rowid out of order, and long runs of leaked - "never used" pages. - - Autocommit (``isolation_level=None``) is preserved: callers run DDL and - ``VACUUM``, which are illegal inside an implicit transaction. - - Applying the barriers is best-effort *by necessity*: SQLite loads the - schema before it runs any statement, so on a malformed schema even - ``PRAGMA synchronous=FULL`` raises ``DatabaseError`` ("malformed database - schema (messages_fts) - table messages_fts already exists"). A malformed - database is precisely this helper's input, so raising there would leave - repair unable to open the file it exists to fix. Strategies that go on to - rewrite the whole file call :func:`_reapply_durability_barriers` once the - schema parses again, which is the point at which the pragmas can stick. - """ - conn = sqlite3.connect(str(db_path), timeout=timeout, isolation_level=None) - _reapply_durability_barriers(conn) - return conn - - -def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool: - """Best-effort (re)application of the macOS write barriers. Never raises. - - Returns True when the pragmas were accepted. Callers about to rewrite the - file wholesale (``VACUUM``, ``REINDEX``) should call this after the schema - becomes parseable, because a connection opened against a malformed schema - could not take them at open time. - """ - try: - _apply_macos_checkpoint_barrier(conn) - _enforce_macos_synchronous_full(conn) - return True - except sqlite3.DatabaseError: - # Schema still unparseable — the pragmas cannot be set yet. - return False - except Exception: - return False - - -def apply_durability_barriers(conn: sqlite3.Connection) -> bool: - """Apply state-store durability barriers without changing journal mode. - - This is the public entry point for secondary users of ``state.db`` that - must inherit its owner's journal mode while retaining per-connection - durability settings. Also applies the configured ``database.synchronous`` - level (a per-connection pragma that would otherwise only ride on the - journal-mode setup path guest connections must not run). - """ - ok = _reapply_durability_barriers(conn) - try: - # Local import avoids a circular import with hermes_cli.config. - from hermes_cli.config import cfg_get, load_config_readonly - - cfg = load_config_readonly() - raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) - if raw_synchronous is not None: - _apply_synchronous_pragma( - conn, raw_synchronous, db_label="state.db (guest)" - ) - except Exception: - pass - return ok - - -@contextmanager -def _exclusive_repair_db_guard(db_path: Path): - """Yield one live connection that excludes writers for repair surgery. - - ``locking_mode=EXCLUSIVE`` retains SQLite's file-level exclusion after the - short ``BEGIN EXCLUSIVE`` transaction is rolled back. That rollback is - essential: ``Connection.backup`` may use the guarded connection as a - *source* while it is transaction-free, and it must be transaction-free - when it is later the promotion *destination*. The connection itself - remains open for the entire snapshot -> strategies -> promotion window, - so another writer cannot commit a change that promotion could overwrite. - - Existing WAL readers make exclusive acquisition fail rather than being - disturbed. In DELETE mode an existing reader similarly prevents - ``BEGIN EXCLUSIVE``; a future reader/writer waits behind the guard. A - repair therefore fails closed whenever this process cannot own that whole - window. - """ - guard: Optional[sqlite3.Connection] = None - try: - # The cross-process repair lock already serializes repairers. Do not - # wait behind an ordinary application connection: a partial repair is - # less safe than an explicit "stop the gateway and retry" result. - guard = _connect_repair_durable(db_path, timeout=0.0) - guard.execute("PRAGMA locking_mode=EXCLUSIVE") - guard.execute("BEGIN EXCLUSIVE") - guard.execute("ROLLBACK") - except (sqlite3.Error, OSError) as exc: - if guard is not None: - try: - guard.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - guard.close() - yield None, exc - return - - try: - yield guard, None - finally: - try: - # Let SQLite release the exclusive locks before close; this also - # avoids a connection-close checkpoint being mistaken for a - # repair write in callers that immediately reopen state.db. - guard.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - guard.close() - - -def _copy_database_snapshot( - source_path: Path, - destination_path: Path, - *, - source_connection: Optional[sqlite3.Connection] = None, - destination_connection: Optional[sqlite3.Connection] = None, -) -> None: - """Copy one complete SQLite snapshot without replacing either file inode. - - SQLite's online backup API incorporates committed WAL frames into the - source snapshot and writes the destination inside one transaction. This - avoids both the main-file-only staging gap and replacing ``state.db`` from - under handles that already refer to it. If backup is interrupted, SQLite - rolls the destination transaction back. - """ - # Work out the deadline before opening an owned source connection. A - # sidecar disappearing while we stat it is an ordinary staging failure, - # but it must not leak a just-opened SQLite descriptor. - deadline_seconds = _repair_snapshot_timeout_seconds(source_path) - deadline = time.monotonic() + deadline_seconds - source = source_connection or _connect_repair_durable(source_path) - destination = destination_connection - own_source = source_connection is None - own_destination = destination_connection is None - - def _check_deadline(_status: int, _remaining: int, _total: int) -> None: - if time.monotonic() >= deadline: - raise TimeoutError( - "timed out copying SQLite repair snapshot after " - f"{deadline_seconds:.0f}s" - ) - - try: - if destination is None: - destination = _connect_repair_durable(destination_path) - elif destination.in_transaction: - # sqlite3_backup requires a transaction-free destination. The - # exclusive repair guard deliberately retains file exclusion via - # locking_mode, not an active transaction, so it satisfies this. - raise sqlite3.ProgrammingError( - "SQLite repair backup destination has an active transaction" - ) - source.backup( - destination, - pages=256, - progress=_check_deadline, - sleep=_REPAIR_LOCK_POLL_SECONDS, - ) - finally: - if own_destination and destination is not None: - destination.close() - if own_source: - source.close() - - -def _db_opens_cleanly(db_path: Path) -> Optional[str]: - """Probe a DB on a fresh connection. Returns None if healthy, else a reason. - - Runs the same first-statement (``PRAGMA journal_mode``) that trips the - malformed-schema parse, then ``PRAGMA integrity_check`` and a canonical - ``sessions`` read, and finally a rolled-back ``messages`` write so that - FTS5 index corruption — which leaves base-table reads and - ``integrity_check`` passing while every ``INSERT INTO messages`` fails - through the FTS triggers — is reported as unhealthy rather than slipping - past as a false "ok" (#50502). - """ - conn = _connect_repair_durable(db_path) - try: - # Best-effort tokenizer load: a DB carrying the messages_fts_cjk - # index needs the cjk_unicode61 extension before any statement can - # touch that table — including the trigger-driven write probe below. - # Without it, this probe sees the DB exactly as a tokenizer-less - # SessionDB open would (which drops the cjk triggers to keep writes - # working), so tokenizer absence must never classify as corruption. - load_fts5_cjk_extension(conn) - conn.execute("PRAGMA journal_mode").fetchone() - rows = conn.execute("PRAGMA integrity_check").fetchall() - problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"] - if problems: - return "; ".join(problems[:3]) - conn.execute("SELECT COUNT(*) FROM sessions").fetchone() - - # FTS5 read probe: run a representative MATCH query against the - # messages_fts* virtual tables. The FTS *write* probe below catches - # the corruption class where base tables read fine but writes fail - # through the triggers (#50502). It does NOT catch partial FTS5 - # index corruption — bad shadow-table segments where reads still - # parse but MATCH / snippet / rank queries error out with - # "database disk image is malformed" (a `sqlite3.DatabaseError`, - # not `OperationalError`). session_search, /resume title resolution, - # and any feature relying on FTS5 discovery then break silently - # because the official repair tool's check-only path reports the - # DB as healthy. #66724. - # Catch the full sqlite3 exception hierarchy (not just - # OperationalError) so the malformed-shadow-table class is reported - # rather than letting it crash the caller. - for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"): - try: - # No-op queries against the actual FTS5 APIs the search - # tools use. The trigram table is included because it backs - # the title-resolution path; either corruption mode would - # break session recall without this probe. MATCH '""' is - # the empty phrase-token probe — FTS5 rejects MATCH '' - # outright ("fts5: syntax error"), but a quoted empty - # phrase parses, scans zero rows, and exercises the same - # shadow-table read path the search tools use. - conn.execute( - f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1" - ).fetchone() - except sqlite3.OperationalError as exc: - # Use the canonical capability classifier instead of a - # hand-rolled substring check. On SQLite builds without the - # fts5 module, the legacy messages_fts table may exist on - # disk (from a prior build that had FTS5) and MATCH queries - # against it raise OperationalError("no such module: fts5"); - # the substring check below would misclassify that as - # corruption and send the DB into the repair path, whose - # final fallback deletes the messages_fts% schema - # (hermes_state.py:645-723). The supported degraded-runtime - # path (SessionDB._is_fts5_unavailable_error + the - # regression suite in tests/test_hermes_state.py:600-632) - # treats both "no such module: fts5" and - # "no such tokenizer: trigram" as the capability error. - if SessionDB._is_fts5_unavailable_error(exc): - # Degraded runtime — not the corruption class we probe. - continue - msg = str(exc).lower() - if "no such table" in msg or "no such column" in msg: - # FTS5 not built yet (brand new file mid-init) — not the - # corruption class we probe. - continue - return f"fts5 read probe failed on {fts_table}: {exc}" - except sqlite3.DatabaseError as exc: - # This is the corruption class #66724 actually wants caught: - # partial shadow-table damage where MATCH / snippet / rank - # queries raise DatabaseError("database disk image is malformed") - # while reads of the FTS5 table itself parse fine. - return f"fts5 read probe failed on {fts_table}: {exc}" - - # FTS write probe: drive a row through the messages_fts* triggers in a - # transaction that is always rolled back, so a corrupt FTS index that - # rejects writes is caught even though reads look healthy. The probe is - # best-effort — if the messages/sessions tables don't exist yet (brand - # new file mid-init) the OperationalError is treated as "not yet a - # populated DB", not corruption. - probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}" - try: - conn.execute("BEGIN IMMEDIATE") - conn.execute( - "INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)", - (probe_session_id, "_health_probe", time.time()), - ) - conn.execute( - "INSERT INTO messages (session_id, role, content, timestamp) " - "VALUES (?, ?, ?, ?)", - (probe_session_id, "user", "_fts_health_probe", time.time()), - ) - conn.execute("ROLLBACK") - except sqlite3.OperationalError as exc: - # Missing tables / FTS disabled — not the corruption class we probe. - try: - conn.execute("ROLLBACK") - except sqlite3.Error: - pass - msg = str(exc).lower() - if "no such table" in msg or "no such column" in msg: - return None - if "no such tokenizer: cjk_unicode61" in msg: - # This probe process couldn't load the cjk extension while - # the DB carries the cjk index — capability gap, not - # corruption. A tokenizer-capable SessionDB serves it fine; - # a tokenizer-less one self-heals by dropping the triggers. - return None - return str(exc) - return None - except sqlite3.DatabaseError as exc: - return str(exc) - finally: - conn.close() - - -def _live_writer_holds_db(db_path: Path) -> bool: - """True when a connection outside this call still holds ``db_path`` open. - - Detection works by asking SQLite for the thing a repair actually needs and - a live writer cannot grant: ``PRAGMA locking_mode=EXCLUSIVE`` followed by - ``BEGIN IMMEDIATE``. In WAL mode, entering exclusive locking mode - requires exclusive locks on the WAL index, so any other open connection — - reader or writer — makes it fail with SQLITE_BUSY. Neither statement - parses the schema, so this works on the malformed databases repair exists - to handle. - - Fails **open** (returns False) on anything other than a positive - busy/locked signal: refusing to repair a database that nobody is actually - holding would strand the very self-heal path this guard protects. - - Scope: the WAL-index exclusive lock is what makes this detect a holder, so - the guard is effective in WAL mode. On SQLite builds carrying the WAL-reset - bug and on NFS/SMB, Hermes deliberately runs ``state.db`` in - ``journal_mode=DELETE`` (see :func:`apply_wal_with_fallback`); there a held - reader takes only a SHARED lock, ``BEGIN IMMEDIATE`` still acquires - RESERVED, and this probe returns False. In that mode repair is serialised - only by the cross-process repairer lock rather than by this holder probe. - The 2026-08 incident that motivated the guard was in WAL mode, which this - covers; broadening detection to DELETE mode is left to a follow-up. - """ - probe = None - try: - probe = _connect_repair_durable(db_path, timeout=0.0) - probe.execute("PRAGMA locking_mode=EXCLUSIVE") - probe.execute("BEGIN IMMEDIATE") - probe.execute("ROLLBACK") - return False - except sqlite3.OperationalError as exc: - lowered = str(exc).lower() - return "locked" in lowered or "busy" in lowered - except sqlite3.DatabaseError: - # Malformed/unreadable: no evidence of a live holder either way. - return False - except Exception: - return False - finally: - if probe is not None: - try: - # Drop exclusive locking mode before closing so the probe - # itself never leaves the file pinned. - probe.execute("PRAGMA locking_mode=NORMAL") - except Exception: - pass - try: - probe.close() - except Exception: - pass - - -def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]: - """Repair a state.db whose ``sqlite_master`` schema is malformed or whose - FTS indexes reject writes. - - Handles two corruption classes: the "duplicate object definition" / - malformed-schema class where even ``PRAGMA`` statements fail, and the FTS - write-corruption class (#50502) where base tables read fine and - ``integrity_check`` passes but writes fail through the ``messages_fts*`` - triggers. Tries least-destructive recovery first and escalates: - - 1. **Rebuild FTS indexes in place** via the FTS5 ``'rebuild'`` command, - which rewrites the internal b-tree segments from the canonical - ``messages`` rows without dropping or recreating anything. Fixes the - FTS write-corruption class while preserving the schema intact. - 2. **De-duplicate** ``sqlite_master`` (keep the lowest rowid per - ``type``/``name``). Fixes the canonical "table X already exists" - case and PRESERVES the existing FTS index intact. - 3. **Drop the FTS schema** (every ``messages_fts*`` object) + ``VACUUM``. - The next ``SessionDB()`` open rebuilds the FTS indexes from the - canonical ``messages`` table. - - Canonical ``sessions`` / ``messages`` rows are never modified by a failed - attempt. Mutating strategies run against a complete SQLite snapshot and a - successful result is copied back transactionally. A timestamped raw backup - is taken first unless ``backup=False``. - - The surgery below is serialised across processes (see - :func:`_cross_process_repair_lock`): the gateway service, the Desktop - app's backend and interactive CLI sessions all open the same file, and - two of them running ``writable_schema`` surgery concurrently is itself a - corruption source. - - Returns a report dict: ``{repaired: bool, strategy: str|None, - backup_path: str|None, error: str|None}``. - """ - report: Dict[str, Any] = { - "repaired": False, - "strategy": None, - "backup_path": None, - "error": None, - } - - # Startup-watchdog progress lease: repair (raw backup copy + surgery + - # VACUUM) is I/O-bound — near-zero CPU on a multi-GB file — which the - # watchdog's CPU fallback would misread as a parked deadlock (OOF-298). - # Single lease is deliberate (clamped to _MAX_LEASE_S=900): honest worst - # case is up to the lease duration of zombie time on a wedged repair, - # accepted over per-chunk renewal complexity in the repair loop. - report_startup_progress(900.0, phase="state_db_repair") - - db_path = Path(db_path) - if not db_path.exists(): - report["error"] = f"{db_path} does not exist" - return report - - # Cross-restart attempt cap (#86747): the in-memory claim bounds one - # process, but a corruption class the strategies below cannot heal - # (b-tree page damage) previously re-ran the whole surgery — and took a - # fresh multi-hundred-MB forensic backup — on EVERY restart, forever. - # After _MAX_PERSISTENT_REPAIR_ATTEMPTS failures against the same - # damaged file, stop retrying and surface a terminal, actionable error. - if _persistent_repair_attempts_exhausted(db_path): - report["error"] = _persistent_repair_exhausted_error(db_path) - logger.error("state.db repair skipped: %s", report["error"]) - return report - - result = report - with _cross_process_repair_lock(db_path) as holding_lock: - if not holding_lock: - # Another process is still inside its critical section, or the - # lock file itself could not be opened (full disk / no fds). It - # may nonetheless have healed the file already (long VACUUM after - # a successful strategy), so re-probe before reporting failure. - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "repaired_by_other_process" - else: - report["error"] = ( - "could not obtain the state.db repair lock (held by " - "another process, or the lock file was unopenable); " - "skipped schema surgery to avoid racing a concurrent " - "repairer" - ) - else: - # The fast check above avoids taking the lock for a known-exhausted - # image. Recheck after acquisition: a queued repairer can have - # recorded the final failure while this process waited, and this - # process must not start a fourth attempt. - if _persistent_repair_attempts_exhausted(db_path): - report["error"] = _persistent_repair_exhausted_error(db_path) - logger.error("state.db repair skipped: %s", report["error"]) - # Keep the existing WAL-holder preflight: it preserves the - # established fail-closed behaviour for active readers before we - # create a forensic backup. It is not the race defence; the - # retained exclusive guard inside the locked routine is what - # excludes writers continuously through promotion. DELETE-mode - # readers which this probe cannot see are still rejected by the - # later BEGIN EXCLUSIVE acquisition. - elif _live_writer_holds_db(db_path): - report["error"] = ( - "a live writer still holds state.db; skipped schema surgery " - "to avoid tearing b-tree pages under a concurrent writer. " - "Stop the gateway (hermes gateway stop) and retry." - ) - logger.error("state.db repair skipped: %s", report["error"]) - else: - # Probe the mode BEFORE surgery (#89674): every repair - # strategy rewrites the file, and a rebuilt SQLite file comes - # back in the default journal mode (delete) — silently moving - # a WAL store out of WAL with nothing in the logs recording - # the flip. The open-time WAL-reset gate never sees this flip - # because it happens inside the repair path (distinct from - # the open-time flip #89393 warns about). A probe of the - # damaged file may fail, in which case the canonical - # database.journal_mode setting is the restore target. - before_mode = _probe_journal_mode_for_repair(db_path) - result = _repair_state_db_schema_locked( - db_path, backup=backup, report=report - ) - if result.get("repaired"): - result["journal_mode_before"] = before_mode - _restore_journal_mode_after_repair(db_path, before_mode) - # Environmental aborts happen before a strategy gets to mutate the - # isolated snapshot. They are retriable operating conditions, not - # proof that the damaged database exhausted a repair strategy. - # Keep that private signal out of the public report while - # successful health checks still clear a stale persistent failure - # record. This ledger update stays under the same cross-process - # lock as surgery, so two repairers cannot lose each other's - # attempt updates. A queued loser must not record at all: its - # owner is responsible for its outcome. - attempted = bool(result.pop("_repair_attempted", False)) - if attempted or result.get("repaired"): - _record_repair_outcome( - db_path, repaired=bool(result.get("repaired")) - ) - return result - - -def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]: - """Best-effort journal-mode probe for a (possibly malformed) DB file. - - Returns the on-disk mode (``wal``/``delete``), or ``None`` when the file - cannot be opened or probed — a malformed header or a concurrent opener's - locks are both expected on the repair path. Callers fall back to the - configured ``database.journal_mode`` for ``None``. - """ - try: - conn = _connect_repair_durable(db_path) - try: - return _on_disk_journal_mode(conn) - finally: - conn.close() - except (sqlite3.Error, OSError): - return None - - -def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str]) -> None: - """Re-apply the journal mode after schema surgery (#89674). - - A repaired/rebuilt SQLite file comes back in the default journal mode - (delete). Without this restore, a corruption event deterministically - moves a WAL store out of WAL and nothing records the change — the - WAL-reset gate at open time never sees the flip because it happened - inside the repair path, not at open (the open-time flip #89393 warns - about is a different door). - - The restore runs through :func:`apply_wal_with_fallback` — the canonical - journal-mode path — rather than issuing a switch pragma directly, so it - inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new - database: on a vulnerable runtime the gate deliberately keeps it in - DELETE, and "restore could not reach WAL" there is the expected outcome, - not a failure), the macOS-NFS silent-refusal handling, and the WAL - companions (size limit, checkpoint barrier, synchronous=FULL) that the - front door applies. ``before_mode`` is the pre-surgery probe (None when - the damaged file could not be probed) and is only used for the log - comparison — the restore target itself is whatever the canonical path - resolves from ``database.journal_mode``. - - Best-effort by design: the repair itself already succeeded, so failures - to re-apply are logged at WARNING, never raised. - """ - try: - conn = _connect_repair_durable(db_path) - try: - after = apply_wal_with_fallback(conn, db_label=db_path.name) - finally: - conn.close() - if before_mode and after != before_mode: - logger.warning( - "state.db repair changed journal_mode %r -> %r " - "(pre-surgery probe %r; restore resolved through " - "apply_wal_with_fallback per database.journal_mode and the " - "WAL-reset gate)", - before_mode, after, before_mode, - ) - except (sqlite3.Error, OSError) as exc: - logger.warning( - "state.db repair at %s: post-surgery journal-mode restore " - "failed (%s); verify with PRAGMA journal_mode on the next open", - db_path, exc, - ) - - -def _repair_state_db_schema_locked( - db_path: Path, *, backup: bool, report: Dict[str, Any] -) -> Dict[str, Any]: - """Repair strategies for :func:`repair_state_db_schema`. - - Caller must hold the cross-process repair lock for *db_path*. - - The strategies run on a SCRATCH COPY and the result is copied back through - SQLite's transactional backup API only once it is proven to open cleanly. - A repair that does not succeed therefore cannot modify or lose committed - canonical data. In WAL mode SQLite may checkpoint already-committed WAL - frames into the main file while the exclusive guard is released; that is - not a repair mutation and does not change the committed database image. - - They used to run in place, and Strategy 2 ends in ``VACUUM``. VACUUM does - not preserve what it cannot parse: it rebuilds the file from the schema - SQLite can still read, so when the damage IS in the schema b-tree — page - 1's child pointers resolving to data pages, which is exactly the - ``malformed database schema ()`` class this function exists to handle — - every table hanging off the unreadable part is silently dropped. The probe - afterwards then correctly reports the file is STILL malformed, so the - function returns ``repaired=False`` and advises a manual restore, having - already destroyed the thing it was asked to save. Destroying the data and - reporting the repair failed are not mutually exclusive outcomes, and - nothing here treated them as a contradiction. - - The pre-repair backup (#69603) does not close this: it is a forensic - artefact that nothing reads back, so recovery still depends on a human - noticing a ``.malformed-backup-*`` file and knowing what to do with it. - Not mutating the original in the first place is the property that holds - without a human in the loop. - """ - scratch = db_path.with_name(f"{db_path.name}.repair-scratch") - cleanup_error = _unlink_db_triple(scratch) - if cleanup_error is not None: - report["error"] = ( - "could not remove a stale repair snapshot before probing state.db: " - f"{cleanup_error}" - ) - logger.error("state.db repair aborted: %s", report["error"]) - return report - - # Re-probe under the lock: a process we queued behind may have just - # repaired the file, in which case redoing the surgery would undo its - # work on a now-healthy DB (the repair/re-corrupt cascade this lock - # exists to break). - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "already_healthy" - return report - - if backup: - bpath, backup_error = _backup_db_file(db_path) - report["backup_path"] = str(bpath) if bpath else None - if bpath is None: - # HARD STOP (#69603). The forensic recovery image remains required - # when corruption defeats every strategy, even though the - # strategies themselves now run against an isolated snapshot. - report["error"] = ( - "pre-repair backup refused; aborting schema repair to avoid " - f"mutating the only copy of the damaged DB: {backup_error}" - ) - logger.error("state.db repair aborted: %s", report["error"]) - return report - - # The forensic copy intentionally happens before this guard: its raw-copy - # safety checks inspect real live holders and would be poisoned by our - # exclusive connection. Everything that can affect the repair image or - # live promotion happens only after writer exclusion is held. - with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error): - if live_guard is None: - report["error"] = ( - "could not acquire exclusive state.db repair ownership; " - "skipped schema surgery to avoid overwriting a concurrent " - f"writer. Stop the gateway and retry: {guard_error}" - ) - if guard_error is not None and _repair_failure_consumes_attempt( - guard_error - ): - report["_repair_attempted"] = True - logger.error("state.db repair skipped: %s", report["error"]) - return report - - space_error = _repair_scratch_space_error(db_path) - if space_error is not None: - report["error"] = space_error - logger.error("state.db repair aborted: %s", report["error"]) - return report - - try: - # Reuse live_guard rather than opening a second source connection: - # the guard owns the exclusion, so a second connection could be - # blocked by our own EXCLUSIVE lock on some SQLite builds. - _copy_database_snapshot( - db_path, scratch, source_connection=live_guard - ) - except (OSError, sqlite3.Error, TimeoutError) as exc: - report["error"] = ( - f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}" - ) - if _repair_failure_consumes_attempt(exc): - report["_repair_attempted"] = True - logger.error("state.db repair aborted: %s", report["error"]) - _unlink_db_triple(scratch) - return report - - try: - # This private marker is consumed by the outer wrapper. A strategy - # failure is a genuine repair outcome and consumes the persistent - # budget. A later promotion failure is classified separately: - # full disks, I/O, permission and lock failures are environmental - # aborts, not evidence that the strategy cannot repair this image. - report["_repair_attempted"] = True - _run_repair_strategies(scratch, report) - if report.get("repaired"): - try: - # Do not os.replace the live DB: Windows rejects - # replacement under open handles, while POSIX would leave - # those handles on the old inode. The same transaction- - # free guard that staged the live image receives the - # promotion, retaining writer exclusion throughout. - _copy_database_snapshot( - scratch, - db_path, - destination_connection=live_guard, - ) - except (OSError, sqlite3.Error, TimeoutError) as exc: - report["repaired"] = False - report["strategy"] = None - report["_repair_attempted"] = _repair_failure_consumes_attempt( - exc - ) - report["error"] = ( - "repaired snapshot could not be promoted transactionally: " - f"{exc}" - ) - logger.error("state.db repair promotion failed: %s", exc) - else: - logger.warning( - "state.db repaired via '%s' and promoted transactionally: %s", - report.get("strategy"), - db_path, - ) - if not report.get("repaired"): - # Logged HERE, not inside the strategies: they run against the - # scratch copy, and naming that throwaway path in the one - # message a human is meant to act on would send them to a file - # that no longer exists by the time they read it. - logger.error( - "state.db schema repair could not recover %s automatically " - "(no committed canonical data was modified or lost; backup: %s); " - "manual restore from backup may be required.", - db_path, - report["backup_path"], - ) - return report - finally: - # Never leave a half-repaired file beside the DB for a later probe - # — or a later human — to mistake for the real thing. - cleanup_error = _unlink_db_triple(scratch) - if cleanup_error is not None: - logger.warning( - "Could not remove state.db repair snapshot after repair: %s", - cleanup_error, - ) - - -def _unlink_db_triple(path: Path) -> Optional[str]: - """Remove *path* and every SQLite sidecar; return any cleanup failure.""" - failures: List[str] = [] - for suffix in ("", *_DB_SIDECAR_SUFFIXES): - victim = path if not suffix else path.with_name(path.name + suffix) - for attempt in range(10): - try: - victim.unlink() - break - except FileNotFoundError: - break - except PermissionError as exc: - # Windows may retain a just-closed SQLite handle for a few - # scheduler ticks. Bound the retry; a later backup open still - # fails safely if the handle truly remains live. - if _IS_WINDOWS and attempt < 9: - time.sleep(0.05) - continue - failures.append(f"{victim}: {exc}") - break - except OSError as exc: - failures.append(f"{victim}: {exc}") - break - return "; ".join(failures) or None - - -def _run_repair_strategies( - db_path: Path, report: Dict[str, Any] -) -> Dict[str, Any]: - """Escalating repair attempts, applied to *db_path* IN PLACE. - - Every strategy here mutates its argument — FTS rebuild, REINDEX, - ``writable_schema`` surgery, ``VACUUM``. It is therefore only ever called - by :func:`_repair_state_db_schema_locked` on a scratch copy that nothing - else holds open, never on the user's database. - """ - # ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ── - # The FTS5 'rebuild' command rewrites the internal index from the canonical - # content table. This is the recommended, least-destructive recovery for a - # corrupt FTS index that rejects message writes while reads still succeed. - try: - conn = _connect_repair_durable(db_path) - try: - # The cjk index can only be rebuilt with its tokenizer loaded; - # best-effort (a tokenizer-less host skips it at the probe below). - load_fts5_cjk_extension(conn) - for table_name in ( - "messages_fts", "messages_fts_trigram", "messages_fts_cjk" - ): - try: - conn.execute( - f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')" - ) - except sqlite3.OperationalError: - # Table absent (FTS disabled / trigram off / cjk not - # present or tokenizer unavailable) — skip it. - continue - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "rebuild_fts" - logger.warning( - "state.db FTS indexes rebuilt in place (schema preserved): %s", - db_path, - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db FTS in-place rebuild pass failed: %s", exc) - - # ── Strategy 0.5: rebuild stale B-tree indexes (#63386) ── - # PRAGMA integrity_check can report "wrong # of entries in index" when a - # B-tree index (e.g. idx_sessions_handoff_state) falls out of sync with its - # base table. REINDEX rewrites the index b-tree from the canonical table - # rows using the existing index definition, fixing the mismatch without - # touching data or FTS schema. - try: - conn = _connect_repair_durable(db_path) - try: - # REINDEX rewrites every index b-tree; take the barriers now that - # the schema parses, in case the open-time attempt was refused. - _reapply_durability_barriers(conn) - conn.execute("REINDEX") - conn.commit() - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "reindex_btree" - logger.warning( - "state.db B-tree indexes rebuilt via REINDEX: %s", db_path - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db REINDEX pass failed: %s", exc) - - # ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ── - try: - conn = _connect_repair_durable(db_path) - try: - conn.execute("PRAGMA writable_schema=ON") - dupes = conn.execute( - "SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep " - "FROM sqlite_master GROUP BY type, name HAVING c > 1" - ).fetchall() - for type_, name, _count, keep in dupes: - conn.execute( - "DELETE FROM sqlite_master " - "WHERE type IS ? AND name IS ? AND rowid <> ?", - (type_, name, keep), - ) - if dupes: - _bump_schema_cookie(conn) - conn.execute("PRAGMA writable_schema=OFF") - conn.commit() - finally: - conn.close() - if _db_opens_cleanly(db_path) is None: - report["repaired"] = True - report["strategy"] = "dedup_schema" - logger.warning( - "state.db schema repaired by de-duplicating sqlite_master " - "(FTS index preserved): %s", db_path - ) - return report - except sqlite3.DatabaseError as exc: - logger.warning("state.db dedup repair pass failed: %s", exc) - - # ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ── - # - # The destructive one, and the reason this whole path now runs on a - # scratch copy. VACUUM rebuilds the file from the schema SQLite can still - # parse, so on a damaged schema b-tree it silently drops every table - # hanging off the unreadable part — and the probe below then correctly - # reports the result is still malformed. On a scratch copy that is merely - # a discarded attempt; on the live file it was data loss. - try: - conn = _connect_repair_durable(db_path) - try: - conn.execute("PRAGMA writable_schema=ON") - conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'") - _bump_schema_cookie(conn) - conn.execute("PRAGMA writable_schema=OFF") - conn.commit() - # The schema is repaired and parseable now, so the barriers can - # finally stick — and VACUUM, which rewrites the entire file, is - # the single most damaging operation to lose halfway. - _reapply_durability_barriers(conn) - conn.execute("VACUUM") - finally: - conn.close() - reason = _db_opens_cleanly(db_path) - if reason is None: - report["repaired"] = True - report["strategy"] = "drop_fts_rebuild" - logger.warning( - "state.db schema repaired by dropping FTS schema; indexes " - "will rebuild from messages on next open: %s", db_path - ) - return report - report["error"] = reason - except sqlite3.DatabaseError as exc: - report["error"] = str(exc) - - # The "could not recover" log lives in the caller: it must name the user's - # database, not the scratch copy these strategies were handed. - return report - # ── CJK-bigram FTS index (replaces the trigram index when available) ──── # @@ -4470,186 +1768,6 @@ def divert_session_transcript_jsonl(session_id: str, messages) -> "Optional[Path return path -# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced, -# against the LIVE state.db. A bare open()/read()/close() there is the -# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this -# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call -# drops the WAL-mode DMS shared lock the writer connection holds on state.db -# (see hermes_cli/sqlite_safe_read.py for the module built around this rule). -# With the DMS lock gone, a fresh opener in another process can treat this -# writer as dead and rerun WAL-index recovery underneath it. -# -# The probe therefore reads through a per-path fd cached for the life of the -# process: opening an fd never cancels locks (only close() does), and -# os.pread takes no shared file position. When the path is re-pointed at a -# new inode (the very replacement this probe exists to detect), the stale fd -# is RETIRED, never closed — closing it would cancel the live connection's -# locks on the old file, the exact bug being avoided. Replacement events are -# rare and halt writes anyway, so the leak is bounded. -_HEADER_PROBE_LOCK = threading.Lock() -_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino) -_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed - - -def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]": - """Lock-safe raw header read of a possibly-live SQLite database. - - POSIX: pread from a cached, never-closed fd (rebound when the path names - a new inode). Windows: plain read — advisory-lock cancellation is a - POSIX-only hazard and msvcrt locks do not share the failure mode. - """ - if _IS_WINDOWS: - try: - with db_path.open("rb") as handle: - return handle.read(length) - except OSError: - return None - key = str(db_path) - try: - st = os.stat(db_path) - except OSError: - return None - with _HEADER_PROBE_LOCK: - cached = _HEADER_PROBE_FDS.get(key) - if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino): - # Path re-pointed at a new file. Retire (never close) the old fd. - _RETIRED_HEADER_PROBE_FDS.append(cached[0]) - cached = None - del _HEADER_PROBE_FDS[key] - if cached is None: - try: - fd = os.open(db_path, os.O_RDONLY) - except OSError: - return None - try: - fst = os.fstat(fd) - except OSError: - _RETIRED_HEADER_PROBE_FDS.append(fd) - return None - cached = (fd, fst.st_dev, fst.st_ino) - _HEADER_PROBE_FDS[key] = cached - try: - return os.pread(cached[0], length, 0) - except OSError: - return None - - -def _read_sqlite_application_id(db_path: Path) -> "Optional[int]": - """Read application_id from the SQLite header without opening a connection. - - Safe against live databases: routed through :func:`_pread_db_header`, - which never issues a ``close()`` that would cancel this process's POSIX - locks on the file (howtocorrupt §2.2). - """ - header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4) - if header is None: - return None - if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4: - return None - if header[:16] != b"SQLite format 3\x00": - return None - return int( - struct.unpack( - ">I", - header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4], - )[0] - ) - - -def _stat_db_file_identity(path: Path) -> "Optional[tuple]": - """Return ``(st_dev, st_ino)`` for *path*, or None when identity is unavailable.""" - try: - st = os.stat(path) - except OSError: - return None - # Windows volumes (and some network FS) report st_ino=0; a (0, 0) - # identity would false-positive every check. Skip the inode half of - # the guard there; generation stamp still applies. - if not st.st_dev or not st.st_ino: - return None - return (st.st_dev, st.st_ino) - - -def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]: - """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars.""" - identities: Dict[str, tuple] = {} - base = os.fspath(db_path) - for suffix in ("-wal", "-shm"): - ident = _stat_db_file_identity(Path(base + suffix)) - if ident is not None: - identities[suffix] = ident - return identities - - -def _canonical_sqlite_path(path: str) -> str: - """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix.""" - return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)"))) - - -def _watched_sqlite_sidecar_paths(db_path) -> Set[str]: - base = os.path.abspath(os.fspath(db_path)) - return { - _canonical_sqlite_path(base + "-wal"), - _canonical_sqlite_path(base + "-shm"), - } - - -def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]: - """Return processes holding an unlinked ``state.db-wal`` / ``-shm``. - - Linux-only (``/proc//fd`` readlink). Windows and other hosts - return ``[]`` — Windows cannot unlink a sidecar another process still - holds, and macOS does not use the `` (deleted)`` suffix. - - The scan includes this process: on the SessionDB open/write refuse - path, the in-process writer that still holds the orphan inode is the - one that must not mint a replacement WAL (and must stop committing). - ``_foreign_state_db_holders`` keeps skipping this PID for FTS - maintenance so a process does not block its own optional repair. - """ - if not sys.platform.startswith("linux"): - return [] - - holders: List[Tuple[int, str]] = [] - watched = _watched_sqlite_sidecar_paths(db_path) - try: - for pid_str in os.listdir("/proc"): - if not pid_str.isdigit(): - continue - pid = int(pid_str) - fd_dir = f"/proc/{pid}/fd" - try: - fds = os.listdir(fd_dir) - except OSError: - continue - for fd in fds: - try: - target = os.readlink(f"{fd_dir}/{fd}") - except OSError: - continue - if " (deleted)" not in target: - continue - if _canonical_sqlite_path(target) in watched: - holders.append((pid, target)) - except Exception as exc: - logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc) - return holders - return holders - - -def refuse_deleted_wal_generation(db_path) -> None: - """Raise if any process holds a deleted WAL/SHM generation for *db_path*. - - Called *before* ``sqlite3.connect`` so a second opener cannot mint a - replacement WAL inode while a live writer still holds the orphan. - """ - holders = iter_deleted_sqlite_sidecar_holders(db_path) - if not holders: - return - logger.error(_DELETED_WAL_GENERATION_MSG) - raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) - - # ── Process-wide shared SessionDB registry (#90837) ── # # The registry itself lives in hermes_state_registry.py — a bounded @@ -4673,478 +1791,9 @@ from hermes_state_registry import ( # noqa: F401 (re-export) ) - -def _connect_tracked_db(path, tracking_path=None, **kwargs): - """``sqlite3.connect`` that registers the open fd for lock-safety. - - While a connection is live, byte-level probes of the same file are - refused: an ``open()``/``close()`` cancels every POSIX advisory lock this - process holds on it -- including a running VACUUM's EXCLUSIVE lock. - Released automatically on ``close()``. - - The ONLY tolerated fallback is the helper being absent entirely - (scaffold/embed installs that ship hermes_state without hermes_cli). A - real connection failure must propagate: silently retrying an *untracked* - connect would disable the guard for the lifetime of that connection, - which is precisely the failure mode this module exists to prevent. - """ - try: - from hermes_cli.sqlite_safe_read import connect_tracked - except ImportError: - logger.debug( - "hermes_cli.sqlite_safe_read unavailable; opening %s untracked " - "(byte-probe guard inactive in this install)", - path, - ) - return sqlite3.connect(str(path), **kwargs) - - # Open through THIS module's sqlite3.connect so callers (and tests) that - # patch hermes_state.sqlite3.connect keep control of connection creation; - # the helper still owns tracking. - return connect_tracked( - path, - tracking_path=tracking_path, - connect_fn=sqlite3.connect, - **kwargs, - ) - - -def is_zeroed_state_db( - path: Path, *, probe_bytes: int = 100, force: bool = False -) -> bool: - """Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header). - - Byte-level probe, so it is only safe BEFORE any connection to *path* - exists in this process: ``close()`` cancels every POSIX advisory lock the - process holds on the file, which can pull the EXCLUSIVE lock out from - under a running VACUUM and corrupt the database. The read is routed - through ``read_header_bytes_preopen``, which refuses (returning False - here) once a connection is live. Pass ``force=True`` only for offline - files -- quarantined copies, snapshots, archives. - - Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this - local copy keeps SessionDB openable without importing the CLI package - in constrained embed paths. - """ - try: - from hermes_cli.backup import is_zeroed_sqlite_file - - return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force) - except Exception: - pass - try: - if not path.is_file(): - # Special files (FIFO, device, socket) are never "zeroed", and - # probing a FIFO would block until a writer appears. - return False - size = path.stat().st_size - except OSError: - return False - if size < 0: - return False - from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen - - if not force and has_live_connection(path): - return False - - head = read_header_bytes_preopen( - path, length=max(16, probe_bytes), force=force - ) - if head is None: - return False - if len(head) == 0: - return True - if head.startswith(b"SQLite format 3"): - return False - return all(byte == 0 for byte in head) - - -@contextlib.contextmanager -def quarantine_cross_process_lock(path: Path, timeout: float = 5.0): - """Acquire the cross-process lock for path.quarantine.lock.""" - import platform - - lock_path = path.with_name(path.name + ".quarantine.lock") - lock_path.parent.mkdir(parents=True, exist_ok=True) - handle = lock_path.open("a+b") - acquired = False - try: - deadline = time.monotonic() + timeout - if platform.system() == "Windows": - import msvcrt - - while True: - try: - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) - acquired = True - break - except OSError: - if time.monotonic() >= deadline: - break - time.sleep(0.020) - else: - import fcntl - - while True: - try: - fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) - acquired = True - break - except (BlockingIOError, OSError): - if time.monotonic() >= deadline: - break - time.sleep(0.020) - yield acquired - finally: - try: - if acquired: - if platform.system() == "Windows": - import msvcrt - - handle.seek(0) - msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) - else: - import fcntl - - fcntl.flock(handle.fileno(), fcntl.LOCK_UN) - except (OSError, AttributeError): - pass - finally: - handle.close() - - -def quarantine_zeroed_state_db( - path: Path, *, already_locked: bool = False -) -> Optional[Path]: - """Move a zeroed state.db aside (preserve bytes) and return quarantine path. - - Uses a cross-process lock (``#68805``) so two concurrent startups cannot - race: the first process moves the zeroed file and the second re-checks - under the lock, finding the file already gone (or a fresh DB in its place) - instead of clobbering the quarantine. - """ - def _do_quarantine(): - if not path.exists(): - logger.info( - "quarantine_zeroed_state_db: %s already moved by another process", - path, - ) - return None - if not is_zeroed_state_db(path): - logger.info( - "quarantine_zeroed_state_db: %s is no longer zeroed (another " - "process quarantined it and a fresh DB was created)", - path, - ) - return None - - try: - ts = time.strftime("%Y%m%d-%H%M%S") - except Exception: - ts = "unknown" - dest = path.with_name( - f"{path.name}.zeroed-{ts}-{os.getpid()}.bak" - ) - n = 0 - while dest.exists(): - n += 1 - dest = path.with_name( - f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak" - ) - try: - path.rename(dest) - except OSError as exc: - logger.error("Failed to quarantine zeroed %s: %s", path, exc) - return None - for suffix in ("-wal", "-shm"): - side = Path(str(path) + suffix) - if side.exists(): - try: - side.rename(Path(str(dest) + suffix)) - except OSError: - pass - return dest - - if already_locked: - return _do_quarantine() - - with quarantine_cross_process_lock(path) as acquired: - if not acquired: - logger.error( - "quarantine lock for %s not acquired within 5s — refusing to " - "quarantine without the cross-process lock. The zeroed file " - "is left in place. If sessions fail to load, restore from " - "state-snapshots via `hermes snapshot list` / " - "`hermes snapshot restore `.", - path, - ) - return None - return _do_quarantine() - - # ── Read-only health/stats probes (hermes doctor, dashboards) ────────── -def collect_state_db_stats(db_path: Path) -> Dict[str, Any]: - """Best-effort, strictly read-only stats snapshot of a state.db file. - - Opens the database with ``mode=ro`` (URI) and a short timeout so it can - run against a *live* database held by a gateway without ever taking a - write lock or mutating the file. Every field is collected independently: - a failed pragma/SELECT yields ``None`` for that field, and the helper - itself never raises. - - Deliberately does NOT instantiate :class:`SessionDB` — its constructor - runs schema DDL (migrations, FTS table creation), which is exactly the - kind of write a diagnostics probe must never perform. - - Returned keys (all present, any may be None on failure): - - - ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values - - ``logical_size_bytes`` — page_count * page_size (post-checkpoint size) - - ``wal_size_bytes`` — stat() of ``-wal`` (0 when absent) - - ``journal_mode`` — PRAGMA journal_mode string - - ``messages`` / ``sessions`` — row counts - - ``fts_tables`` — dict of {table_name: bool} presence for - messages_fts / messages_fts_trigram / messages_fts_cjk - - ``fts_storage_version`` — int from state_meta, None when the marker is - absent (legacy pre-v23 inline layout) - - ``fts_rebuild_pending`` — True when the deferred v23 backfill has not - finished (high_water present and progress < high_water) - - ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints - - ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present - """ - stats: Dict[str, Any] = { - "page_count": None, - "page_size": None, - "freelist_count": None, - "logical_size_bytes": None, - "wal_size_bytes": None, - "journal_mode": None, - "messages": None, - "sessions": None, - "fts_tables": None, - "fts_storage_version": None, - "fts_rebuild_pending": None, - "fts_rebuild_high_water": None, - "fts_rebuild_progress": None, - "fts_rebuild_deferral": None, - } - - # WAL sidecar size needs no connection at all. - try: - wal_path = Path(str(db_path) + "-wal") - stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0 - except OSError: - pass - - conn = None - try: - # mode=ro refuses to create the file and refuses every write; a - # short timeout keeps doctor snappy when a writer holds the lock. - # Route through the tracked connect so byte-probe helpers - # (read_header_bytes_preopen) see this connection and refuse raw - # opens that could cancel our POSIX locks mid-read. - conn = _connect_tracked_db( - f"file:{Path(db_path)}?mode=ro", - tracking_path=Path(db_path), - uri=True, - timeout=2.0, - ) - except Exception as exc: - logger.debug("collect_state_db_stats: cannot open %s read-only: %s", - db_path, exc) - return stats - - def _scalar(sql: str) -> Any: - try: - row = conn.execute(sql).fetchone() - return row[0] if row else None - except Exception: - return None - - try: - pc = _scalar("PRAGMA page_count") - ps = _scalar("PRAGMA page_size") - stats["page_count"] = int(pc) if pc is not None else None - stats["page_size"] = int(ps) if ps is not None else None - if stats["page_count"] is not None and stats["page_size"] is not None: - stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"] - - fl = _scalar("PRAGMA freelist_count") - stats["freelist_count"] = int(fl) if fl is not None else None - - jm = _scalar("PRAGMA journal_mode") - stats["journal_mode"] = str(jm) if jm is not None else None - - msgs = _scalar("SELECT COUNT(*) FROM messages") - stats["messages"] = int(msgs) if msgs is not None else None - sess = _scalar("SELECT COUNT(*) FROM sessions") - stats["sessions"] = int(sess) if sess is not None else None - - # FTS table presence via sqlite_master (never SELECTs from the - # virtual tables themselves — a corrupt index must not fail stats). - try: - names = { - row[0] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name IN (?, ?, ?)", - ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"), - ).fetchall() - } - stats["fts_tables"] = { - t: (t in names) - for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") - } - except Exception: - pass - - # Raw state_meta reads — cheap, and independent of SessionDB. - def _meta_int(key: str) -> Optional[int]: - try: - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ?", (key,) - ).fetchone() - return int(row[0]) if row and row[0] is not None else None - except Exception: - return None - - stats["fts_storage_version"] = _meta_int("fts_storage_version") - high_water = _meta_int("fts_rebuild_high_water") - progress = _meta_int("fts_rebuild_progress") - stats["fts_rebuild_high_water"] = high_water - stats["fts_rebuild_progress"] = progress - if high_water is None: - stats["fts_rebuild_pending"] = False - else: - stats["fts_rebuild_pending"] = (progress or 0) < high_water - try: - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ? LIMIT 1", - (FTS_REBUILD_DEFERRAL_KEY,), - ).fetchone() - if row: - parsed = json.loads(row[0]) - if isinstance(parsed, dict): - stats["fts_rebuild_deferral"] = parsed - except Exception: - pass - finally: - try: - conn.close() - except Exception: - pass - - return stats - - -def count_db_holders(db_path: Path) -> Optional[int]: - """Best-effort count of processes holding ``db_path`` open (Linux only). - - Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns - the number of distinct PIDs with the file open, or ``None`` on any - error or on non-Linux platforms. Never raises; no lsof dependency. - Unreadable per-process fd dirs (other users' processes without root) - are silently skipped, so the count is a lower bound. - """ - try: - if not sys.platform.startswith("linux"): - return None - target = os.path.realpath(str(db_path)) - holders = 0 - for pid in os.listdir("/proc"): - if not pid.isdigit(): - continue - fd_dir = f"/proc/{pid}/fd" - try: - fds = os.listdir(fd_dir) - except OSError: - continue # process gone or not ours - for fd in fds: - try: - if os.readlink(f"{fd_dir}/{fd}") == target: - holders += 1 - break # one hit per PID - except OSError: - continue - return holders - except Exception: - return None - - -def _is_inactive_orphan_desktop_holder( - *, - ppid: int, - age_seconds: float, - min_age_seconds: float, - ephemeral_backend: bool, - connection_statuses: List[str], -) -> bool: - """Pure safety predicate for the narrow Desktop holder reap.""" - return ( - ppid in (0, 1) - and age_seconds >= min_age_seconds - and ephemeral_backend - and "ESTABLISHED" not in connection_statuses - ) - - -def _concrete_state_db_holder_pids( - db_path: Path, holders: List[Tuple[int, str]] -) -> List[int]: - """Return unique PIDs proven to hold this DB or one of its sidecars.""" - canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path))) - watched = { - canonical_db, - canonical_db + "-wal", - canonical_db + "-shm", - } - pids: List[int] = [] - seen = set() - for pid, path in holders: - canonical_path = os.path.normcase( - os.path.abspath(path.removesuffix(" (deleted)")) - ) - if pid <= 0 or pid in seen or canonical_path not in watched: - continue - seen.add(pid) - pids.append(pid) - return pids - - -def _read_proc_cmdline(pid: int) -> Optional[str]: - """Read /proc//cmdline, world-readable even when fd table is not. - - Returns the cmdline as a space-joined string, or None when unreadable - (process exited, or hidepid mount). - """ - try: - with open(f"/proc/{pid}/cmdline", "rb") as f: - raw = f.read() - if not raw: - return None - return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() - except OSError: - return None - - -_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve", - "hermes-agent", "hermes gateway", "hermes chat") - - -def _looks_like_hermes(cmdline: str) -> bool: - """Heuristic: does this cmdline look like a Hermes process? - - Used to decide whether an uninspectable process (fd table unreadable - due to different user) should be treated as a potential state.db holder. - We only flag processes that look like Hermes, not every system daemon. - """ - lower = cmdline.lower() - return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS) - - # Lifecycle statuses surfaced by session pickers. Classification looks ONLY at # a session's final message row — role, whether it carries tool_calls, and its # finish_reason — so it stays O(1) per session (see @@ -5199,7 +1848,18 @@ _SAME_KEY_NAMESPACE_SQL = ( ) -class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin): +class SessionDB( + SessionSearchMixin, + SessionSchemaMixin, + SessionPortabilityMixin, + SessionTelegramTopicsMixin, + SessionCompressionMixin, + SessionGatewayMixin, + SessionMaintenanceMixin, + SessionUsageMixin, + SessionTitlesMixin, + SessionMessagesMixin, +): """ SQLite-backed session storage with FTS5 search. @@ -6077,7 +2737,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return False - sql = (row[0] if not isinstance(row, sqlite3.Row) else row["sql"]) or "" + sql = row[0] or "" # The v23 table declares tool_name/tool_calls columns. Their absence # means a legacy shape that doesn't index tool metadata → optimize. return "tool_name" not in sql @@ -6448,6 +3108,46 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) continue raise + def _write_sql( + self, + sql: str, + params: Any = (), + *, + many: bool = False, + patience_s: Optional[float] = None, + ) -> None: + """Run one INSERT/UPDATE/DELETE through ``_execute_write``.""" + def _do(conn): + (conn.executemany if many else conn.execute)(sql, params) + + self._execute_write(_do, patience_s=patience_s) + + def _write_rowcount( + self, sql: str, params: Any = (), *, patience_s: Optional[float] = None + ) -> int: + """Run one UPDATE/DELETE through ``_execute_write``; return rows changed. + + Falls back to ``SELECT changes()`` when the driver reports an unknown + rowcount (None / negative). + """ + def _do(conn): + rowcount = conn.execute(sql, params).rowcount + if rowcount is None or rowcount < 0: + rowcount = conn.execute("SELECT changes()").fetchone()[0] + return rowcount + + return self._execute_write(_do, patience_s=patience_s) + + def _read_one(self, sql: str, params: Any = ()) -> Optional[sqlite3.Row]: + """``fetchone()`` of one read-only statement via ``_read_ctx``.""" + with self._read_ctx() as conn: + return conn.execute(sql, params).fetchone() + + def _read_all(self, sql: str, params: Any = ()) -> List[sqlite3.Row]: + """``fetchall()`` of one read-only statement via ``_read_ctx``.""" + with self._read_ctx() as conn: + return conn.execute(sql, params).fetchall() + def _ensure_db_file_generation(self) -> None: """Mint a once-per-file generation stamp (state_meta + application_id). @@ -6498,8 +3198,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._db_file_application_id = disk_id elif self._conn is not None and not self._db_file_application_id: try: - with self._read_ctx() as conn: - pragma_row = conn.execute("PRAGMA application_id").fetchone() + pragma_row = self._read_one("PRAGMA application_id") if pragma_row and pragma_row[0]: self._db_file_application_id = int(pragma_row[0]) except sqlite3.Error: @@ -6819,66 +3518,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return holders or [(-1, f"open-file scan failed: {exc}")] return holders - def _reap_inactive_orphan_desktop_holders( - self, holders: List[Tuple[int, str]], *, min_age_seconds: float - ) -> List[int]: - """Terminate old PPID-1 Desktop ephemeral backends with no client. - - Inspection fails closed: anything whose parent, age, argv, or network - connections cannot be proved safe remains a repair-blocking holder. - """ - if not sys.platform.startswith("linux") or psutil is None: - return [] - try: - from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend - except Exception: - return [] - - now = time.time() - candidates = [] - for pid in _concrete_state_db_holder_pids(self.db_path, holders): - try: - process = psutil.Process(pid) - statuses = [ - conn.status for conn in process.net_connections(kind="inet") - ] - if not _is_inactive_orphan_desktop_holder( - ppid=process.ppid(), - age_seconds=now - process.create_time(), - min_age_seconds=min_age_seconds, - ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()), - connection_statuses=statuses, - ): - continue - except Exception: - continue - candidates.append(process) - - signalled: List[int] = [] - for process in candidates: - try: - process.terminate() - signalled.append(process.pid) - except (psutil.Error, OSError): - continue - if not signalled: - return [] - - try: - _gone, alive = psutil.wait_procs(candidates, timeout=1.5) - except Exception: - alive = [] - for process in alive: - try: - process.kill() - except (psutil.Error, OSError): - continue - if alive: - try: - psutil.wait_procs(alive, timeout=1.5) - except Exception: - pass - return signalled def _enter_fts_fail_open(self, exc: sqlite3.DatabaseError) -> bool: """Detach corrupt FTS indexes so canonical writes can continue. @@ -7233,51 +3872,33 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) -> None: """Insert a session row, enriching NULL metadata on conflict. - The gateway's ``get_or_create_session`` creates a bare row (source + - user_id) *before* the agent exists; the agent's later - ``create_session`` then carries the real ``model`` / ``model_config`` / - ``system_prompt``. A plain ``INSERT OR IGNORE`` silently dropped that - enrichment, leaving gateway sessions with NULL model/billing metadata. - The ``ON CONFLICT`` upsert backfills those fields via ``COALESCE`` — - only filling columns that are still NULL, never overwriting values an - earlier writer already set (so a later bare call with source="unknown" - can't clobber a real source/model). + The gateway creates a bare row (source + user_id) before the agent + exists and the agent's later ``create_session`` carries the real + model/model_config/system_prompt; ``INSERT OR IGNORE`` dropped that + enrichment, so the upsert ``COALESCE``-fills columns that are still + NULL and never overwrites what an earlier writer set (a later bare + source="unknown" call cannot clobber a real source/model). - ``chat_id``/``thread_id`` record the messaging origin (the chat/room and - thread the session was started in) so that gateway ``/resume`` can prove - a persisted, now-inactive row belongs to the caller's chat/thread before - switching to it (IDOR scoping — without them the ``sessions`` table has - no chat/thread to compare). + ``chat_id``/``thread_id`` record the messaging origin so gateway + ``/resume`` can prove an inactive row belongs to the caller's + chat/thread (IDOR scoping). - When ``parent_session_id`` is set (compression fork, delegate/subagent - spawn, branch continuation) and this row's own ``cwd``/``git_repo_root``/ - ``git_branch``/``profile_name`` are still NULL after the insert, they are - backfilled from the parent row. Callers of ``create_session`` for a child - session historically didn't propagate these fields themselves (e.g. the - compression-fork path), so a lineage could silently lose its working - directory and drop out of the project sidebar every time it forked - (#64709), or lose its owning profile and be aggregated as "default" every - time it rotated or branched (the cross-profile session-jump bug). This - only fills NULLs — an explicit value on the child is never overwritten. - For compression forks specifically - (parent ended with ``end_reason='compression'``), the gateway origin - columns (``user_id``/``session_key``/``chat_id``/``chat_type``/ - ``thread_id``/``display_name``/``origin_json``) are inherited too, so a - crash before the gateway re-records the peer can't strand the child - without a recoverable routing mapping (#59527). + With ``parent_session_id`` (compression fork, delegate spawn, branch), + NULL ``cwd``/``git_repo_root``/``git_branch``/``profile_name`` are + backfilled from the parent — child creators historically did not + propagate them, so lineages dropped out of the project sidebar or were + aggregated as "default" on every fork. NULL-fill only. Compression + forks (parent ``end_reason='compression'``) also inherit the gateway + origin columns (user_id/session_key/chat_id/chat_type/thread_id/ + display_name/origin_json) so a crash before the gateway re-records the + peer cannot strand the child without a routing mapping. - When the caller passes no ``profile_name`` at all, the row is stamped - with THIS store's own profile (:meth:`_own_profile_name`) instead of - NULL. Every ``state.db`` belongs to exactly one profile — the same - single-match contract :meth:`backfill_null_session_profiles` relies - on — so the stamp is derivation, not a guess. Rows minted NULL after - that one-shot #94724 backfill ran stayed NULL forever, and - profile-keyed consumers (desktop sidebar scope matching, - ``@session:/`` deep links, the fail-closed owner ladder) - treat NULL as unowned: the session vanishes from the sidebar even - though its transcript is intact (#99222). Stores outside the profile - tree (explicit ``db_path`` in tests, ad-hoc copies) derive nothing - and keep NULL — never guess. + With no ``profile_name`` the row is stamped with THIS store's own + profile (:meth:`_own_profile_name`): every state.db belongs to exactly + one profile (the contract :meth:`backfill_null_session_profiles` + relies on), and profile-keyed consumers treat NULL as unowned, hiding + the session from the sidebar. Stores outside the profile tree derive + nothing and keep NULL — never guess. """ if not (profile_name or "").strip(): profile_name = self._own_profile_name() @@ -7426,140 +4047,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._insert_session_row(session_id, source, **kwargs) return session_id - def record_gateway_session_peer( - self, - session_id: str, - *, - source: str, - user_id: str = None, - session_key: str = None, - chat_id: str = None, - chat_type: str = None, - thread_id: str = None, - display_name: str = None, - origin_json: str = None, - include_compression_ancestors: bool = False, - ) -> None: - """Persist the gateway routing peer for an existing session row. - - ``display_name`` / ``origin_json`` carry the gateway's presentation - and full origin metadata (#9006) so consumers (mcp_serve, mirror, - channel directory) can read routing data from state.db instead of - sessions.json. They are COALESCE'd only in the sense that ``None`` - leaves the existing value untouched. - - ``include_compression_ancestors`` keeps a logical compression lineage - on one routing peer when an explicit gateway resume moves its tip to a - different lane. Normal per-turn metadata refreshes update only the - supplied row. - - Self-healing (#82616): when the target row does not exist yet — the - gateway's ``create_session`` write failed and was deferred, or a - crash landed between routing publication and row creation — this - recorder INSERTs the row with the full identity instead of silently - no-opping. Every per-turn peer refresh is therefore a repair - opportunity: a gateway session row can no longer be first-created by - an identity-less lazy writer (``update_token_counts`` / - ``record_auxiliary_usage``) and stay unroutable forever. - """ - if not session_id or not session_key: - return - - def _do(conn): - lineage_cte = "" - target_clause = "WHERE id = ?" - query_params = [] - if include_compression_ancestors: - lineage_cte = """ - WITH RECURSIVE compression_lineage(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM compression_lineage lineage - JOIN sessions child ON child.id = lineage.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - AND json_extract( - COALESCE(child.model_config, '{}'), - '$._branched_from' - ) IS NULL - AND json_extract( - COALESCE(child.model_config, '{}'), - '$._delegate_from' - ) IS NULL - AND COALESCE(child.source, '') != 'tool' - ) - """ - target_clause = "WHERE id IN (SELECT id FROM compression_lineage)" - query_params.append(session_id) - query_params.extend( - ( - session_key, - source, - user_id, - chat_id, - chat_type, - thread_id, - display_name, - origin_json, - ) - ) - if not include_compression_ancestors: - query_params.append(session_id) - conn.execute( - f"""{lineage_cte} - UPDATE sessions - SET session_key = ?, source = ?, user_id = ?, chat_id = ?, - chat_type = ?, thread_id = ?, - display_name = COALESCE(?, display_name), - origin_json = COALESCE(?, origin_json) - {target_clause}""", - query_params, - ) - # Self-heal (#82616): the UPDATE is a silent no-op when the row - # is missing (create_session failed earlier, or a crash landed - # between routing publication and row creation). Insert it with - # the full identity so the session is durably routable — never - # leave first-creation to an identity-less lazy writer. - if not include_compression_ancestors: - cur = conn.execute( - "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,) - ) - if cur.fetchone() is None: - conn.execute( - """INSERT INTO sessions ( - id, source, user_id, session_key, chat_id, - chat_type, thread_id, display_name, origin_json, - profile_name, started_at - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(id) DO UPDATE SET - session_key = COALESCE(sessions.session_key, excluded.session_key), - chat_id = COALESCE(sessions.chat_id, excluded.chat_id), - chat_type = COALESCE(sessions.chat_type, excluded.chat_type), - thread_id = COALESCE(sessions.thread_id, excluded.thread_id), - display_name = COALESCE(sessions.display_name, excluded.display_name), - origin_json = COALESCE(sessions.origin_json, excluded.origin_json)""", - ( - session_id, - source, - user_id, - session_key, - chat_id, - chat_type, - thread_id, - display_name, - origin_json, - # Same ownership stamp as _insert_session_row: a - # self-healed row is a first creation too, and an - # unowned (NULL) row vanishes from profile-keyed - # consumers (#99222). - self._own_profile_name(), - time.time(), - ), - ) - - self._execute_write(_do) def set_expiry_finalized(self, session_id: str, finalized: bool = True) -> None: """Mark a gateway session's expiry-finalization flag in state.db. @@ -7570,248 +4057,13 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not session_id: return - def _do(conn): - conn.execute( - "UPDATE sessions SET expiry_finalized = ? WHERE id = ?", - (1 if finalized else 0, session_id), - ) - - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET expiry_finalized = ? WHERE id = ?", + (1 if finalized else 0, session_id), + ) # ── Gateway routing index (replaces sessions.json, #9006 follow-up) ──── - def save_gateway_routing_entry( - self, session_key: str, entry_json: str, *, scope: str = "" - ) -> None: - """Upsert one gateway routing entry (session_key -> SessionEntry JSON). - - The gateway_routing table is the durable replacement for - sessions.json: one row per routing key, holding the full serialized - ``SessionEntry`` so the gateway can rehydrate exactly what it wrote. - - ``scope`` namespaces the index the way separate sessions.json files - did (one per sessions_dir) — callers pass their sessions_dir path so - two stores with different directories never share routing state. - """ - if not session_key or not entry_json: - return - - def _do(conn): - conn.execute( - """INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(scope, session_key) DO UPDATE SET - entry_json = excluded.entry_json, - updated_at = excluded.updated_at""", - (scope, session_key, entry_json, time.time()), - ) - - self._execute_write(_do) - - def replace_gateway_routing_entries( - self, entries: Dict[str, str], *, scope: str = "" - ) -> None: - """Atomically replace the routing index for *scope* with *entries*. - - Mirrors the sessions.json full-rewrite semantics: keys absent from - *entries* are removed (pruned/reset sessions disappear from the - index). Runs as a single write transaction. Other scopes are - untouched. - """ - now = time.time() - - def _do(conn): - conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,)) - if entries: - conn.executemany( - "INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) " - "VALUES (?, ?, ?, ?)", - [(scope, k, v, now) for k, v in entries.items() if k and v], - ) - - self._execute_write(_do) - - def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]: - """Load routing entries for *scope* as {session_key: entry_json}.""" - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?", - (scope,), - ).fetchall() - return {r["session_key"]: r["entry_json"] for r in rows} - - def delete_gateway_routing_entries( - self, session_keys: List[str], *, scope: str = "" - ) -> None: - """Remove routing entries for the given session keys in *scope*.""" - if not session_keys: - return - - def _do(conn): - conn.executemany( - "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", - [(scope, k) for k in session_keys], - ) - - self._execute_write(_do) - - def list_never_active_keyed_sessions( - self, *, older_than_days: float - ) -> List[Dict[str, Any]]: - """Keyed gateway rows that were opened and then never used at all. - - Selects rows that are keyed (``session_key IS NOT NULL``), still open - (``ended_at IS NULL``) and carry no evidence of a single turn: no - messages, no tokens, no tool or API calls, no recorded activity, no - title. Such a row is indistinguishable from "never happened". - - That is exactly the shape of a leaked test fixture (#82770) — and - also of a chat that was routed but never answered. Both are safe to - drop: there is no transcript to lose, and the gateway mints a fresh - session on the next inbound message either way. - - ``bulk prune``/``archive`` cannot reach these rows: their shared - selector is pinned to ``ended_at IS NOT NULL`` so that a live session - is never picked, which permanently excludes every never-closed row. - Hence a separate, narrower selector rather than another filter flag. - - ``pinned`` and ``archived`` rows are excluded — both are explicit - user intent to keep the row around. - """ - cutoff = time.time() - (float(older_than_days) * 86400.0) - with self._read_ctx() as conn: - rows = conn.execute( - """ - SELECT s.id, s.session_key, s.source, s.chat_id, - s.chat_type, s.user_id, s.started_at - FROM sessions s - WHERE s.session_key IS NOT NULL - AND s.ended_at IS NULL - AND s.title IS NULL - AND s.last_activity_at IS NULL - AND COALESCE(s.message_count, 0) = 0 - AND COALESCE(s.tool_call_count, 0) = 0 - AND COALESCE(s.api_call_count, 0) = 0 - AND COALESCE(s.input_tokens, 0) = 0 - AND COALESCE(s.output_tokens, 0) = 0 - AND COALESCE(s.pinned, 0) = 0 - AND COALESCE(s.archived, 0) = 0 - AND s.started_at IS NOT NULL - AND s.started_at < ? - AND NOT EXISTS ( - SELECT 1 FROM messages m WHERE m.session_id = s.id - ) - ORDER BY s.started_at - """, - (cutoff,), - ).fetchall() - return [dict(r) for r in rows] - - def _delete_routing_entries_for_sessions(self, session_ids: Set[str]) -> int: - """Drop ``gateway_routing`` rows pointing at any of *session_ids*. - - Routing entries are keyed by ``(scope, session_key)`` and record their - target session inside ``entry_json``, so there is no way to reach them - by session id in SQL — the match is done in Python over all scopes. - """ - if not session_ids: - return 0 - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT scope, session_key, entry_json FROM gateway_routing" - ).fetchall() - doomed: List[Tuple[str, str]] = [] - for row in rows: - try: - entry = json.loads(row["entry_json"] or "{}") - except Exception: - continue - if isinstance(entry, dict) and entry.get("session_id") in session_ids: - doomed.append((row["scope"], row["session_key"])) - if not doomed: - return 0 - - def _do(conn): - conn.executemany( - "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", - doomed, - ) - - self._execute_write(_do) - return len(doomed) - - def prune_never_active_keyed_sessions( - self, - *, - older_than_days: float, - sessions_dir: Optional[Path] = None, - ) -> Tuple[int, int]: - """Delete never-active keyed rows and the routing entries naming them. - - Returns ``(sessions_deleted, routing_entries_deleted)``. - - The routing entries go first: a stale entry that outlived its target - would leave the gateway resuming a session id that no longer exists. - Deleting the pair is what leaving them both would have amounted to - anyway — the target had no transcript to resume. - - Deletion goes through :meth:`delete_session` rather than a bulk - ``DELETE`` so the delegate cascade, FTS bookkeeping and on-disk - transcript cleanup stay owned by one implementation. - """ - candidates = self.list_never_active_keyed_sessions( - older_than_days=older_than_days - ) - if not candidates: - return (0, 0) - ids = {str(row["id"]) for row in candidates} - routing_deleted = self._delete_routing_entries_for_sessions(ids) - deleted = 0 - for session_id in ids: - if self.delete_session(session_id, sessions_dir=sessions_dir): - deleted += 1 - return (deleted, routing_deleted) - - def list_gateway_sessions( - self, - *, - platform: Optional[str] = None, - active_only: bool = True, - ) -> List[Dict[str, Any]]: - """List gateway sessions (rows with a session_key) from state.db. - - Returns the newest row per session_key — the same shape consumers got - from sessions.json: one live mapping per routing key. ``platform`` - filters on ``source``; ``active_only`` restricts to sessions that - have not ended. - """ - # Full rows carry token/cost totals (MCP listings, /status) — drain - # queued async accounting deltas so consumers see exact counters. - self.flush_token_counts() - query = f""" - SELECT sessions.*, - COALESCE(sp.prompt, sessions.system_prompt) - AS _system_prompt_resolved, - {_sql_session_last_active("sessions")} AS last_active - FROM sessions - LEFT JOIN system_prompts sp - ON sp.hash = sessions.system_prompt_hash - WHERE session_key IS NOT NULL - AND started_at = ( - SELECT MAX(s2.started_at) FROM sessions s2 - WHERE s2.session_key = sessions.session_key - ) - """ - params: list = [] - if platform: - query += " AND LOWER(source) = LOWER(?)" - params.append(platform) - if active_only: - query += " AND ended_at IS NULL" - query += " ORDER BY last_active DESC" - with self._read_ctx() as conn: - rows = conn.execute(query, params).fetchall() - return [self._session_row_dict(r) for r in rows] def find_session_by_origin( self, @@ -7863,132 +4115,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return None return str(rows[0]["id"]) - def find_latest_gateway_session_for_peer( - self, - *, - source: str, - user_id: Optional[str] = None, - session_key: Optional[str] = None, - chat_id: Optional[str] = None, - chat_type: Optional[str] = None, - thread_id: Optional[str] = None, - ) -> Optional[Dict[str, Any]]: - """Find the latest recoverable gateway session for a routing peer. - - ``sessions.json`` is the fast routing index, but it can be missing or - pruned after process-level restart bugs. New gateway sessions persist - the deterministic ``session_key`` on the durable session row so the - mapping can be rebuilt exactly. Rows ended only by older gateway - cleanup's ``agent_close`` bug or a mistaken TUI ``ws_orphan_reap`` - (dashboard viewer disconnect before #60609) are treated as recoverable; - explicit conversation boundaries such as /new, /resume switches, and - compression splits are not. - - Ordering and emptiness (#82616): candidates are ranked by actual - conversation recency (``last_activity_at``, falling back to - ``started_at``) — ``started_at`` alone resurrected days-old zombie - rows over the live conversation. Rows with messages are preferred, - but an empty keyed row is still returned rather than ``None``: - returning ``None`` mints a brand-new session id, which is a worse - outcome than resuming an empty-but-correctly-keyed row (and "empty" - may just mean the transcript lives under a compression child). - - Reset boundaries fence recovery (#68539): an intentional boundary - such as ``session_reset`` (or any explicit non-recoverable - end_reason) must block fallback to an *older* row for the same - peer. Without the fence, the has-messages ranking above could reach - behind a /new reset and silently restore the exact context the user - reset. Each candidate is therefore rejected when a boundary row for - the peer ended *after* the candidate's last activity — if the - conversation's most recent event is an intentional reset, recovery - returns nothing rather than reaching behind it. - """ - if not session_key: - return None - with self._read_ctx() as conn: - row = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) AS _has_messages - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.session_key = ? - AND s.source = ? - AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) - AND NOT EXISTS ( - SELECT 1 FROM sessions b - WHERE b.session_key = s.session_key - AND b.source = s.source - AND b.ended_at IS NOT NULL - AND b.end_reason IN ({_RESET_END_REASONS_SQL}) - AND b.ended_at - > COALESCE(s.last_activity_at, s.started_at) - ) - ORDER BY _has_messages DESC, - COALESCE(s.last_activity_at, s.started_at) DESC - LIMIT 1 - """, - (session_key, source), - ).fetchone() - if row is not None: - return self._session_row_dict(row) - - # Conservative fallback for rows created by current code but with a - # temporarily-missing exact key: still require the complete peer - # tuple so we never cross chats/threads/users. - if chat_id is None or chat_type is None: - return None - # Profile fence (#74285): a Telegram DM's peer tuple is identical - # for every bot (chat_id == user_id, no thread), so a sibling - # profile's row written into this store before the per-profile - # partition (legacy data) would otherwise be adopted here. Every - # profile-tree store has one owner; a row is ours when its - # profile_name is the owner or NULL (legacy rows this store - # minted). Stores outside the tree derive no owner and keep the - # historical unfenced behavior. - owner = self._own_profile_name() - row = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) AS _has_messages - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.source = ? - AND COALESCE(s.user_id, '') = COALESCE(?, '') - AND COALESCE(s.chat_id, '') = COALESCE(?, '') - AND COALESCE(s.chat_type, '') = COALESCE(?, '') - AND COALESCE(s.thread_id, '') = COALESCE(?, '') - AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?) - AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) - AND (COALESCE(s.message_count, 0) > 0 OR EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 - )) - AND NOT EXISTS ( - SELECT 1 FROM sessions b - WHERE b.source = s.source - AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '') - AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '') - AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '') - AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '') - AND b.ended_at IS NOT NULL - AND b.end_reason IN ({_RESET_END_REASONS_SQL}) - AND b.ended_at - > COALESCE(s.last_activity_at, s.started_at) - ) - ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC - LIMIT 1 - """, - (source, user_id, chat_id, chat_type, thread_id, owner, owner, owner), - ).fetchone() - return self._session_row_dict(row) if row else None # ── Orphaned gateway-session repair (#82616) ────────────────────────── # A write-path failure (corrupt FTS, crash between routing publication @@ -8005,219 +4131,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # 15 minutes stays generous without spanning unrelated conversations. _ORPHAN_ADOPTION_MAX_GAP_S = 900.0 - def find_orphaned_gateway_sessions( - self, *, max_gap_s: Optional[float] = None - ) -> List[Dict[str, Any]]: - """Report message-bearing session rows that lost their routing identity. - - A row is a candidate orphan when it has messages but no - ``session_key``. It is only *adoptable* when exactly one keyed - predecessor can be named as the conversation it continues: - - * ``lineage`` — ``parent_session_id`` points at a keyed row of the - same source. That is a recorded fact, so no time window applies. - * ``contiguity`` — exactly one keyed row of the same source (and - compatible ``user_id``) fell quiet within *max_gap_s* of the - orphan's start, and is older than the orphan's own last activity. - - Anything ambiguous is reported with ``adoptable=False`` and a reason - rather than guessed at: mis-adopting would splice one person's - conversation into another person's chat. Branch/delegate/tool rows - are excluded outright — they are unkeyed by design, not by damage. - """ - gap = ( - self._ORPHAN_ADOPTION_MAX_GAP_S - if max_gap_s is None - else float(max_gap_s) - ) - orphan_active = _sql_session_last_active("o") - donor_active = _sql_session_last_active("d") - donor_columns = ( - "d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, " - "d.user_id, d.origin_json, d.display_name, d.end_reason" - ) - records: List[Dict[str, Any]] = [] - - with self._read_ctx() as conn: - orphans = conn.execute( - f""" - SELECT o.id, o.source, o.user_id, o.started_at, - o.parent_session_id, - {orphan_active} AS last_active, - (SELECT COUNT(*) FROM messages m - WHERE m.session_id = o.id) AS message_count - FROM sessions o - WHERE o.session_key IS NULL - AND EXISTS (SELECT 1 FROM messages m - WHERE m.session_id = o.id) - AND COALESCE(o.source, '') != 'tool' - AND json_extract(COALESCE(o.model_config, '{{}}'), - '$._branched_from') IS NULL - AND json_extract(COALESCE(o.model_config, '{{}}'), - '$._delegate_from') IS NULL - ORDER BY o.started_at ASC - """ - ).fetchall() - - for orphan in orphans: - donor = None - evidence = "" - reason = "" - - if orphan["parent_session_id"]: - evidence = "lineage" - donor = conn.execute( - f""" - SELECT {donor_columns} - FROM sessions d - WHERE d.id = ? - AND d.session_key IS NOT NULL - AND COALESCE(d.source, '') = COALESCE(?, '') - """, - (orphan["parent_session_id"], orphan["source"]), - ).fetchone() - if donor is None: - reason = ( - "parent session carries no gateway identity of " - "this source" - ) - else: - evidence = "contiguity" - candidates = conn.execute( - f""" - SELECT {donor_columns}, {donor_active} AS last_active - FROM sessions d - WHERE d.session_key IS NOT NULL - AND d.id != ? - AND COALESCE(d.source, '') = COALESCE(?, '') - AND (COALESCE(d.user_id, '') = '' - OR COALESCE(?, '') = '' - OR d.user_id = ?) - AND {donor_active} BETWEEN ? AND ? - AND {donor_active} < ? - ORDER BY last_active DESC - LIMIT 2 - """, - ( - orphan["id"], - orphan["source"], - orphan["user_id"], - orphan["user_id"], - (orphan["started_at"] or 0) - gap, - (orphan["started_at"] or 0) + gap, - orphan["last_active"], - ), - ).fetchall() - if not candidates: - reason = ( - f"no keyed predecessor fell quiet within {gap:.0f}s " - "of this session's start" - ) - elif len(candidates) > 1: - reason = ( - "ambiguous: more than one keyed predecessor " - "matches this window" - ) - else: - donor = candidates[0] - - records.append( - { - "orphan_id": orphan["id"], - "source": orphan["source"], - "message_count": orphan["message_count"], - "started_at": orphan["started_at"], - "last_active": orphan["last_active"], - "donor_id": donor["id"] if donor else None, - "session_key": donor["session_key"] if donor else None, - "evidence": evidence if donor else "", - "adoptable": donor is not None, - "reason": reason, - } - ) - - # Two unkeyed successors claiming the same predecessor means at most - # one of them continues that chat, and nothing here says which. - contested = { - r["donor_id"] - for r in records - if r["adoptable"] - and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1 - } - for record in records: - if record["donor_id"] in contested: - record["adoptable"] = False - record["reason"] = ( - "ambiguous: more than one unkeyed session claims this " - "predecessor" - ) - return records - - def adopt_orphaned_gateway_session( - self, orphan_id: str, donor_id: str - ) -> bool: - """Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*. - - Re-verifies the pair inside the write transaction, so a concurrent - gateway that healed either row in the meantime turns this into a - no-op instead of a conflicting write. Existing non-NULL columns on - the orphan are preserved. Returns True when the adoption applied. - """ - if not orphan_id or not donor_id or orphan_id == donor_id: - return False - - def _do(conn): - donor = conn.execute( - "SELECT session_key, chat_id, chat_type, thread_id, user_id, " - "origin_json, display_name, source FROM sessions WHERE id = ?", - (donor_id,), - ).fetchone() - orphan = conn.execute( - "SELECT session_key, source FROM sessions WHERE id = ?", - (orphan_id,), - ).fetchone() - if donor is None or orphan is None: - return False - if not donor["session_key"] or orphan["session_key"]: - return False - if (donor["source"] or "") != (orphan["source"] or ""): - return False - - conn.execute( - """UPDATE sessions - SET session_key = ?, - chat_id = COALESCE(chat_id, ?), - chat_type = COALESCE(chat_type, ?), - thread_id = COALESCE(thread_id, ?), - user_id = COALESCE(user_id, ?), - origin_json = COALESCE(origin_json, ?), - display_name = COALESCE(display_name, ?), - parent_session_id = COALESCE(parent_session_id, ?) - WHERE id = ? AND session_key IS NULL""", - ( - donor["session_key"], - donor["chat_id"], - donor["chat_type"], - donor["thread_id"], - donor["user_id"], - donor["origin_json"], - donor["display_name"], - donor_id, - orphan_id, - ), - ) - # Retire the predecessor under a reason recovery does NOT treat - # as resumable — 'agent_close'/'ws_orphan_reap' would keep it in - # the running, and the newly keyed orphan could lose the chat - # again on the next restart. - conn.execute( - "UPDATE sessions SET ended_at = COALESCE(ended_at, ?), " - "end_reason = 'superseded_by_repair' WHERE id = ?", - (time.time(), donor_id), - ) - return True - - return self._execute_write(_do) # Children that carry a ``parent_session_id`` but are NOT compression # continuations: branches, delegate/subagent runs, and tool sessions. @@ -8237,371 +4150,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) " AND COALESCE({alias}source, '') != 'tool'\n" ) - def find_live_compression_child( - self, parent_session_id: str - ) -> Optional[Dict[str, Any]]: - """Return the unique live direct child of a compression-ended session. - - A stale agent may observe that another compression path already rotated - its parent. Recovery is safe only when the durable lineage identifies - exactly one live direct continuation. Multiple children are treated as - ambiguous and fail closed rather than guessing which transcript owns - subsequent messages. - """ - if not parent_session_id: - return None - with self._read_ctx() as conn: - parent = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (parent_session_id,), - ).fetchone() - if ( - parent is None - or parent["ended_at"] is None - or parent["end_reason"] != "compression" - ): - return None - rows = conn.execute( - """ - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved - FROM sessions s - LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash - WHERE s.parent_session_id = ? - AND s.ended_at IS NULL - """ - + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="s.") - + """ - ORDER BY s.started_at ASC - LIMIT 2 - """, - (parent_session_id, parent_session_id, parent_session_id), - ).fetchall() - return self._session_row_dict(rows[0]) if len(rows) == 1 else None - - def reopen_orphaned_compression_session(self, session_id: str) -> bool: - """Reopen a compression parent only when no continuation was published. - - Compression publication is atomic in current builds, but older builds - could leave a closed parent behind after an interrupted handoff. This - recovery is deliberately conservative: an active compression lease or - any canonical child means the lineage is still owned by another path, - so the caller must fail closed instead of reopening the parent. - """ - if not session_id: - return False - - def _do(conn): - parent = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - parent is None - or parent["ended_at"] is None - or parent["end_reason"] != "compression" - ): - return False - - # Treat any direct non-branch/non-delegate/non-tool child as a - # continuation, regardless of its current ended state. Reopening - # in that case could create a second live head for one lineage. - child = conn.execute( - """ - SELECT 1 - FROM sessions - WHERE parent_session_id = ? - """ - + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="") - + """ - LIMIT 1 - """, - (session_id, session_id, session_id), - ).fetchone() - if child is not None: - return False - - # refresh_compression_lock() deliberately lets an owner revive its - # own expired row. Reclaim that row inside this write transaction - # before reopening: refresh-first makes the lease active and aborts - # recovery; recovery-first deletes the holder identity so a later - # refresh cannot resurrect it. - now = time.time() - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if lock_row is not None: - expires_at = lock_row["expires_at"] - if expires_at is None or float(expires_at) >= now: - return False - deleted = conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ? AND expires_at = ?", - (session_id, lock_row["holder"], expires_at), - ) - if deleted.rowcount != 1: - return False - - updated = conn.execute( - "UPDATE sessions SET ended_at = NULL, end_reason = NULL " - "WHERE id = ? AND ended_at IS NOT NULL " - "AND end_reason = 'compression'", - (session_id,), - ) - # rowcount==1 is guaranteed by the parent SELECT at the top of - # this same BEGIN IMMEDIATE transaction. If this is ever edited - # to return False past this point, note that the lease DELETE - # above will still COMMIT (_execute_write commits unless _do - # raises) — raise instead of returning False to roll back. - return updated.rowcount == 1 - - return bool(self._execute_write(_do)) - - def publish_compression_child( - self, - *, - parent_session_id: str, - child_session_id: str, - source: str, - messages: List[Dict[str, Any]], - model: str = None, - model_config: Dict[str, Any] = None, - system_prompt: str = None, - cwd: str = None, - profile_name: str = None, - compression_lock_holder: str = None, - require_compression_lease: bool = True, - require_lease_refresh: bool = False, - lease_ttl_seconds: float = 300.0, - watermark: Optional[int] = None, - watermark_ceiling: Optional[int] = None, - ) -> None: - """Atomically close a parent and publish its durable compression child. - - The parent closure, child row, and compacted handoff become visible in - one transaction. Readers can therefore observe either the live parent or - a complete child, never an ended parent with a missing/empty child. - - Concurrent-append safety (#75316): when *watermark* is provided (the - parent's :meth:`get_active_message_watermark` captured at compression - start), parent rows that arrived during the slow summary call - (``id > watermark``) are cloned into the child AFTER the handoff — - same pure-SQL column clone as :meth:`archive_and_compact`, with the - session id rewritten — so a mid-compression append survives rotation - instead of stranding in the closed parent. - - *watermark_ceiling* bounds the clone from above: the rotation path - flushes its OWN un-persisted input transcript to the parent right - before publishing (#47202), and those rows are already represented in - the compacted handoff — cloning them would duplicate the transcript. - The caller captures ``MAX(id)`` immediately BEFORE that flush; only - rows in ``(watermark, watermark_ceiling]`` are foreign concurrent - tail. ``None`` = unbounded (no internal flush happened). - - When *require_lease_refresh* is True and *compression_lock_holder* is - set, the lease is refreshed inside the same transaction before the - expiry check. This gives a refresher - that stopped due to transient DB failures one final chance to extend - the lease, preventing wasted compression work. The refresh uses the - same ``conn`` as the publication, so there is no TOCTOU window. - """ - def _do(conn): - if require_lease_refresh and compression_lock_holder: - conn.execute( - "UPDATE compression_locks SET expires_at = ? " - "WHERE session_id = ? AND holder = ?", - (time.time() + lease_ttl_seconds, parent_session_id, - compression_lock_holder), - ) - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks WHERE session_id = ?", - (parent_session_id,), - ).fetchone() - if require_compression_lease and ( - lock_row is None - or not compression_lock_holder - or lock_row["holder"] != compression_lock_holder - or float(lock_row["expires_at"]) <= time.time() - ): - raise CompressionSessionBusyError( - f"Compression lease lost before publication: {parent_session_id}" - ) - parent = conn.execute( - """SELECT ended_at, end_reason, cwd, git_branch, git_repo_root, - user_id, session_key, chat_id, chat_type, - thread_id, display_name, origin_json, profile_name - FROM sessions WHERE id = ?""", - (parent_session_id,), - ).fetchone() - if parent is None: - raise RuntimeError(f"Compression parent not found: {parent_session_id}") - if parent["ended_at"] is not None: - # A parent stamped ended by AUTOMATIC cleanup (tui_shutdown, - # ws_disconnect, orphan reap, idle/LRU evict) while a live - # agent is publishing its rotation is stale by construction — - # this writer holds the compression lease and is actively - # continuing the conversation the stamp claims is over. - # Left in place it wedges rotation forever: every attempt - # aborts here, nothing clears the stamp, and each attempt's - # pre-publish flush re-grows the parent until the provider - # rejects the request (#88197: 303 unique messages → 2,611 - # rows → HTTP 400). Clear it in this same transaction and - # proceed; the closure UPDATE below re-stamps the parent with - # its true boundary (end_reason='compression'). Deliberate - # boundaries (compression, session_reset, explicit close) - # still fail closed — those mean another path owns lineage. - if is_automatic_end_reason(parent["end_reason"]): - conn.execute( - "UPDATE sessions SET ended_at = NULL, end_reason = NULL " - "WHERE id = ?", - (parent_session_id,), - ) - else: - raise RuntimeError( - f"Compression parent already ended: {parent_session_id}" - ) - if not messages: - raise RuntimeError("Compression child handoff must not be empty") - system_prompt_hash = self._store_system_prompt(conn, system_prompt) - - conn.execute( - """INSERT INTO sessions ( - id, source, model, model_config, system_prompt, - system_prompt_hash, - parent_session_id, cwd, git_branch, git_repo_root, - profile_name, user_id, session_key, chat_id, chat_type, - thread_id, display_name, origin_json, started_at - ) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - child_session_id, - source, - model, - json.dumps(model_config) if model_config else None, - system_prompt_hash, - parent_session_id, - cwd or parent["cwd"], - parent["git_branch"], - parent["git_repo_root"], - # Same inheritance contract as _insert_session_row's - # compression-fork backfill (#59527 / cross-profile jump - # fix): the child stays on the parent's profile and keeps - # the gateway routing/origin columns so peer recovery - # still works after a crash at the boundary. When neither - # names an owner (legacy NULL parent), stamp this store's - # own profile so the rotated child doesn't extend the - # unowned lineage (#99222). - profile_name - or parent["profile_name"] - or self._own_profile_name(), - parent["user_id"], - parent["session_key"], - parent["chat_id"], - parent["chat_type"], - parent["thread_id"], - parent["display_name"], - parent["origin_json"], - time.time(), - ), - ) - total_messages, total_tool_calls = self._insert_message_rows( - conn, child_session_id, messages - ) - if watermark is not None: - # Clone the parent's concurrent tail (rows landed after the - # watermark, at or below the ceiling — see docstring) into the - # child, after the handoff. Column-exact except id/session_id; - # originals stay in the (closed) parent for lineage recovery. - _ceiling_clause = "" - _params: list = [parent_session_id, int(watermark)] - if watermark_ceiling is not None: - _ceiling_clause = " AND id <= ?" - _params.append(int(watermark_ceiling)) - tail_rows = conn.execute( - "SELECT id, tool_calls FROM messages " - "WHERE session_id = ? AND active = 1 AND id > ?" - f"{_ceiling_clause} ORDER BY id", - _params, - ).fetchall() - if tail_rows: - tail_ids = [int(r["id"]) for r in tail_rows] - placeholders = ",".join("?" for _ in tail_ids) - clone_cols = [ - c for c in self._message_column_names(conn) - if c not in ("id", "session_id", "active", "compacted") - ] - col_list = ", ".join(clone_cols) - conn.execute( - f"INSERT INTO messages ({col_list}, session_id, active, compacted) " - f"SELECT {col_list}, ?, 1, 0 FROM messages " - f"WHERE id IN ({placeholders}) ORDER BY id", - [child_session_id, *tail_ids], - ) - total_messages += len(tail_ids) - for r in tail_rows: - raw = r["tool_calls"] - if raw: - try: - parsed = json.loads(raw) if isinstance(raw, str) else raw - total_tool_calls += len(parsed) if isinstance(parsed, list) else 0 - except (TypeError, ValueError): - pass - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (total_messages, total_tool_calls, child_session_id), - ) - updated = conn.execute( - "UPDATE sessions SET ended_at = ?, end_reason = 'compression' " - "WHERE id = ? AND ended_at IS NULL", - (time.time(), parent_session_id), - ) - if updated.rowcount != 1: - raise RuntimeError( - f"Compression parent changed during publication: {parent_session_id}" - ) - - self._execute_write(_do) - - def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None: - """Advance this peer's conversation generation past a boundary. - - Called inside the transaction that writes the boundary, so the - generation and the ``end_reason`` that caused it commit together. - - Only ``_RESET_END_REASONS`` count: ``compression`` continues one - conversation, and an accidental close is not a replacement. Rows with - no ``session_key`` have no routing peer to advance. - - The counter deliberately does NOT read the session rows. An aggregate - over them (COUNT/MAX of boundaries) can return a pair it already - emitted once ``delete_session()`` or bulk pruning removes an ended row, - which would hand a new conversation a retired affinity identity. This - value only ever increments, so a generation is never reused for a peer - even if every row behind it is gone. - """ - if end_reason not in _RESET_END_REASONS: - return - row = conn.execute( - "SELECT source, session_key FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return - source = str(row["source"] or "").strip() - session_key = str(row["session_key"] or "").strip() - if not source or not session_key: - return - conn.execute( - """ - INSERT INTO conversation_generations (source, session_key, generation) - VALUES (?, ?, 1) - ON CONFLICT(source, session_key) DO UPDATE - SET generation = conversation_generations.generation + 1 - """, - (source, session_key), - ) def end_session(self, session_id: str, end_reason: str) -> None: """Mark a session as ended. @@ -8753,7 +4301,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if current is None: return None - current_cwd = current["cwd"] if isinstance(current, sqlite3.Row) else current[0] + current_cwd = current[0] sets = [ "cwd = ?", "git_metadata_generation = COALESCE(git_metadata_generation, 0) + 1", @@ -8778,7 +4326,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return None - value = row["git_metadata_generation"] if isinstance(row, sqlite3.Row) else row[0] + value = row[0] return int(value) return self._execute_write(_do) @@ -8816,16 +4364,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) params.append(repo_root) params.extend((session_id, cwd, generation)) - def _do(conn): - cursor = conn.execute( - f"UPDATE sessions SET {', '.join(sets)} " - "WHERE id = ? AND cwd = ? " - "AND git_metadata_generation = ?", - params, - ) - return cursor.rowcount == 1 - - return bool(self._execute_write(_do)) + return self._write_rowcount( + f"UPDATE sessions SET {', '.join(sets)} " + "WHERE id = ? AND cwd = ? " + "AND git_metadata_generation = ?", + params, + ) == 1 def backfill_repo_roots(self, cwd_to_root: Dict[str, str]) -> None: """Persist resolved git repo roots for cwds that don't have one yet. @@ -8848,344 +4392,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._execute_write(_do) - def record_compression_failure_cooldown( - self, - session_id: str, - cooldown_until: float, - error: Optional[str] = None, - ) -> None: - """Persist the active compression-failure cooldown for a session.""" - if not session_id: - return - - def _do(conn): - # Merge-max with any longer live deadline so a later shorter - # write cannot reopen the thrash window (#96775). The error - # column always takes the latest diagnostic. - conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = CASE " - "WHEN compression_failure_cooldown_until IS NOT NULL " - " AND compression_failure_cooldown_until > ? " - "THEN compression_failure_cooldown_until ELSE ? END, " - "compression_failure_error = ? WHERE id = ?", - (cooldown_until, cooldown_until, error, session_id), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "record_compression_failure_cooldown(%s) failed: %s", - session_id, exc, - ) - - def get_compression_failure_cooldown( - self, - session_id: str, - ) -> Optional[Dict[str, Any]]: - """Return the active compression-failure cooldown for ``session_id``.""" - if not session_id: - return None - now = time.time() - with self._read_ctx() as conn: - row = conn.execute( - "SELECT compression_failure_cooldown_until, compression_failure_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return None - cooldown_until = ( - row["compression_failure_cooldown_until"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - if cooldown_until is None: - return None - cooldown_until = float(cooldown_until) - if cooldown_until <= now: - return None - error = ( - row["compression_failure_error"] - if isinstance(row, sqlite3.Row) - else row[1] - ) - return { - "cooldown_until": cooldown_until, - "remaining_seconds": cooldown_until - now, - "error": error, - } - - def get_compression_failure_cooldown_row( - self, - session_id: str, - ) -> Dict[str, Any]: - """Return the exact stored cooldown columns without expiry filtering. - - Compression cancellation uses this under its session lease so rollback - can preserve an expired row, a partially-null row, or an absent session - exactly instead of converting those states through the active-cooldown - API. - """ - if not session_id: - return {"session_exists": False, "cooldown_until": None, "error": None} - with self._read_ctx() as conn: - row = conn.execute( - "SELECT compression_failure_cooldown_until, compression_failure_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return {"session_exists": False, "cooldown_until": None, "error": None} - cooldown_until = ( - row["compression_failure_cooldown_until"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - error = ( - row["compression_failure_error"] - if isinstance(row, sqlite3.Row) - else row[1] - ) - return { - "session_exists": True, - "cooldown_until": ( - float(cooldown_until) if cooldown_until is not None else None - ), - "error": error, - } - - def restore_compression_failure_cooldown_row( - self, - session_id: str, - snapshot: Dict[str, Any], - ) -> None: - """Restore and verify an exact cooldown-row snapshot. - - Unlike the ordinary record/clear helpers, this transactional rollback - API deliberately propagates write and verification failures. A caller - must not report cancellation as mutation-free when compensation failed. - """ - expected_exists = bool(snapshot.get("session_exists", False)) - if not expected_exists: - actual = self.get_compression_failure_cooldown_row(session_id) - if actual.get("session_exists", False): - raise RuntimeError( - "cannot restore absent compression cooldown row: session now exists" - ) - return - - deadline = snapshot.get("cooldown_until") - error = snapshot.get("error") - - def _do(conn): - cursor = conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = ?, " - "compression_failure_error = ? WHERE id = ?", - (deadline, error, session_id), - ) - if cursor.rowcount != 1: - raise RuntimeError( - f"compression cooldown rollback session missing: {session_id}" - ) - - self._execute_write(_do) - actual = self.get_compression_failure_cooldown_row(session_id) - expected = { - "session_exists": True, - "cooldown_until": float(deadline) if deadline is not None else None, - "error": error, - } - if actual != expected: - raise RuntimeError( - f"compression cooldown rollback verification failed: " - f"expected={expected!r}, actual={actual!r}" - ) - - def clear_compression_failure_cooldown(self, session_id: str) -> None: - """Clear any persisted compression-failure cooldown for a session.""" - if not session_id: - return - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_failure_cooldown_until = NULL, " - "compression_failure_error = NULL WHERE id = ?", - (session_id,), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "clear_compression_failure_cooldown(%s) failed: %s", - session_id, exc, - ) - - def get_compression_fallback_streak(self, session_id: str) -> int: - """Return the persisted deterministic-fallback streak.""" - if not session_id: - return 0 - with self._read_ctx() as conn: - if conn is None: - return 0 - row = conn.execute( - "SELECT compression_fallback_streak FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0 - value = ( - row["compression_fallback_streak"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0, int(value or 0)) - except (TypeError, ValueError): - return 0 - - def set_compression_fallback_streak(self, session_id: str, streak: int) -> None: - """Persist the deterministic-fallback streak for one session.""" - if not session_id: - return - normalized = max(0, int(streak)) - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?", - (normalized, session_id), - ) - - self._execute_write(_do) - - def increment_hygiene_failure_streak(self, session_key: str) -> int: - """Atomically increment the session-hygiene failure streak for one chat.""" - if not session_key: - return 1 - result = [] - - def _do(conn): - conn.execute( - """INSERT INTO gateway_hygiene_state (session_key, failure_streak) - VALUES (?, 1) - ON CONFLICT(session_key) DO UPDATE SET - failure_streak = gateway_hygiene_state.failure_streak + 1""", - (session_key,), - ) - row = conn.execute( - "SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?", - (session_key,), - ).fetchone() - result.append(int(row[0])) - - self._execute_write(_do) - return result[0] - - def reset_hygiene_failure_streak(self, session_key: str) -> None: - """Clear the persisted session-hygiene failure streak for one chat.""" - if not session_key: - return - - def _do(conn): - conn.execute( - "DELETE FROM gateway_hygiene_state WHERE session_key = ?", - (session_key,), - ) - - self._execute_write(_do) - - def get_compression_ineffective_count(self, session_id: str) -> int: - """Return the persisted ineffective-compaction strike count. - - Mirrors ``get_compression_fallback_streak``: this is the durable half - of the anti-thrash guard (``_ineffective_compression_count`` on the - built-in compressor), persisted so that a fresh compressor bound to a - resumed session inherits an armed/tripped guard instead of starting - from zero across process restarts (#54923). - """ - if not session_id: - return 0 - with self._read_ctx() as conn: - if conn is None: - return 0 - row = conn.execute( - "SELECT compression_ineffective_count FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0 - value = ( - row["compression_ineffective_count"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0, int(value or 0)) - except (TypeError, ValueError): - return 0 - - def set_compression_ineffective_count(self, session_id: str, count: int) -> None: - """Persist the ineffective-compaction strike count for one session.""" - if not session_id: - return - normalized = max(0, int(count)) - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?", - (normalized, session_id), - ) - - self._execute_write(_do) - - def get_compression_recovery_deadline(self, session_id: str) -> float: - """Return the persisted anti-thrash recovery deadline (wall-clock epoch). - - ``0.0`` means "not armed". The deadline is the durable half of the - #14694 recovery clock: the gateway rebuilds the compressor on every - turn / cache eviction, so a process-local deadline restarted the - wait on each rebuild and a tripped session never earned its probe - (#100185). - """ - if not session_id: - return 0.0 - with self._read_ctx() as conn: - if conn is None: - return 0.0 - row = conn.execute( - "SELECT compression_recovery_deadline FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if row is None: - return 0.0 - value = ( - row["compression_recovery_deadline"] - if isinstance(row, sqlite3.Row) - else row[0] - ) - try: - return max(0.0, float(value or 0.0)) - except (TypeError, ValueError): - return 0.0 - - def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None: - """Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it.""" - if not session_id: - return - try: - normalized = max(0.0, float(deadline or 0.0)) - except (TypeError, ValueError): - normalized = 0.0 - stored = normalized if normalized > 0.0 else None - - def _do(conn): - conn.execute( - "UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?", - (stored, session_id), - ) - - self._execute_write(_do) # ────────────────────────────────────────────────────────────────────── # Compression locks @@ -9208,398 +4414,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # the compress() call plus the rotation. ``holder`` identifies the # current owner (pid:tid:nonce) for diagnostics; the lock is recovered # via ``expires_at`` if the holder process crashed without releasing. - def refresh_compression_lock( - self, - session_id: str, - holder: str, - ttl_seconds: float = 300.0, - ) -> bool: - """Extend the compression lock lease if ``holder`` still owns it. - Ownership is decided by the ``holder`` column alone, deliberately NOT - by ``expires_at``: a live owner whose refresher thread was starved - (GC pause, loaded CI runner, a slow write escaping ``_execute_write``'s - retry budget) past its own TTL must be able to revive its still-unclaimed - row on the next tick. Requiring ``expires_at >= now`` here made such a - stall permanent — every later refresh matched 0 rows, so the owner kept - compressing and rotating with no lease at all, which is exactly the - unprotected window a competing path can fork the session lineage in. - - This does not resurrect a lock somebody else already took: SQLite - serialises writes, so a reclaim (DELETE-expired + INSERT-or-IGNORE in - :meth:`try_acquire_compression_lock`) and this UPDATE never interleave. - Reclaim-first replaces ``holder``, so this UPDATE matches nothing and - returns False; refresh-first pushes ``expires_at`` into the future, so - the reclaimer's DELETE-expired matches nothing and its acquire fails. - """ - if not session_id or not holder: - return False - now = time.time() - expires_at = now + ttl_seconds - - def _do(conn): - cur = conn.execute( - "UPDATE compression_locks SET expires_at = ? " - "WHERE session_id = ? AND holder = ?", - (expires_at, session_id, holder), - ) - return cur.rowcount > 0 - - try: - return bool(self._execute_write(_do)) - except sqlite3.Error as exc: - logger.warning( - "refresh_compression_lock(%s) failed: %s", - session_id, exc, - ) - return False - - def try_acquire_compression_lock( - self, - session_id: str, - holder: str, - ttl_seconds: float = 300.0, - ) -> bool: - """Try to atomically acquire the compression lock for ``session_id``. - - Returns ``True`` on success (caller now owns the lock and must - release via :meth:`release_compression_lock`). Returns ``False`` - if another holder already owns a non-expired lock — the caller - MUST NOT proceed with compression in that case (its rotation would - race against the holder's, splitting the session lineage). - - Expired locks (``expires_at < now``) are reclaimed transparently. - Structured holders whose local ``pid=`` no longer exists are reclaimed - immediately, so a gateway killed during compression does not stall the - replacement process for the full lease TTL. - - Implementation: single-transaction DELETE-expired + INSERT-or-IGNORE, - followed by a SELECT to confirm we got the row. SQLite serialises - writes, so the whole sequence is atomic against other writers. - """ - if not session_id: - return False - now = time.time() - expires_at = now + ttl_seconds - - def _do(conn): - reclaimed_holder = None - row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if row is not None: - current_holder = ( - row["holder"] if isinstance(row, sqlite3.Row) else row[0] - ) - current_expires_at = ( - row["expires_at"] if isinstance(row, sqlite3.Row) else row[1] - ) - if ( - current_expires_at < now - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, current_holder), - ) - reclaimed_holder = current_holder - # Then: try to insert. INSERT OR IGNORE returns no rowcount - # difference — verify ownership via SELECT. - conn.execute( - "INSERT OR IGNORE INTO compression_locks " - "(session_id, holder, acquired_at, expires_at) " - "VALUES (?, ?, ?, ?)", - (session_id, holder, now, expires_at), - ) - row = conn.execute( - "SELECT holder FROM compression_locks WHERE session_id = ?", - (session_id,), - ).fetchone() - acquired = row is not None and ( - row["holder"] if isinstance(row, sqlite3.Row) else row[0] - ) == holder - return acquired, reclaimed_holder - - try: - acquired, reclaimed_holder = self._execute_write(_do) - if reclaimed_holder: - logger.warning( - "Reclaimed stale compression lock for session=%s " - "(holder=%s)", - session_id, - reclaimed_holder, - ) - return bool(acquired) - except sqlite3.Error as exc: - logger.warning( - "try_acquire_compression_lock(%s) failed: %s", - session_id, exc, - ) - # Fail open: returning False makes the caller skip compression, - # which is the safe behaviour when the lock subsystem is broken. - return False - - def release_compression_lock(self, session_id: str, holder: str) -> None: - """Release the compression lock for ``session_id`` iff we own it. - - Idempotent: no-op when the lock has already expired and been - reclaimed by a different holder, or when no lock exists. The - ``holder`` check prevents a late-returning compressor from - clobbering a fresh lock held by someone else. - """ - if not session_id: - return - - def _do(conn): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, holder), - ) - - try: - self._execute_write(_do) - except sqlite3.Error as exc: - logger.warning( - "release_compression_lock(%s) failed: %s", - session_id, exc, - ) - - def _session_turn_lease_key_on_conn(self, conn, session_id: str) -> str: - """Walk compression parents on ``conn`` to the conversation lease key. - - Must run on the same connection as the lease INSERT/UPDATE/DELETE. - A prior ``get_session`` failure must not compute a child id that the - later write then persists: refresh would walk to the parent and - fail-close. Markers bind to ``parent_session_id`` (same contract as - ``_NON_CONTINUATION_CHILD_FILTER_SQL``). Lock errors propagate so - ``_execute_write`` / ``acquire_session_turn_lease`` can retry. - """ - if not session_id: - return session_id - - def _row(sid: str): - row = conn.execute( - "SELECT id, parent_session_id, source, model_config, end_reason " - "FROM sessions WHERE id = ?", - (sid,), - ).fetchone() - return dict(row) if row else None - - current = _row(session_id) - seen = {session_id} - while current: - parent_id = current.get("parent_session_id") - if ( - not parent_id - or parent_id in seen - or self._is_explicit_fork_child_row(current) - ): - break - parent = _row(parent_id) - if not parent or parent.get("end_reason") != "compression": - break - seen.add(parent_id) - current = parent - return str(current.get("id") or session_id) if current else session_id - - def _session_turn_lease_key(self, session_id: str) -> str: - """Return the stable serialization key for every compression segment. - - Acquire/refresh/release resolve this inside their write transaction. - This helper is for tests and diagnostics; it does not swallow lock - errors (a swallowed walk plus a later successful write was the - fail-open that replayed the post-rotation refresh miss). - """ - if not session_id: - return session_id - with self._read_ctx() as conn: - return self._session_turn_lease_key_on_conn(conn, session_id) - - def try_acquire_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - patience_s: Optional[float] = None, - ) -> bool: - """Atomically acquire the cross-process turn lease for a conversation. - - Compression rotates a session into child segments, so the durable key - is the lineage root rather than the current segment id. The walk and - INSERT share one write transaction. Expired leases and leases whose - structured local holder PID is known dead are reclaimed in that same - transaction. - """ - if not session_id or not holder: - return False - now = time.time() - expires_at = now + max(0.1, float(ttl_seconds)) - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - row = conn.execute( - "SELECT holder, expires_at FROM session_turn_leases " - "WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - if row is not None: - current_holder = row["holder"] - if ( - float(row["expires_at"]) <= now - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, current_holder), - ) - conn.execute( - "INSERT OR IGNORE INTO session_turn_leases " - "(conversation_id, holder, acquired_at, expires_at) " - "VALUES (?, ?, ?, ?)", - (conversation_id, holder, now, expires_at), - ) - owner = conn.execute( - "SELECT holder FROM session_turn_leases WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - return owner is not None and owner["holder"] == holder - - return bool(self._execute_write(_do, patience_s=patience_s)) - - def acquire_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - wait_seconds: float = 1800.0, - poll_interval_seconds: float = 1.0, - on_wait=None, - wait_notice_interval_seconds: float = 15.0, - should_abort=None, - acquire_patience_s: float = 0.5, - ) -> bool: - """Wait for a cross-process turn lease without holding a SQLite lock. - - ``on_wait(elapsed_seconds)`` is best-effort: invoked when the first - attempt fails (elapsed ~0) and again about every - ``wait_notice_interval_seconds`` while still waiting, so UIs can show - that another process holds the conversation. - - When ``should_abort()`` returns True (for example the agent received - ``/stop`` while waiting), acquisition stops immediately and returns - False without consuming the full ``wait_seconds`` budget. - """ - deadline = time.monotonic() + max(0.0, float(wait_seconds)) - wait_started = None - last_notice_at = None - notice_every = max(0.0, float(wait_notice_interval_seconds)) - while True: - if should_abort is not None: - try: - if should_abort(): - return False - except Exception: - logger.debug( - "session turn lease should_abort callback failed", - exc_info=True, - ) - try: - if self.try_acquire_session_turn_lease( - session_id, - holder, - ttl_seconds=ttl_seconds, - patience_s=acquire_patience_s, - ): - return True - except sqlite3.Error as exc: - # Long holder transactions (compression publish, large - # flushes) can exhaust a single write-patience budget. - # Keep polling until wait_seconds or should_abort. - if classify_persistence_error(exc) != "locked": - raise - now = time.monotonic() - remaining = deadline - now - if remaining <= 0: - return False - if wait_started is None: - wait_started = now - if on_wait is not None and ( - last_notice_at is None - or notice_every == 0.0 - or (now - last_notice_at) >= notice_every - ): - try: - on_wait(max(0.0, now - wait_started)) - except Exception: - logger.debug( - "session turn lease on_wait callback failed", - exc_info=True, - ) - last_notice_at = now - time.sleep(min(max(0.01, float(poll_interval_seconds)), remaining)) - - def refresh_session_turn_lease( - self, - session_id: str, - holder: str, - *, - ttl_seconds: float = 300.0, - ) -> bool: - """Extend a turn lease only while ``holder`` still owns it.""" - if not session_id or not holder: - return False - expires_at = time.time() + max(0.1, float(ttl_seconds)) - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - cursor = conn.execute( - "UPDATE session_turn_leases SET expires_at = ? " - "WHERE conversation_id = ? AND holder = ?", - (expires_at, conversation_id, holder), - ) - return cursor.rowcount > 0 - - return bool(self._execute_write(_do)) - - def release_session_turn_lease(self, session_id: str, holder: str) -> None: - """Release a turn lease iff ``holder`` still owns it; idempotent.""" - if not session_id or not holder: - return - - def _do(conn): - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, holder), - ) - - self._execute_write(_do) - - def get_compression_lock_holder(self, session_id: str) -> Optional[str]: - """Return the current (non-expired) holder for ``session_id``, or None. - - Diagnostic helper — not used by the locking protocol itself. - """ - if not session_id: - return None - now = time.time() - with self._read_ctx() as conn: - row = conn.execute( - "SELECT holder FROM compression_locks " - "WHERE session_id = ? AND expires_at >= ?", - (session_id, now), - ).fetchone() - if row is None: - return None - return row["holder"] if isinstance(row, sqlite3.Row) else row[0] def touch_session_activity( self, @@ -9631,22 +4446,20 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) desc = bound_activity_description(description) prov = normalize_activity_provenance(provenance).value - def _do(conn): - conn.execute( - "UPDATE sessions SET " - "last_activity_at = ?, " - "last_activity_description = ?, " - "last_activity_provenance = ? " - "WHERE id = ? AND (last_activity_at IS NULL OR last_activity_at < ?)", - (when, desc, prov, session_id, when), - ) - # Observation-only write: never let it ride the full routine # write-patience budget (#76354 review S1). Under contention a # heartbeat that waits ~20s would delay the response-critical path # it is merely observing; give up after a sub-second budget instead # (the next due window retries naturally). - self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S) + self._write_sql( + "UPDATE sessions SET " + "last_activity_at = ?, " + "last_activity_description = ?, " + "last_activity_provenance = ? " + "WHERE id = ? AND (last_activity_at IS NULL OR last_activity_at < ?)", + (when, desc, prov, session_id, when), + patience_s=self._ACTIVITY_WRITE_PATIENCE_S, + ) def clear_session_activity_labels(self, session_id: str) -> None: """Clear mid-turn activity labels after a turn ends. @@ -9670,32 +4483,29 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # No-op fast path: skip the transaction when there is nothing to # clear. Read-only, no write lock. try: - with self._read_ctx() as conn: - row = conn.execute( - "SELECT last_activity_description, last_activity_provenance " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() + row = self._read_one( + "SELECT last_activity_description, last_activity_provenance " + "FROM sessions WHERE id = ?", + (session_id,), + ) except sqlite3.Error: row = None if row is not None: - desc = row[0] if not isinstance(row, sqlite3.Row) else row["last_activity_description"] - prov = row[1] if not isinstance(row, sqlite3.Row) else row["last_activity_provenance"] + desc = row[0] + prov = row[1] if not desc and ( not prov or prov == ActivityProvenance.UNKNOWN.value ): return - def _do(conn): - conn.execute( - "UPDATE sessions SET " - "last_activity_description = ?, " - "last_activity_provenance = ? " - "WHERE id = ?", - ("", ActivityProvenance.UNKNOWN.value, session_id), - ) - - self._execute_write(_do, patience_s=self._ACTIVITY_WRITE_PATIENCE_S) + self._write_sql( + "UPDATE sessions SET " + "last_activity_description = ?, " + "last_activity_provenance = ? " + "WHERE id = ?", + ("", ActivityProvenance.UNKNOWN.value, session_id), + patience_s=self._ACTIVITY_WRITE_PATIENCE_S, + ) def get_session_activity(self, session_id: str) -> Optional[Dict[str, Any]]: """Return the durable activity snapshot for *session_id*, or None.""" @@ -9727,12 +4537,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # Barrier against queued token deltas — see update_session_model. self.flush_token_counts() - def _do(conn): - conn.execute( - "UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?", - (model_config_json, model, session_id), - ) - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET model_config = ?, model = COALESCE(?, model) WHERE id = ?", + (model_config_json, model, session_id), + ) def update_system_prompt( self, session_id: str, system_prompt: Optional[str] @@ -9760,12 +4568,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ payload = json.dumps(list(tool_names)) if tool_names is not None else None - def _do(conn): - conn.execute( - "UPDATE sessions SET tool_names = ? WHERE id = ?", - (payload, session_id), - ) - self._execute_write(_do) + self._write_sql( + "UPDATE sessions SET tool_names = ? WHERE id = ?", + (payload, session_id), + ) def update_session_model( self, session_id: str, model: str, provider: Optional[str] = None @@ -9848,7 +4654,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if on_missing == "raise": raise ValueError(f"Session not found: {session_id}") return _MODEL_CONFIG_ROW_MISSING - raw = row["model_config"] if isinstance(row, sqlite3.Row) else row[0] + raw = row[0] config: Dict[str, Any] = {} if isinstance(raw, str) and raw.strip(): try: @@ -9997,93 +4803,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return False return bool(raw.get("yolo_mode")) - @staticmethod - def session_gateway_runtime(session_meta: Optional[Dict[str, Any]]) -> Dict[str, Any]: - """Read the persisted runtime route off a session row dict. - - Accepts the dict returned by ``get_session`` (``model_config`` is a - JSON string) or an already-parsed dict. Prefers the nested - ``gateway_runtime`` key (written by the gateway's - ``_sync_session_model_from_agent`` and the CLI ``/model`` persist), - falling back to the top-level ``provider``/``base_url``/``api_mode`` - keys the TUI gateway's ``_runtime_model_config`` writes. As a last - resort, falls back to the ``billing_provider`` column (written on - every session's first accounted API call) so sessions that never ran - ``/model`` still restore the provider that actually served them. - Returns an empty dict on any parse failure — resume falls back to - ambient config resolution. - """ - raw = (session_meta or {}).get("model_config") - if isinstance(raw, str): - try: - raw = json.loads(raw) - except Exception: - raw = {} - if not isinstance(raw, dict): - raw = {} - runtime = raw.get("gateway_runtime") - if isinstance(runtime, dict) and runtime.get("provider"): - # Filter None values: the persist path writes or-None to trigger - # deletion in the top-level merge, but gateway_runtime is replaced - # as a whole dict (not deep-merged), so None values survive here. - return {k: v for k, v in runtime.items() if v is not None} - top_level = { - key: raw.get(key) - for key in ("provider", "base_url", "api_mode") - if raw.get(key) - } - if top_level: - return top_level - # Last resort: billing_provider column. Written via COALESCE on every - # session's first accounted API call — the only durable record for - # sessions that never ran /model. Mirrors the TUI gateway's - # _stored_session_runtime_overrides fallback. Bare billing buckets - # ("auto"/"custom") are not routable identities — filter them out so - # resume falls back to the ambient config default instead. - billing_provider = str( - (session_meta or {}).get("billing_provider") or "" - ).strip() - if ( - billing_provider - and billing_provider.lower() not in _BARE_BILLING_PROVIDERS - ): - return {"provider": billing_provider} - return {k: v for k, v in (runtime or {}).items() if v is not None} if isinstance(runtime, dict) else {} - - def update_session_billing_route( - self, - session_id: str, - *, - provider: str, - base_url: str, - billing_mode: Optional[str] = None, - ) -> None: - """Unconditionally update the billing provider/base_url for a session. - - Unlike ``update_token_counts`` which uses ``COALESCE(billing_provider, ?)`` - (only filling in NULL), this unconditionally sets the billing fields so - that the dashboard reflects the user's latest /model switch. - - Also nulls ``system_prompt`` so the cached snapshot (which embeds a - stale ``Model:`` / ``Provider:`` header) is rebuilt — matching the - behavior of ``update_session_model`` (see #48173, #48248). - """ - # Barrier against queued token deltas — see update_session_model. - self.flush_token_counts() - - def _do(conn): - conn.execute( - """UPDATE sessions SET - billing_provider = ?, - billing_base_url = ?, - billing_mode = COALESCE(?, billing_mode), - system_prompt = NULL, - system_prompt_hash = NULL - WHERE id = ?""", - (provider, base_url, billing_mode, session_id), - ) - self._delete_unreferenced_system_prompts(conn) - self._execute_write(_do) # ── Async token accounting ── # update_token_counts() runs a sessions UPDATE (plus a per-model usage @@ -10112,524 +4831,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) "billing_provider", "billing_base_url", "billing_mode", ) - def queue_token_counts(self, session_id: str, **kwargs) -> None: - """Enqueue a token/cost delta for the background writer. - - Accepts the same keyword arguments as :meth:`update_token_counts` - and applies them asynchronously with identical semantics. Cheap - (append + notify) — safe to call on the turn thread after every - API call. After close() has stopped the writer, falls back to the - synchronous path and may raise like :meth:`update_token_counts`. - """ - with self._token_queue_cond: - thread = self._token_writer_thread - writer_stopped = self._token_writer_stop and ( - thread is None or not thread.is_alive() - ) - if not writer_stopped: - self._token_queue.append((session_id, kwargs)) - if thread is None or not thread.is_alive(): - # Daemon so process exit never hangs on accounting; the - # atexit hook drains anything still queued at interpreter - # shutdown (registered once per instance, on first use). - # ``not is_alive()`` (rather than ``is None`` only) - # respawns the writer if it ever died from an unexpected - # escape — otherwise a dead thread object would block - # respawn forever and deltas would pile up on the deque - # until a reader's flush drained them synchronously. - thread = threading.Thread( - target=self._token_writer_loop, - name="session-db-token-writer", - daemon=True, - ) - self._token_writer_thread = thread - thread.start() - if self._token_atexit_hook is None: - self_ref = weakref.ref(self) - - def _drain_at_exit() -> None: - db = self_ref() - if db is not None: - db._drain_token_queue_at_exit() - - self._token_atexit_hook = _drain_at_exit - atexit.register(_drain_at_exit) - self._token_queue_cond.notify_all() - if writer_stopped: - # Writer permanently stopped (close() ran; a stop-flagged but - # still-live writer keeps accepting — its loop drains before - # exiting). Enqueueing now would drop the delta silently: no - # writer will run and close() already unregistered the atexit - # hook. Apply inline instead so a closed-connection failure - # raises at the call site, exactly like the old synchronous - # update_token_counts path these call sites still guard for. - self.update_token_counts(session_id, **kwargs) - - def flush_token_counts(self, timeout: float = 5.0) -> bool: - """Block until every queued token delta has been applied. - - Returns True when the queue is fully drained, False on timeout - (callers then read totals that are stale by the still-queued - deltas — no worse than reading before the flush existed). - Never raises: apply failures are logged by the writer. - """ - # Fast path — nothing queued, nothing in flight. - if not self._token_queue and not self._token_writer_busy: - return True - batch = None - with self._token_queue_cond: - deadline = time.monotonic() + timeout - while self._token_queue or self._token_writer_busy: - # A live writer is authoritative even when stop-flagged - # (close() in progress): its loop drains the queue before - # exiting, and draining here instead would race its - # in-flight batch — newer deltas committing before older - # ones breaks the last-non-None-wins / first-accounted- - # route / COALESCE-backfill fields. Only when the writer is - # dead (or never started for these deltas) does the caller - # take the leftovers. Re-checked each wakeup: the writer - # can exit mid-wait with deltas enqueued after its final - # empty-queue check. busy is claimed while draining (same - # protocol as the writer) so a concurrent flush cannot - # report drained — or pop a newer delta — while this batch - # is still unapplied; a claimed busy therefore also means - # "wait", never "drain alongside". - thread = self._token_writer_thread - if ( - (thread is None or not thread.is_alive()) - and not self._token_writer_busy - ): - self._token_writer_busy = True - batch = list(self._token_queue) - self._token_queue.clear() - break - remaining = deadline - time.monotonic() - if remaining <= 0: - return False - self._token_queue_cond.wait(remaining) - if batch: - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - return True - - def _token_writer_loop(self) -> None: - while True: - with self._token_queue_cond: - idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS - while not self._token_queue and not self._token_writer_stop: - remaining = idle_deadline - time.monotonic() - if remaining <= 0: - # Publish retirement under the same lock used by - # queue_token_counts() to decide whether to spawn. An - # enqueue cannot strand a delta behind an exiting worker. - self._token_writer_thread = None - return - self._token_queue_cond.wait(remaining) - if not self._token_queue: - self._token_writer_thread = None - return # stop requested and fully drained - # busy is set BEFORE the queue is cleared: the lock-free - # fast path in flush_token_counts() reads queue-then-busy, - # so this order guarantees it can never observe an empty - # queue while the popped batch is still unapplied. - self._token_writer_busy = True - batch = list(self._token_queue) - self._token_queue.clear() - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - - def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None: - """Apply queued deltas in order, coalescing where safe. Never raises.""" - try: - coalesced = self._coalesce_token_deltas(batch) - except Exception as exc: - # Coalescing must never kill the writer thread (a dead writer - # can't be observed by callers). Fall back to applying the raw - # batch delta-by-delta — the merge is an optimization only. - logger.warning( - "async token accounting: coalesce failed, applying raw " - "batch: %s", exc, - ) - coalesced = batch - for session_id, kwargs in coalesced: - try: - self.update_token_counts(session_id, **kwargs) - except Exception as exc: - # Same contract as the old inline call sites: accounting - # loss is logged, never raised into a turn. - logger.warning( - "async token accounting: apply failed (session=%s): %s", - session_id, exc, - ) - - def _coalesce_token_deltas( - self, batch: List[Tuple[str, Dict[str, Any]]] - ) -> List[Tuple[str, Dict[str, Any]]]: - """Merge consecutive incremental deltas with an identical route. - - Only adjacent deltas merge, so ordering across sessions and across - a mid-session /model switch is preserved exactly. absolute=True - deltas (cumulative overwrites) never merge. - """ - groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = [] - for session_id, kwargs in batch: - key = None - if not kwargs.get("absolute"): - key = (session_id,) + tuple( - kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS - ) - if groups and key is not None and groups[-1][0] == key: - merged = groups[-1][2] - for f in self._TOKEN_DELTA_SUM_FIELDS: - merged[f] = merged.get(f, 0) + kwargs.get(f, 0) - for f in self._TOKEN_DELTA_COST_FIELDS: - value = kwargs.get(f) - if value is not None: - # None-preserving sum: an all-None run must stay - # None so COALESCE keeps the stored value untouched. - merged[f] = (merged.get(f) or 0.0) + value - else: - groups.append((key, session_id, dict(kwargs))) - return [(sid, kw) for _, sid, kw in groups] - - def _stop_token_writer(self, join_timeout: float = 10.0) -> None: - """Stop the writer thread and drain remaining deltas. Never raises.""" - with self._token_queue_cond: - self._token_writer_stop = True - self._token_queue_cond.notify_all() - thread = self._token_writer_thread - if thread is not None and thread.is_alive(): - thread.join(timeout=join_timeout) - if thread.is_alive(): - # Writer stuck mid-apply (pathological lock contention). - # Leave any queued deltas unapplied rather than racing the - # stuck apply and misordering/double-counting. - logger.warning( - "async token accounting: writer did not stop within %.0fs; " - "%d queued delta(s) not persisted", - join_timeout, len(self._token_queue), - ) - return - # Writer exited (or never started) — apply leftovers synchronously. - # Claim busy like the writer/flush drains do, so a concurrent - # flush_token_counts cannot fast-path True while this batch is - # still being applied; conversely, wait out a flush caller-drain - # that already claimed busy — close() nulls the connection right - # after this returns, and must not yank it mid-batch. - with self._token_queue_cond: - deadline = time.monotonic() + join_timeout - while self._token_writer_busy: - remaining = deadline - time.monotonic() - if remaining <= 0: - logger.warning( - "async token accounting: concurrent drain did not " - "finish within %.0fs; %d queued delta(s) not persisted", - join_timeout, len(self._token_queue), - ) - return - self._token_queue_cond.wait(remaining) - # busy is claimed BEFORE the queue is cleared — same ordering - # as the writer loop and the flush caller-drain. The lock-free - # fast path in flush_token_counts() reads queue-then-busy - # without the cond, so clearing first would let a concurrent - # flush observe "empty and idle" and return True while this - # popped batch is still unapplied. - batch = list(self._token_queue) - if batch: - self._token_writer_busy = True - self._token_queue.clear() - if batch: - try: - self._apply_token_batch(batch) - finally: - with self._token_queue_cond: - self._token_writer_busy = False - self._token_queue_cond.notify_all() - - def _drain_token_queue_at_exit(self) -> None: - try: - self._stop_token_writer() - except Exception: - pass # Best effort — never fatal at interpreter shutdown. - - def update_token_counts( - self, - session_id: str, - input_tokens: int = 0, - output_tokens: int = 0, - model: str = None, - cache_read_tokens: int = 0, - cache_write_tokens: int = 0, - reasoning_tokens: int = 0, - estimated_cost_usd: Optional[float] = None, - actual_cost_usd: Optional[float] = None, - cost_status: Optional[str] = None, - cost_source: Optional[str] = None, - pricing_version: Optional[str] = None, - billing_provider: Optional[str] = None, - billing_base_url: Optional[str] = None, - billing_mode: Optional[str] = None, - api_call_count: int = 0, - absolute: bool = False, - ) -> None: - """Update token counters and backfill model if not already set. - - When *absolute* is False (default), values are **incremented** — use - this for per-API-call deltas (CLI path). - - When *absolute* is True, values are **set directly** — use this when - the caller already holds cumulative totals (gateway path, where the - cached agent accumulates across messages). - """ - # Ensure the session row exists so the UPDATE doesn't silently affect - # 0 rows. Under concurrent load (cron + kanban + delegate_task) the - # initial create_session() may have failed due to SQLite locking. - # INSERT OR IGNORE is cheap and idempotent. - self._insert_session_row(session_id, "unknown", model=model) - if absolute: - sql = """UPDATE sessions SET - input_tokens = ?, - output_tokens = ?, - cache_read_tokens = ?, - cache_write_tokens = ?, - reasoning_tokens = ?, - estimated_cost_usd = COALESCE(?, 0), - actual_cost_usd = CASE - WHEN ? IS NULL THEN actual_cost_usd - ELSE ? - END, - cost_status = COALESCE(?, cost_status), - cost_source = COALESCE(?, cost_source), - pricing_version = COALESCE(?, pricing_version), - billing_provider = COALESCE(billing_provider, ?), - billing_base_url = COALESCE(billing_base_url, ?), - billing_mode = COALESCE(billing_mode, ?), - model = COALESCE(model, ?), - api_call_count = ? - WHERE id = ?""" - else: - sql = """UPDATE sessions SET - input_tokens = input_tokens + ?, - output_tokens = output_tokens + ?, - cache_read_tokens = cache_read_tokens + ?, - cache_write_tokens = cache_write_tokens + ?, - reasoning_tokens = reasoning_tokens + ?, - estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0), - actual_cost_usd = CASE - WHEN ? IS NULL THEN actual_cost_usd - ELSE COALESCE(actual_cost_usd, 0) + ? - END, - cost_status = COALESCE(?, cost_status), - cost_source = COALESCE(?, cost_source), - pricing_version = COALESCE(?, pricing_version), - billing_provider = COALESCE(billing_provider, ?), - billing_base_url = COALESCE(billing_base_url, ?), - billing_mode = COALESCE(billing_mode, ?), - model = COALESCE(model, ?), - api_call_count = COALESCE(api_call_count, 0) + ? - WHERE id = ?""" - has_accounted_usage = bool( - input_tokens or output_tokens or cache_read_tokens - or cache_write_tokens or reasoning_tokens or api_call_count - or estimated_cost_usd or actual_cost_usd - ) - params = ( - input_tokens, - output_tokens, - cache_read_tokens, - cache_write_tokens, - reasoning_tokens, - estimated_cost_usd, - actual_cost_usd, - actual_cost_usd, - cost_status, - cost_source, - pricing_version, - billing_provider if has_accounted_usage else None, - billing_base_url if has_accounted_usage else None, - billing_mode if has_accounted_usage else None, - model if has_accounted_usage else None, - api_call_count, - session_id, - ) - # Per-model usage attribution. ``update_token_counts`` is the single - # chokepoint every per-API-call delta flows through (CLI, gateway, cron, - # delegated runs — see conversation_loop / codex_runtime), and each call - # carries the model/provider *active at the time of that call*. The - # ``sessions`` row only keeps one (model, billing_provider) pair, so a - # mid-session ``/model`` switch otherwise attributes every token to the - # initial model (issue #51607). Recording the per-call delta into - # session_model_usage keyed by the live model preserves an accurate - # per-model breakdown regardless of how many times the user switches. - # - # Only the incremental path records here. Absolute cumulative updates - # cannot be split back into routes; Insights reconciles any positive - # residual against the aggregate session row instead. - record_model_usage = (not absolute) and ( - input_tokens or output_tokens or cache_read_tokens - or cache_write_tokens or reasoning_tokens or api_call_count - or estimated_cost_usd - ) - - def _do(conn): - row = conn.execute( - "SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - existing_model = row["model"] if row is not None else None - existing_provider = row["billing_provider"] if row is not None else None - existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0) - - # Session creation records the requested primary route before any API - # call. If it fails and fallback succeeds, the first accounted usage - # event is the first authoritative route. After that, preserve the - # legacy row: one row cannot represent mixed-provider usage. - first_accounted_route = ( - existing_api_calls == 0 - and has_accounted_usage - and bool(model) - and bool(billing_provider) - and (existing_model != model or existing_provider != billing_provider) - ) - if first_accounted_route: - conn.execute( - """UPDATE sessions - SET model = ?, billing_provider = ?, - billing_base_url = ?, billing_mode = ? - WHERE id = ?""", - (model, billing_provider, billing_base_url, billing_mode, session_id), - ) - conn.execute(sql, params) - if record_model_usage: - self._record_model_usage( - conn, - session_id, - model=model, - billing_provider=billing_provider, - billing_base_url=billing_base_url, - billing_mode=billing_mode, - input_tokens=input_tokens, - output_tokens=output_tokens, - cache_read_tokens=cache_read_tokens, - cache_write_tokens=cache_write_tokens, - reasoning_tokens=reasoning_tokens, - estimated_cost_usd=estimated_cost_usd, - actual_cost_usd=actual_cost_usd, - cost_status=cost_status, - cost_source=cost_source, - api_call_count=api_call_count, - ) - self._execute_write(_do) - - def _record_model_usage( - self, - conn, - session_id: str, - *, - model: Optional[str], - billing_provider: Optional[str], - billing_base_url: Optional[str], - billing_mode: Optional[str], - input_tokens: int, - output_tokens: int, - cache_read_tokens: int, - cache_write_tokens: int, - reasoning_tokens: int, - estimated_cost_usd: Optional[float], - actual_cost_usd: Optional[float], - cost_status: Optional[str], - cost_source: Optional[str], - api_call_count: int, - task: str = "", - ) -> None: - """Accumulate a per-API-call usage delta into session_model_usage. - - Runs inside the caller's write transaction (after the ``sessions`` - UPDATE) so the per-model rows stay consistent with the summary row. - When the caller omits the model/provider (some paths only pass token - deltas), fall back to the values already recorded on the session row — - the same COALESCE-from-session behaviour the summary update uses. - - ``task`` distinguishes what kind of work consumed the tokens: - ``''`` (empty) is the main agent loop; auxiliary calls record their - task name (``vision``, ``compression``, ``title_generation``, ...) - via :meth:`record_auxiliary_usage` (issue #23270). - """ - row = conn.execute( - "SELECT model, billing_provider, billing_base_url, billing_mode " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - sess_model = row["model"] if row is not None else None - sess_provider = row["billing_provider"] if row is not None else None - sess_base_url = row["billing_base_url"] if row is not None else None - sess_billing_mode = row["billing_mode"] if row is not None else None - - # Aux-task rows (task != '') must NOT inherit the session's main-loop - # route: an aux call may use a completely different provider/model - # (vision on gemini while the main loop runs anthropic). Missing info - # stays 'unknown'/empty rather than borrowing a misleading route. - if task: - eff_model = model or "unknown" - eff_provider = billing_provider or "" - eff_base_url = billing_base_url or "" - eff_billing_mode = billing_mode or "" - else: - eff_model = model or sess_model or "unknown" - eff_provider = billing_provider or sess_provider or "" - eff_base_url = billing_base_url or sess_base_url or "" - eff_billing_mode = billing_mode or sess_billing_mode or "" - now = time.time() - conn.execute( - """INSERT INTO session_model_usage ( - session_id, model, billing_provider, billing_base_url, billing_mode, - task, api_call_count, input_tokens, output_tokens, - cache_read_tokens, cache_write_tokens, reasoning_tokens, - estimated_cost_usd, actual_cost_usd, cost_status, cost_source, - first_seen, last_seen - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task) - DO UPDATE SET - api_call_count = api_call_count + excluded.api_call_count, - input_tokens = input_tokens + excluded.input_tokens, - output_tokens = output_tokens + excluded.output_tokens, - cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens, - cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens, - reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens, - estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd, - actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd, - cost_status = COALESCE(excluded.cost_status, cost_status), - cost_source = COALESCE(excluded.cost_source, cost_source), - last_seen = excluded.last_seen""", - ( - session_id, - eff_model, - eff_provider, - eff_base_url, - eff_billing_mode, - task or "", - api_call_count or 0, - input_tokens or 0, - output_tokens or 0, - cache_read_tokens or 0, - cache_write_tokens or 0, - reasoning_tokens or 0, - float(estimated_cost_usd or 0.0), - float(actual_cost_usd or 0.0), - cost_status, - cost_source, - now, - now, - ), - ) def ensure_session( self, @@ -10642,281 +4843,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._insert_session_row(session_id, source, model=model, **kwargs) return session_id - def record_auxiliary_usage( - self, - session_id: str, - task: str, - *, - model: Optional[str] = None, - billing_provider: Optional[str] = None, - billing_base_url: Optional[str] = None, - input_tokens: int = 0, - output_tokens: int = 0, - cache_read_tokens: int = 0, - cache_write_tokens: int = 0, - reasoning_tokens: int = 0, - estimated_cost_usd: Optional[float] = None, - api_call_count: int = 1, - ) -> None: - """Record an auxiliary LLM call's usage against *session_id* (issue #23270). - - Auxiliary calls (vision, compression, title_generation, web_extract, - session_search, ...) historically discarded their usage, leaving the - dashboard's per-model analytics blind to aux model spend. This writes - a per-(model, provider, task) delta into ``session_model_usage`` — - the same table the main loop's ``update_token_counts`` feeds — WITHOUT - touching the ``sessions`` summary row. That separation is deliberate: - the gateway overwrites session counters with absolute main-loop totals, - so folding aux tokens into the summary row would either be clobbered - or double-counted. Insights/analytics read the union of both. - - ``api_call_count`` defaults to 1 (one aux LLM call). Background-review - forks record an aggregate of N fork API calls in one write with - ``task='background_review'`` (issue #87250). - - Best-effort by contract: callers must never fail an aux call because - accounting failed. - """ - if not session_id or not task: - return - # FK on session_model_usage.session_id → sessions.id: ensure the row - # exists (same INSERT OR IGNORE guard update_token_counts uses — the - # initial create_session() can fail under concurrent SQLite locking). - self._insert_session_row(session_id, "unknown") - - def _do(conn): - self._record_model_usage( - conn, - session_id, - model=model, - billing_provider=billing_provider, - billing_base_url=billing_base_url, - billing_mode=None, - input_tokens=input_tokens or 0, - output_tokens=output_tokens or 0, - cache_read_tokens=cache_read_tokens or 0, - cache_write_tokens=cache_write_tokens or 0, - reasoning_tokens=reasoning_tokens or 0, - estimated_cost_usd=estimated_cost_usd, - actual_cost_usd=None, - cost_status=None, - cost_source=None, - api_call_count=( - 1 if api_call_count is None else int(api_call_count) - ), - task=task, - ) - self._execute_write(_do) - - def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int: - """Remove empty TUI ghost sessions (no messages, no title, >24hr old).""" - cutoff = time.time() - 86400 # Only sessions older than 24 hours - - def _do(conn): - rows = conn.execute(""" - SELECT id FROM sessions - WHERE source = 'tui' - AND title IS NULL - AND ended_at IS NOT NULL - AND started_at < ? - AND NOT EXISTS ( - SELECT 1 FROM messages WHERE messages.session_id = sessions.id - ) - """, (cutoff,)).fetchall() - ids = [r[0] if isinstance(r, (tuple, list)) else r["id"] for r in rows] - if ids: - placeholders = ",".join("?" * len(ids)) - conn.execute( - f"DELETE FROM sessions WHERE id IN ({placeholders})", ids - ) - self._delete_unreferenced_system_prompts(conn) - return ids - - removed_ids = self._execute_write(_do) or [] - # Clean up any on-disk session files (belt-and-suspenders) - if sessions_dir and removed_ids: - for sid in removed_ids: - self._remove_session_files(sessions_dir, sid) - return len(removed_ids) - - def finalize_orphaned_compression_sessions(self) -> int: - """Mark orphaned compression continuation sessions as ended. - - Targets child sessions that were never finalized: parent is ended - with reason='compression', child has messages but no end_reason/ended_at - and api_call_count=0. Non-destructive: preserves all messages and sets - end_reason='orphaned_compression'. Fix for #20001. - """ - cutoff = time.time() - 604800 # 7 days - - def _do(conn): - now = time.time() - result = conn.execute( - """ - UPDATE sessions - SET ended_at = ?, - end_reason = 'orphaned_compression' - WHERE api_call_count = 0 - AND end_reason IS NULL - AND ended_at IS NULL - AND started_at < ? - AND parent_session_id IS NOT NULL - AND EXISTS ( - SELECT 1 FROM sessions p - WHERE p.id = sessions.parent_session_id - AND p.end_reason = 'compression' - AND p.ended_at IS NOT NULL - ) - AND EXISTS ( - SELECT 1 FROM messages m - WHERE m.session_id = sessions.id - ) - """, - (now, cutoff), - ) - return result.rowcount - - return self._execute_write(_do) or 0 - - def sweep_orphaned_sessions( - self, - *, - max_idle_seconds: float, - sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), - exclude_ids: Tuple[str, ...] = (), - exclude_pinned: bool = False, - heartbeat_staleness_seconds: Optional[float] = None, - heartbeat_ownership_grace_seconds: Optional[float] = None, - respect_gateway_heartbeats: bool = True, - ) -> List[str]: - """Close session rows orphaned by a dead gateway process (#65194, #94895). - - The TUI/desktop gateway reaps disconnected websocket sessions with an - in-process ``threading.Timer`` grace timer; a gateway restart destroys - the timer and leaves the row ``ended_at IS NULL`` forever. This is the - startup-time complement: it closes rows for the given ``sources`` whose - ``started_at`` and canonical last-activity time are both older than - ``max_idle_seconds``, with a distinct - ``end_reason='startup_orphan_reap'`` for traceability. - - Canonical activity is the newest of ``last_activity_at`` (the in-turn - heartbeat) and the newest durable message timestamp, falling back to - ``started_at``. The separate ``started_at`` predicate protects freshly - created compression/branch children whose copied activity is old. - - Only pass sources whose lifecycle the caller owns (never messaging-gateway - platforms like ``telegram`` — ending those triggers the #60609 routing - loop). ``exclude_ids`` spares rows this process still holds in memory - (a ``session.resume`` that landed during the startup grace window). - ``exclude_pinned`` is intended for broad automatic sweeps; pinned rows - remain explicitly recoverable. Non-destructive: messages are preserved - and the row remains resumable. First-reason-wins is preserved via - ``ended_at IS NULL``. - - Cross-backend liveness (#94895): when one ``state.db`` is shared by N - serve / gateway processes, each backend refreshes a row in - ``gateway_heartbeats``. With ``respect_gateway_heartbeats`` enabled, a - row is only reaped when activity staleness holds AND no live backend - (heartbeat refreshed within ``heartbeat_staleness_seconds``, default - ``2 * max_idle_seconds``) could plausibly own it. Disable that gate only - for sources whose lifecycle is explicitly owned by state.db itself. - - Ownership inference: a live backend B ``owns`` a session S if - ``B.started_at <= S.started_at + heartbeat_ownership_grace_seconds`` - (default ``heartbeat_staleness_seconds``). The grace window covers a - migrating backend whose existing sessions predate its first heartbeat, - but is bounded so a fresh PID-reuse respawn cannot protect rows forever. - With no fresh heartbeat the predicate falls back to the legacy sweep. - - The SELECT, live-lease validation, and UPDATE run in one - ``BEGIN IMMEDIATE`` transaction. Active turn leases or compression - locks spare the row; expired/reclaimed guards are removed so their - former owner is fenced. Returns the swept session ids. - """ - srcs = tuple(s for s in sources if s) - if max_idle_seconds <= 0 or not srcs: - return [] - hb_staleness = ( - heartbeat_staleness_seconds - if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0 - else max_idle_seconds * 2 - ) - hb_grace = ( - heartbeat_ownership_grace_seconds - if heartbeat_ownership_grace_seconds is not None - and heartbeat_ownership_grace_seconds >= 0 - else hb_staleness - ) - now = time.time() - cutoff = now - max_idle_seconds - hb_cutoff = now - hb_staleness - placeholders = ",".join("?" for _ in srcs) - staleness = ( - f"started_at < ? AND {_sql_session_last_active('sessions')} < ?" - ) - pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else "" - heartbeat_params: Tuple[float, ...] = () - orphan_predicate = staleness - if respect_gateway_heartbeats: - orphan_predicate += ( - " AND NOT EXISTS (" - "SELECT 1 FROM gateway_heartbeats h" - " WHERE h.last_heartbeat >= ?" - " AND h.started_at <= sessions.started_at + ?" - ")" - ) - heartbeat_params = (hb_cutoff, hb_grace) - - def _do(conn): - rows = conn.execute( - f"SELECT id FROM sessions WHERE ended_at IS NULL" - f" AND source IN ({placeholders}){pin_scope}" - f" AND {orphan_predicate}", - (*srcs, cutoff, cutoff, *heartbeat_params), - ).fetchall() - excluded = {str(x) for x in exclude_ids if x} - victims = [] - for row in rows: - sid = str(row["id"]) - if sid in excluded: - continue - try: - self._check_transcript_write_guards( - conn, - sid, - compression_lock_holder=None, - turn_lease_holder=None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - except ( - SessionCompressionInProgressError, - SessionTurnLeaseLostError, - ): - continue - victims.append(sid) - if not victims: - return [] - closed_at = time.time() - marks = ",".join("?" for _ in victims) - # Re-apply every scope/liveness predicate under the write lock. - conn.execute( - f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" - f" WHERE id IN ({marks}) AND ended_at IS NULL" - f" AND source IN ({placeholders}){pin_scope}" - f" AND {orphan_predicate}", - ( - closed_at, - *victims, - *srcs, - cutoff, - cutoff, - *heartbeat_params, - ), - ) - return victims - - return self._execute_write(_do) or [] # ── Cross-backend heartbeat API (#94895) ─────────────────────────── # Each serve / tui_gateway process registers a heartbeat row at startup @@ -10926,98 +4852,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # their own row on graceful shutdown; a row that survives a crash is # reclaimed by the staleness sweep once ``last_heartbeat`` ages out. - def register_backend_heartbeat( - self, - *, - backend_id: str, - pid: int, - started_at: float, - last_heartbeat: Optional[float] = None, - profile: str = "", - host: str = "", - ) -> None: - """Upsert this backend's liveness row (#94895). - - ``backend_id`` MUST be stable for the lifetime of the process - (e.g. ``f"{profile}@{host}:{pid}"``) so a respawn cannot accidentally - inherit the dead predecessor's heartbeat and protect stale rows. - ``started_at`` records when THIS process started (not the wall clock - at first refresh) so a long-lived backend whose previous run died - cannot be confused with a freshly-spawned sibling. - """ - if not backend_id: - return - ts = time.time() if last_heartbeat is None else float(last_heartbeat) - def _do(conn): - conn.execute( - "INSERT INTO gateway_heartbeats" - " (backend_id, pid, started_at, last_heartbeat, profile, host)" - " VALUES (?, ?, ?, ?, ?, ?)" - " ON CONFLICT(backend_id) DO UPDATE SET" - " pid = excluded.pid," - " started_at = excluded.started_at," - " last_heartbeat = excluded.last_heartbeat," - " profile = excluded.profile," - " host = excluded.host", - (str(backend_id), int(pid), float(started_at), ts, - str(profile), str(host)), - ) - self._execute_write(_do) - - def clear_backend_heartbeat(self, backend_id: str) -> bool: - """Remove this backend's heartbeat row (#94895). - - Called from ``atexit`` so a graceful shutdown doesn't leave a stale - row behind. A crashed backend's row is reclaimed later by - ``prune_stale_heartbeats``. Returns True if a row was removed. - """ - if not backend_id: - return False - def _do(conn): - cur = conn.execute( - "DELETE FROM gateway_heartbeats WHERE backend_id = ?", - (str(backend_id),), - ) - return cur.rowcount > 0 - return bool(self._execute_write(_do)) - - def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]: - """Drop heartbeat rows whose ``last_heartbeat`` is older than the - staleness window. Returns the removed backend ids. Safe to call - from any process; only stale rows are touched. - """ - if max_age_seconds <= 0: - return [] - cutoff = time.time() - max_age_seconds - def _do(conn): - cur = conn.execute( - "DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?" - " RETURNING backend_id", - (cutoff,), - ) - return [str(r[0]) for r in cur.fetchall()] - return list(self._execute_write(_do) or []) - - def list_backend_heartbeats(self) -> List[Dict[str, Any]]: - """Snapshot of every registered backend's heartbeat (for diagnostics - and tests). The fields mirror ``gateway_heartbeats`` exactly. - """ - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT backend_id, pid, started_at, last_heartbeat," - " profile, host FROM gateway_heartbeats" - " ORDER BY last_heartbeat DESC" - ).fetchall() - out: List[Dict[str, Any]] = [] - for r in rows: - if isinstance(r, sqlite3.Row): - out.append({k: r[k] for k in r.keys()}) - else: - out.append({ - "backend_id": r[0], "pid": r[1], "started_at": r[2], - "last_heartbeat": r[3], "profile": r[4], "host": r[5], - }) - return out def get_session(self, session_id: str) -> Optional[Dict[str, Any]]: """Get a session by ID.""" @@ -11025,16 +4859,14 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # row through here; drain queued token deltas so they see exact # totals. No-op attribute check when nothing is queued. self.flush_token_counts() - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.id = ?", - (session_id,), - ) - row = cursor.fetchone() + row = self._read_one( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.id = ?", + (session_id,), + ) return self._session_row_dict(row) if row else None def get_dominant_session_model_route( @@ -11048,22 +4880,21 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) its dominant main-loop route when one is available. """ self.flush_token_counts() - with self._read_ctx() as conn: - row = conn.execute( - """SELECT model, billing_provider, billing_base_url, billing_mode, - api_call_count - FROM session_model_usage - WHERE session_id = ? - AND task = '' - AND model <> 'unknown' - AND billing_provider <> '' - ORDER BY api_call_count DESC, - (input_tokens + output_tokens + cache_read_tokens + - cache_write_tokens + reasoning_tokens) DESC, - last_seen DESC - LIMIT 1""", - (session_id,), - ).fetchone() + row = self._read_one( + """SELECT model, billing_provider, billing_base_url, billing_mode, + api_call_count + FROM session_model_usage + WHERE session_id = ? + AND task = '' + AND model <> 'unknown' + AND billing_provider <> '' + ORDER BY api_call_count DESC, + (input_tokens + output_tokens + cache_read_tokens + + cache_write_tokens + reasoning_tokens) DESC, + last_seen DESC + LIMIT 1""", + (session_id,), + ) return dict(row) if row else None def resolve_session_id(self, session_id_or_prefix: str) -> Optional[str]: @@ -11078,12 +4909,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return exact["id"] escaped = _escape_like(session_id_or_prefix) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2", - (f"{escaped}%",), - ) - matches = [row["id"] for row in cursor.fetchall()] + matches = [row["id"] for row in self._read_all( + "SELECT id FROM sessions WHERE id LIKE ? ESCAPE '\\' ORDER BY started_at DESC LIMIT 2", + (f"{escaped}%",), + )] if len(matches) == 1: return matches[0] return None @@ -11111,291 +4940,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # row holding it (#92473). CANONICAL_BOT_CHAT_TITLE = "Bot Chat" - @classmethod - def _title_rank(cls, source: Optional[str]) -> int: - """Rank a stored title_source. NULL means a pre-provenance row. - - Rows written before this column existed carry NULL. They were almost - always set by the old auto-titler, but a manual ``/title`` from that - era is indistinguishable — so treat NULL as ``user`` and refuse to - overwrite it. Auto-titling only ever fills genuinely empty titles on - legacy rows, which is the conservative direction. - """ - if source is None: - return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER] - return cls._TITLE_SOURCE_RANK.get(str(source), 0) - - @staticmethod - def sanitize_title(title: Optional[str]) -> Optional[str]: - """Validate and sanitize a session title. - - - Strips leading/trailing whitespace - - Removes ASCII control characters (0x00-0x1F, 0x7F) and problematic - Unicode control chars (zero-width, RTL/LTR overrides, etc.) - - Collapses internal whitespace runs to single spaces - - Normalizes empty/whitespace-only strings to None - - Enforces MAX_TITLE_LENGTH - - Returns the cleaned title string or None. - Raises ValueError if the title exceeds MAX_TITLE_LENGTH after cleaning. - """ - if not title: - return None - - # Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at - # UTF-8 encode time) — scrub them like every other write path here. - title = _sanitize_surrogates(title) - - # Remove ASCII control characters (0x00-0x1F, 0x7F) but keep - # whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be - # normalized to spaces by the whitespace collapsing step below - cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title) - - # Remove problematic Unicode control characters: - # - Zero-width chars (U+200B-U+200F, U+FEFF) - # - Directional overrides (U+202A-U+202E, U+2066-U+2069) - # - Object replacement (U+FFFC), interlinear annotation (U+FFF9-U+FFFB) - cleaned = re.sub( - r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]', - '', cleaned, - ) - - # Collapse internal whitespace runs and strip - cleaned = re.sub(r'\s+', ' ', cleaned).strip() - - if not cleaned: - return None - - if len(cleaned) > SessionDB.MAX_TITLE_LENGTH: - raise ValueError( - f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})" - ) - - return cleaned - - def _is_compression_ancestor( - self, conn, *, ancestor_id: str, descendant_id: str - ) -> bool: - """Return True if *ancestor_id* is a compression predecessor of - *descendant_id* (walking parent links up the continuation chain). - - The continuation edge is the canonical one shared with - :func:`_ephemeral_child_sql` / :meth:`set_session_archived` - (``_COMPRESSION_CHILD_SQL``): a parent → child edge counts only when the - parent ended with ``end_reason = 'compression'`` and the child started - at or after the parent's ``ended_at``, which distinguishes continuations - from delegate subagents / branch children that also carry a - ``parent_session_id``. Expressed as a single recursive CTE rather than a - per-hop Python walk so the edge definition lives in exactly one place. - """ - if not ancestor_id or not descendant_id or ancestor_id == descendant_id: - return False - # Walk parent links up from the descendant, following only compression - # continuation edges, and check whether ancestor_id is reached. - edge = _COMPRESSION_CHILD_SQL.format(a="child") - row = conn.execute( - f""" - WITH RECURSIVE ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE {edge} - ) - SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1 - """, - (descendant_id, ancestor_id, descendant_id), - ).fetchone() - return row is not None - - def _set_session_title( - self, - session_id: str, - title: str, - *, - source: str, - ) -> bool: - """Write a title, enforcing provenance precedence. - - ``source`` is one of ``TITLE_SOURCE_{DERIVED,LLM,USER}``. A ``user`` - write always lands — an explicit rename is authoritative. An automatic - write (``derived``/``llm``) lands only when the row is untitled or the - stored title has strictly lower authority, so the instant ``derived`` - title upgrades to ``llm`` exactly once and neither can ever overwrite a - name the user typed. Re-running the titler on an already-``llm`` row is - a no-op, which is what stops a session renaming itself. The one thing - no writer may do is move a hidden canonical Bot Chat off its title. - - The read and the write are one compare-and-swap inside a single - transaction, so a manual ``/title`` racing an in-flight generation - cannot be clobbered by the late arrival. - """ - title = self.sanitize_title(title) - is_user = source == self.TITLE_SOURCE_USER - new_rank = self._title_rank(source) if not is_user else None - - def _do(conn): - current = conn.execute( - "SELECT title, title_source, hidden FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if current is None: - return 0 - # The canonical Bot Chat's NAME is its identity: Bot Mode resolves - # the forever-chat by exact-title lookup on every open, so renaming - # the row orphans the entire conversation — the next click mints an - # empty replacement and UNIQUE(title) then blocks ever renaming - # back (#92473). Refuse the rename at the single write path every - # surface funnels through (gateway session.title, /title, CLI - # rename, REST). Hidden is the discriminator: canonical chats are - # born hidden; an ordinary visible session a user happens to call - # "Bot Chat" stays freely renameable. Provenance-blind: an - # automatic llm write outranks a derived title, so the auto-titler - # would otherwise rename the row too (#99517) — it no-ops instead. - if ( - (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE - and bool(current["hidden"]) - and title != self.CANONICAL_BOT_CHAT_TITLE - ): - if is_user: - raise ValueError( - "This is the bot's canonical Bot Chat — its name is its " - "identity, and renaming it would orphan the conversation. " - "To start fresh, create a new bot instead." - ) - return 0 - if not is_user and current["title"] is not None: - if self._title_rank(current["title_source"]) >= new_rank: - return 0 - - if title: - # Check uniqueness (allow the same session to keep its own title) - cursor = conn.execute( - "SELECT id FROM sessions WHERE title = ? AND id != ?", - (title, session_id), - ) - conflict = cursor.fetchone() - if conflict: - conflict_id = conflict["id"] - # A compression continuation is the live, projected-forward - # head of its conversation; its compressed predecessors are - # ended and hidden from the session list (list_sessions_rich - # projects roots → tip). When the title that "conflicts" is - # held by such a hidden ancestor, the user has no way to free - # it — renaming the visible tip back to the base name would - # dead-end with "already in use by ". - # Treat this as a transfer: move the title off the ancestor - # onto the continuation. Uniqueness is preserved (still only - # one session carries the exact title) and the parent-link - # lineage is untouched. - if self._is_compression_ancestor( - conn, ancestor_id=conflict_id, descendant_id=session_id - ): - conn.execute( - "UPDATE sessions SET title = NULL WHERE id = ?", - (conflict_id,), - ) - else: - raise ValueError( - f"Title '{title}' is already in use by session {conflict_id}" - ) - # Compare-and-swap on the exact values we just read (``IS`` is - # NULL-safe in SQLite), so a concurrent write between the SELECT - # and here loses instead of being silently overwritten. - cursor = conn.execute( - "UPDATE sessions SET title = ?, title_source = ? " - "WHERE id = ? AND title IS ? AND title_source IS ?", - ( - title, - source if title else None, - session_id, - current["title"], - current["title_source"], - ), - ) - return cursor.rowcount - - rowcount = self._execute_write(_do) - return rowcount > 0 - - def set_session_title(self, session_id: str, title: str) -> bool: - """Set or update a session's title on the user's behalf. - - Returns True if session was found and title was set. - Raises ValueError if title is already in use by another session, - or if the title fails validation (too long, invalid characters). - Empty/whitespace-only strings are normalized to None (clearing the title). - - This records ``user`` provenance, so auto-titling will never replace - the result. Automatic callers must use :meth:`set_auto_title`. - """ - return self._set_session_title( - session_id, title, source=self.TITLE_SOURCE_USER - ) - - def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool: - """Set an automatically generated title, honoring provenance precedence. - - Returns True when the title was written, False when a higher-authority - title already holds the row (nothing is modified in that case). - """ - if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM): - raise ValueError(f"invalid automatic title source: {source!r}") - return self._set_session_title(session_id, title, source=source) - - def set_auto_title_if_empty(self, session_id: str, title: str) -> bool: - """Back-compat shim: set an LLM title only if nothing better exists. - - Retained because older callers (and third-party plugins) reference it - by name. New code should call :meth:`set_auto_title` with an explicit - source. - """ - return self.set_auto_title( - session_id, title, source=self.TITLE_SOURCE_LLM - ) - - def get_session_title(self, session_id: str) -> Optional[str]: - """Get the title for a session, or None.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title FROM sessions WHERE id = ?", (session_id,) - ) - row = cursor.fetchone() - return row["title"] if row else None - - def get_session_title_source(self, session_id: str) -> Optional[str]: - """Get the provenance of a session's title, or None when untitled.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title, title_source FROM sessions WHERE id = ?", - (session_id,), - ) - row = cursor.fetchone() - if not row or row["title"] is None: - return None - return row["title_source"] - - def set_session_title_source(self, session_id: str, source: str) -> bool: - """Overwrite a title's provenance without touching the title text. - - Used when a title is carried across a session boundary (compression - rotation) and the copy must keep the original's authority rather than - the authority of whichever setter performed the copy. - """ - if source not in self._TITLE_SOURCE_RANK: - raise ValueError(f"invalid title source: {source!r}") - - def _do(conn): - cursor = conn.execute( - "UPDATE sessions SET title_source = ? " - "WHERE id = ? AND title IS NOT NULL", - (source, session_id), - ) - return cursor.rowcount - - return self._execute_write(_do) > 0 def backfill_null_session_profiles(self, profile_name: str) -> int: """One-shot owner backfill for legacy pre-ownership session rows. @@ -11422,19 +4966,54 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not stamp: return 0 - def _do(conn): - cursor = conn.execute( - """UPDATE sessions - SET profile_name = ? - WHERE profile_name IS NULL OR TRIM(profile_name) = ''""", - (stamp,), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount + return int(self._write_rowcount( + """UPDATE sessions + SET profile_name = ? + WHERE profile_name IS NULL OR TRIM(profile_name) = ''""", + (stamp,), + ) or 0) - return int(self._execute_write(_do) or 0) + def _set_lineage_column(self, column: str, session_id: str, value: Any) -> bool: + """Set one ``sessions`` column across a whole compression lineage. + + Walks ancestors and descendants joined by ``end_reason='compression'`` + so the root and every continuation flip as a unit — Desktop projects + compression roots forward to their latest tip, and updating only the + displayed tip would let the untouched root resurrect it on refresh. + Returns True when at least one row changed. + """ + return self._write_rowcount( + f""" + WITH RECURSIVE + ancestors(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM ancestors a + JOIN sessions child ON child.id = a.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE parent.end_reason = 'compression' + ), + descendants(id) AS ( + SELECT ? + UNION + SELECT child.id + FROM descendants d + JOIN sessions parent ON parent.id = d.id + JOIN sessions child ON child.parent_session_id = parent.id + WHERE parent.end_reason = 'compression' + ), + lineage(id) AS ( + SELECT id FROM ancestors + UNION + SELECT id FROM descendants + ) + UPDATE sessions + SET {column} = ? + WHERE id IN (SELECT id FROM lineage) + """, + (session_id, session_id, value), + ) > 0 def set_session_archived(self, session_id: str, archived: bool) -> bool: """Archive or unarchive a session. @@ -11446,45 +5025,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) displayed tip lets the still-unarchived root resurrect it on refresh. Returns True when at least one row was updated. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET archived = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if archived else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('archived', session_id, 1 if archived else 0) # Accidental end reasons that recovery treats as resumable. Single source # of truth: hermes_state_common._RECOVERABLE_END_REASONS, interpolated @@ -11555,45 +5096,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) vice-versa) no matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET pinned = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if pinned else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('pinned', session_id, 1 if pinned else 0) def set_session_hidden(self, session_id: str, hidden: bool) -> bool: """Hide or unhide a session (and its whole compression lineage). @@ -11609,45 +5112,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET hidden = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, 1 if hidden else 0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('hidden', session_id, 1 if hidden else 0) def set_session_read(self, session_id: str, read: bool = True) -> bool: """Mark a session read or unread (and its whole compression lineage). @@ -11669,45 +5134,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) tip clears the root (and vice-versa) no matter which id the caller holds. Returns True when at least one row changed. """ - def _do(conn): - cursor = conn.execute( - """ - WITH RECURSIVE - ancestors(id) AS ( - SELECT ? - UNION - SELECT parent.id - FROM ancestors a - JOIN sessions child ON child.id = a.id - JOIN sessions parent ON parent.id = child.parent_session_id - WHERE parent.end_reason = 'compression' - ), - descendants(id) AS ( - SELECT ? - UNION - SELECT child.id - FROM descendants d - JOIN sessions parent ON parent.id = d.id - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.end_reason = 'compression' - ), - lineage(id) AS ( - SELECT id FROM ancestors - UNION - SELECT id FROM descendants - ) - UPDATE sessions - SET last_read_at = ? - WHERE id IN (SELECT id FROM lineage) - """, - (session_id, session_id, time.time() if read else 0.0), - ) - rowcount = cursor.rowcount - if rowcount is None or rowcount < 0: - rowcount = conn.execute("SELECT changes()").fetchone()[0] - return rowcount - rowcount = self._execute_write(_do) - return rowcount > 0 + return self._set_lineage_column('last_read_at', session_id, time.time() if read else 0.0) @staticmethod def session_unread(session_row: Dict[str, Any]) -> bool: @@ -11723,157 +5150,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) last_active = session_row.get("last_active") or session_row.get("started_at") return float(last_active or 0) > float(last_read) - def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]: - """Look up a session by exact title. Returns session dict or None.""" - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.title = ?", - (title,), - ) - row = cursor.fetchone() - return self._session_row_dict(row) if row else None - - def resolve_session_by_title(self, title: str) -> Optional[str]: - """Resolve a title to a session ID, preferring the latest in a lineage. - - If the exact title exists, returns that session's ID. - If not, searches for "title #N" variants and returns the latest one. - If the exact title exists AND numbered variants exist, returns the - latest numbered variant (the most recent continuation). - """ - # First try exact match - exact = self.get_session_by_title(title) - - # Also search for numbered variants: "title #2", "title #3", etc. - # Escape SQL LIKE wildcards (%, _) in the title to prevent false matches - escaped = _escape_like(title) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT id, title, started_at FROM sessions " - "WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC", - (f"{escaped} #%",), - ) - numbered = cursor.fetchall() - - if numbered: - # Return the most recent numbered variant - return numbered[0]["id"] - elif exact: - return exact["id"] - return None - - def get_next_title_in_lineage(self, base_title: str) -> str: - """Generate the next title in a lineage (e.g., "my session" → "my session #2"). - - Strips any existing " #N" suffix to find the base name, then finds - the highest existing number and increments. - """ - # Strip existing #N suffix to find the true base - match = re.match(r'^(.*?) #(\d+)$', base_title) - if match: - base = match.group(1) - else: - base = base_title - - # Find all existing numbered variants - # Escape SQL LIKE wildcards (%, _) in the base to prevent false matches - escaped = _escape_like(base) - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'", - (base, f"{escaped} #%"), - ) - existing = [row["title"] for row in cursor.fetchall()] - - if not existing: - return base # No conflict, use the base name as-is - - # Find the highest number - max_num = 1 # The unnumbered original counts as #1 - for t in existing: - m = re.match(r'^.* #(\d+)$', t) - if m: - max_num = max(max_num, int(m.group(1))) - - return f"{base} #{max_num + 1}" - - def get_compression_chain(self, session_id: str) -> List[str]: - """Walk the compression-continuation chain forward and return every id. - - Root-first order, ending at the tip; ``[session_id]`` when no - continuation exists. ``get_compression_tip`` is this walk's last - element — kept as the single implementation so the two can never - disagree about what the chain is. - - A compression continuation is a child of a session whose - ``end_reason = 'compression'``. Older builds tried to distinguish - continuations from branches/subagents by requiring - ``child.started_at >= parent.ended_at``. That ordering is too brittle: - gateway + compression races can insert the real continuation row before - the parent row's ``ended_at`` is written, while a stale websocket later - creates/reuses a sibling that *does* satisfy the timestamp test. The - visible symptom is brutal: desktop resume follows the stale sibling and - the user's latest messages look "lost" even though they are persisted in - the real continuation chain. - - Instead, only follow children of compression-ended parents, exclude - explicit branch/delegate/tool children, and prefer children that are - themselves continuing the compression chain (``end_reason='compression'``) - or still live over stale closed siblings such as ``ws_orphan_reap``. - Returns the latest continuation tip, or the input id when no - continuation exists. - """ - current = session_id - chain = [current] if current else [] - seen = {current} if current else set() - # Bound the walk defensively — compression chains this deep are - # pathological and shouldn't happen in practice. 100 = plenty. - for _ in range(100): - with self._read_ctx() as conn: - cursor = conn.execute( - f""" - SELECT child.id - FROM sessions parent - JOIN sessions child ON child.parent_session_id = parent.id - WHERE parent.id = ? - AND parent.end_reason = 'compression' - AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL - AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL - AND COALESCE(child.source, '') != 'tool' - ORDER BY - CASE - WHEN child.end_reason = 'compression' THEN 0 - WHEN child.ended_at IS NULL THEN 1 - ELSE 2 - END, - {_sql_session_last_active("child")} DESC, - child.started_at DESC, - child.id DESC - LIMIT 1 - """, - (current,), - ) - row = cursor.fetchone() - if row is None: - return chain - child_id = row["id"] - if not child_id or child_id in seen: - return chain - seen.add(child_id) - current = child_id - chain.append(child_id) - return chain - - def get_compression_tip(self, session_id: str) -> Optional[str]: - """The live tip of a compression-continuation chain (see - ``get_compression_chain`` for the walk's semantics). Returns the input - id when no continuation exists.""" - chain = self.get_compression_chain(session_id) - return chain[-1] if chain else session_id # Columns excluded from compact_rows projections: only the payload-heavy # blob no list consumer renders. Everything else — including gateway @@ -11886,34 +5162,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) _session_compact_cols_sql: Optional[str] = None - def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]: - """Tokens and spend across this store, as one aggregate. - - The sidebar shows a profile's totals beside a page of its sessions, so - summing the rows it happens to have loaded would report a fraction of - the truth and shrink as paging changed. SQLite adds the columns up over - every row instead, at the cost of one scan. - - Spend is the billed figure when the provider returned one and the - estimate otherwise — the same precedence a single row renders. - """ - where = ["parent_session_id IS NULL", "message_count >= ?"] - params: List[Any] = [min_message_count] - if not include_archived: - where.append("COALESCE(archived, 0) = 0") - - with self._read_ctx() as conn: - row = conn.execute( - f""" - SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0), - COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0) - FROM sessions - WHERE {' AND '.join(where)} - """, - params, - ).fetchone() - - return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)} def list_sessions_rich( self, @@ -11936,59 +5184,28 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) session_key: str = None, include_hidden: bool = False, ) -> List[Dict[str, Any]]: - """List sessions with preview (first user message) and last active timestamp. + """List sessions with preview (first user message) and last_active, in one query. - Returns dicts with keys: id, source, model, title, started_at, ended_at, - message_count, preview (first 60 chars of first user message), - last_active (freshest of last_activity_at heartbeat and latest - message timestamp, else started_at). + ``last_active`` = freshest of the activity heartbeat and the latest + message timestamp, else ``started_at``. Implementation-detail children + (subagent runs, compression continuations) are excluded unless + ``include_children``; branch/reset children stay listable. - Uses a single query with correlated subqueries instead of N+2 queries. - - By default, child sessions that represent implementation details - (subagent runs, compression continuations) are excluded. User-visible - branch and reset children remain listable. Pass ``include_children=True`` - to include every child. - - With ``project_compression_tips=True`` (default), sessions that are - roots of compression chains are projected forward to their latest - continuation — one logical conversation = one list entry, showing the - live continuation's id/message_count/title/last_active. This prevents - compressed continuations from being invisible to users while keeping - delegate subagents and branches hidden. Pass ``False`` to return the - raw root rows (useful for admin/debug UIs). - - Pass ``order_by_last_active=True`` to sort by most-recent activity - instead of original conversation start time. For compression chains, - the "most-recent activity" is taken from the live tip (not the root), - so an old conversation that was compressed and continued recently - surfaces in the correct slot. Ordering is computed at SQL level via - a recursive CTE that walks compression-continuation edges, so LIMIT - and OFFSET still apply efficiently. - - ``search_query`` matches case-insensitive substrings against each - surfaced row's title and id (and, like ``id_query``, every title/id in - its forward compression chain). A punctuation-stripped variant is also - matched so e.g. ``an94`` finds ``AN-94``. Only honored in the - ``order_by_last_active`` path. - - Pass ``compact_rows=True`` for dashboard and picker callers that only - need lightweight metadata. This omits the ``system_prompt`` blob from - the SELECT so SQLite never copies it out of the B-tree page — a - significant I/O saving on large databases where the blob routinely - runs to tens of kilobytes per row. - - Pass ``include_pinned=True`` to back-fill any conversation carrying the - durable ``pinned`` flag that the LIMIT/OFFSET window left out. A pin is - a "this must always be reachable" statement, so a pinned conversation - aging past the requested page is a bug, not a paging outcome — the - desktop sidebar would render an empty Pinned section. Back-filled rows - obey the same filters (source, archived, min_message_count) as the - page: an archived or filtered-out conversation stays out. - - Pass ``session_key`` to restrict results to one stable gateway - conversation scope (DM, group, channel, or thread, including the - configured per-user isolation policy). + ``project_compression_tips`` (default) surfaces each compression chain + as ONE entry showing the live tip's id/message_count/title/last_active; + ``False`` returns raw root rows (admin/debug UIs). + ``order_by_last_active`` sorts by the chain TIP's activity via a + recursive CTE, so LIMIT/OFFSET stay cheap and a recently continued old + conversation lands in the right slot. ``search_query`` (that path + only) substring-matches title/id across the forward chain, plus a + punctuation-stripped variant (``an94`` finds ``AN-94``). + ``compact_rows`` omits the ``system_prompt`` blob from the SELECT so + SQLite never copies tens of KB per row out of the B-tree page. + ``include_pinned`` back-fills pinned conversations the page window left + out (a pin means "always reachable"; the desktop sidebar would + otherwise render an empty Pinned section) — they still obey the + source/archived/min_message_count filters. ``session_key`` restricts to + one gateway conversation scope. """ # Rows carry token/cost totals — drain queued deltas first so # listings (sidebar, /resume, dashboards) show exact counters. @@ -12186,9 +5403,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) LIMIT ? OFFSET ? """ params.extend([limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute(query, params) - rows = cursor.fetchall() + rows = self._read_all(query, params) sessions = [] for row in rows: s = self._session_row_dict(row) @@ -12227,9 +5442,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) {pinned_where} ORDER BY s.started_at DESC """ - with self._read_ctx() as conn: - pinned_cursor = conn.execute(pinned_query, base_where_params) - pinned_rows = pinned_cursor.fetchall() + pinned_rows = self._read_all(pinned_query, base_where_params) for row in pinned_rows: s = self._session_row_dict(row) if s["id"] in seen_ids: @@ -12343,8 +5556,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) GROUP BY session_id ) latest ON m.id = latest.max_id """ - with self._read_ctx() as conn: - rows = conn.execute(query, ids).fetchall() + rows = self._read_all(query, ids) for row in rows: statuses[row["session_id"]] = classify_session_status( role=row["role"], @@ -12363,1633 +5575,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # cannot collide with real user content. _CONTENT_JSON_PREFIX = "\x00json:" - @classmethod - def _encode_content(cls, content: Any) -> Any: - """Serialize structured (list/dict) message content for sqlite. - - sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None`` - to query parameters. Multimodal messages have ``content`` as a list of - parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which - raises ``ProgrammingError: Error binding parameter N: type 'list' is - not supported`` when bound directly. - - Returns the value unchanged when it's already a safe scalar, or a - sentinel-prefixed JSON string for lists/dicts. Paired with - :meth:`_decode_content` on read. - """ - if isinstance(content, str): - # Lone UTF-16 surrogates reach here inside tool results scraped - # from the web/social platforms (the same input that crashed the - # guardrail hasher). The proactive sanitizer upstream only cleans - # the *api_messages* copy, and the recovery sanitizer only runs - # after the API call itself raises — which it no longer does — so - # the canonical history keeps them and this write is where they - # land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is - # abandoned, and the session silently stops persisting for the - # rest of its life. Scrub so persistence never fails. - return _sanitize_surrogates(content) - if content is None or isinstance(content, (bytes, int, float)): - return content - try: - # json.dumps defaults to ensure_ascii=True, which escapes any - # surrogate as \udXXX — already safe to bind. - return cls._CONTENT_JSON_PREFIX + json.dumps(content) - except (TypeError, ValueError): - # Last-resort fallback: stringify so persistence never fails. - return _sanitize_surrogates(str(content)) - - @classmethod - def _decode_content(cls, content: Any) -> Any: - """Reverse :meth:`_encode_content`; returns scalars unchanged.""" - if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX): - try: - return json.loads(content[len(cls._CONTENT_JSON_PREFIX):]) - except (json.JSONDecodeError, TypeError): - logger.warning( - "Failed to decode JSON-encoded message content; " - "returning raw string" - ) - return content - return content - - @staticmethod - def _encode_display_metadata(display_metadata: Any) -> Optional[str]: - """Serialize ``display_metadata`` for its TEXT column without double-encoding. - - Import/replace paths can hand us an already-serialized JSON string (the - same hazard ``tool_calls`` guards against above). ``json.dumps`` on that - string would store a quoted JSON string, and the single ``json.loads`` - on read then yields a ``str`` instead of a dict. - """ - if not display_metadata: - return None - if isinstance(display_metadata, str): - try: - parsed = json.loads(display_metadata) - except (json.JSONDecodeError, TypeError): - logger.warning("Ignoring non-JSON display metadata on write") - return None - if not isinstance(parsed, dict): - logger.warning("Ignoring non-object display metadata on write") - return None - return json.dumps(parsed) - if isinstance(display_metadata, dict): - return json.dumps(display_metadata) - logger.warning( - "Ignoring unexpected display metadata type on write: %s", - type(display_metadata).__name__, - ) - return None - - def _check_transcript_write_guards( - self, - conn, - session_id: str, - compression_lock_holder: Optional[str], - turn_lease_holder: Optional[str] = None, - turn_lease_ttl_seconds: float = 300.0, - reject_active_turn_lease: bool = False, - reject_active_compression_lock: bool = False, - allow_closed_compression_parent: bool = False, - ) -> None: - """Transcript-write admission checks, run INSIDE the write txn. - - Shared by :meth:`append_message` and :meth:`append_messages_batch` so - the two writers can never diverge on these correctness invariants - (this guard has already needed targeted fixes — see the #74478 patience - note below). User-initiated transcript mutations may opt in to rejecting - an active unowned turn lease in that same transaction. - """ - # NOTE (#75316 redesign): appends do NOT check compression_locks. - # The lock's job is to stop two COMPRESSIONS colliding, not to fence - # ordinary transcript writes. Concurrent appends during a compression - # are safe by construction: archive_and_compact() commits against a - # watermark captured at compression start and clones every row that - # arrived after it back into the live transcript, in the same write - # transaction. Blocking appends here was the root cause of a whole - # symptom family — turns dying as session_persistence_failed while a - # slow provider summary held the lease (#74568, #77386), including - # stale locks from dead PIDs blocking writes for the full TTL. - # Destructive user mutations are different: a compressor that already - # captured its watermark can otherwise publish the pre-rewind snapshot - # after the mutation and resurrect the removed turn. Keep that narrow - # fence opt-in so ordinary appends retain the watermark behavior. - if reject_active_compression_lock: - active_lock = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if active_lock is not None: - current_holder = active_lock["holder"] - if ( - float(active_lock["expires_at"]) <= time.time() - or _compression_lock_holder_process_is_dead(current_holder) - ): - conn.execute( - "DELETE FROM compression_locks " - "WHERE session_id = ? AND holder = ?", - (session_id, current_holder), - ) - elif current_holder != compression_lock_holder: - raise SessionCompressionInProgressError( - f"Session {session_id!r} is being compressed by another writer" - ) - if turn_lease_holder or reject_active_turn_lease: - conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) - lease = conn.execute( - "SELECT holder, expires_at FROM session_turn_leases " - "WHERE conversation_id = ?", - (conversation_id,), - ).fetchone() - now = time.time() - if turn_lease_holder: - if lease is None or lease["holder"] != turn_lease_holder: - raise SessionTurnLeaseLostError( - f"Session turn lease lost; refusing transcript write " - f"for {session_id!r}" - ) - if float(lease["expires_at"]) <= now: - # Expiry makes the row reclaimable; it does not prove that a - # takeover occurred. BEGIN IMMEDIATE serializes this renewal - # with acquisition, so a still-matching owner can recover from - # a starved refresher without weakening the foreign-holder fence. - conn.execute( - "UPDATE session_turn_leases SET expires_at = ? " - "WHERE conversation_id = ? AND holder = ?", - ( - now + max(0.1, float(turn_lease_ttl_seconds)), - conversation_id, - turn_lease_holder, - ), - ) - elif lease is not None: - current_holder = lease["holder"] - if ( - float(lease["expires_at"]) <= now - or _compression_lock_holder_process_is_dead(current_holder) - ): - # Match acquisition semantics: an expired or provably dead - # owner is reclaimable. Deleting it inside this BEGIN IMMEDIATE - # transaction also fences a stale late flush after the mutation. - conn.execute( - "DELETE FROM session_turn_leases " - "WHERE conversation_id = ? AND holder = ?", - (conversation_id, current_holder), - ) - else: - raise SessionTurnLeaseLostError( - f"Session has an active turn lease; refusing transcript " - f"mutation for {session_id!r}" - ) - session = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - session is not None - and session["ended_at"] is not None - and session["end_reason"] == "compression" - and not allow_closed_compression_parent - ): - raise CompressionSessionClosedError(session_id) - - @staticmethod - def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]: - """Decode a ``display_metadata`` column into the dict every reader expects. - - Every message read path must go through this. Returning the raw TEXT - instead reaches the desktop as a string, where ``'task_count' in meta`` - throws and fails the whole resume. Rows written before the encode guard - landed are double-encoded, so unwrap a second layer when we find one. - """ - if raw is None: - return None - try: - meta = json.loads(raw) if isinstance(raw, str) else raw - if isinstance(meta, str): - meta = json.loads(meta) - except (json.JSONDecodeError, TypeError): - logger.warning("Ignoring invalid display metadata on message row") - return None - if not isinstance(meta, dict): - logger.warning("Ignoring non-object display metadata on message row") - return None - return meta - - @staticmethod - def _reasoning_json_text(value: Any) -> Optional[str]: - """Serialize a structured reasoning field for its TEXT column. - - ``reasoning_details`` / ``codex_reasoning_items`` / ``codex_message_items`` - arrive as list/dict structures from the live runtime, but callers that - round-trip stored rows — ``get_messages`` straight into - ``replace_messages``, e.g. the POST /api/sessions/{id}/fork handler — - hand back the raw TEXT these columns already hold, because - ``get_messages`` only deserializes ``content`` and ``tool_calls``. - Re-dumping that TEXT double-encodes it, and the forked session's next - ``get_messages_as_conversation`` json.loads then yields the inner - string instead of the original list, so every reasoning-replay consumer - (all of which check ``isinstance(..., list)``) silently drops it. - Strings are therefore stored as-is; structures are dumped. - """ - if not value: - return None - if isinstance(value, str): - return value - return json.dumps(value) - - def append_message( - self, - session_id: str, - role: str, - content: str = None, - tool_name: str = None, - tool_calls: Any = None, - tool_call_id: str = None, - token_count: int = None, - finish_reason: str = None, - reasoning: str = None, - reasoning_content: str = None, - reasoning_details: Any = None, - codex_reasoning_items: Any = None, - codex_message_items: Any = None, - platform_message_id: str = None, - observed: bool = False, - effect_disposition: Optional[str] = None, - _compressed_summary: bool = False, - timestamp: Any = None, - api_content: Optional[str] = None, - display_kind: Optional[str] = None, - display_metadata: Optional[Dict[str, Any]] = None, - compression_lock_holder: Optional[str] = None, - turn_lease_holder: Optional[str] = None, - turn_lease_ttl_seconds: float = 300.0, - ) -> int: - """ - Append a message to a session. Returns the message row ID. - - Also increments the session's message_count (and tool_call_count - if role is 'tool' or tool_calls is present). - - ``platform_message_id`` is the external messaging platform's own - message ID (e.g. Telegram update_id, Yuanbao msg_id). It is - independent of the SQLite autoincrement primary key and is used by - platform-specific flows like yuanbao's recall guard to redact a - message by its platform-side identifier. - - ``api_content`` is the exact content string sent to the API for this - message when it differs from ``content`` (ephemeral memory/plugin - injections, persist overrides). It is a byte-fidelity sidecar for - prompt-cache-stable replay — stored as sent, except lone surrogates - (which sqlite3 cannot bind and which the conversation loop scrubs - from every outgoing payload anyway, so the scrubbed form IS the - wire bytes). - """ - # Display metadata is presentation-only and never changes the model - # context role/content replayed to providers. - display_metadata_json = self._encode_display_metadata(display_metadata) - # Serialize structured fields to JSON before entering the write txn - reasoning_details_json = self._reasoning_json_text(reasoning_details) - codex_items_json = self._reasoning_json_text(codex_reasoning_items) - codex_message_items_json = self._reasoning_json_text(codex_message_items) - # tool_calls may arrive as a Python list (from the live agent) or - # as a JSON string (from import/export). Parse first to avoid - # double-encoding. - if isinstance(tool_calls, str): - try: - tool_calls = json.loads(tool_calls) - except (json.JSONDecodeError, TypeError): - tool_calls = [] - tool_calls_json = json.dumps(tool_calls) if tool_calls else None - # Multimodal content (list of parts) must be JSON-encoded: sqlite3 - # cannot bind list/dict parameters directly. - stored_content = self._encode_content(content) - - message_timestamp = time.time() - if timestamp is not None: - try: - if hasattr(timestamp, "timestamp"): - message_timestamp = float(timestamp.timestamp()) - else: - message_timestamp = float(timestamp) - except (TypeError, ValueError): - logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp) - - # Pre-compute tool call count - num_tool_calls = 0 - if tool_calls is not None: - num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1 - - def _do(conn): - self._check_transcript_write_guards( - conn, - session_id, - compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - cursor = conn.execute( - """INSERT INTO messages (session_id, role, content, tool_call_id, - tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, - reasoning, reasoning_content, reasoning_details, codex_reasoning_items, - codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - session_id, - role, - stored_content, - tool_call_id, - tool_calls_json, - _scrub_surrogates(tool_name), - effect_disposition, - message_timestamp, - token_count, - finish_reason, - _scrub_surrogates(reasoning), - _scrub_surrogates(reasoning_content), - reasoning_details_json, - codex_items_json, - codex_message_items_json, - platform_message_id, - 1 if observed else 0, - 1 if _compressed_summary else 0, - 1, - _scrub_surrogates(api_content) if isinstance(api_content, str) else None, - _scrub_surrogates(display_kind) if isinstance(display_kind, str) else None, - display_metadata_json, - ), - ) - msg_id = cursor.lastrowid - - # Update counters - if num_tool_calls > 0: - conn.execute( - """UPDATE sessions SET message_count = message_count + 1, - tool_call_count = tool_call_count + ? WHERE id = ?""", - (num_tool_calls, session_id), - ) - else: - conn.execute( - "UPDATE sessions SET message_count = message_count + 1 WHERE id = ?", - (session_id,), - ) - return msg_id - - # Transcript append is THE critical write: its failure aborts the - # user's turn (session_persistence_failed). Use the long patience so - # a sibling process legitimately holding the write lock for seconds - # (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge - # process's FTS optimize) can't destroy a healthy turn (#74478). - return self._execute_write( - _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S - ) - - def append_messages_batch( - self, - session_id: str, - messages: List[Dict[str, Any]], - compression_lock_holder: Optional[str] = None, - turn_lease_holder: Optional[str] = None, - chunk_rows: Optional[int] = None, - turn_lease_ttl_seconds: float = 300.0, - ) -> int: - """Append multiple messages atomically in ONE write transaction. - - ``messages`` is a list of dicts in the same shape - :meth:`_insert_message_rows` already consumes for replace/compact/ - import (role, content, tool_name, tool_calls, tool_call_id, - finish_reason, reasoning*, codex_*, timestamp, api_content, - display_kind, display_metadata, ...). Reusing that helper keeps ONE - row-serialization path for every multi-row writer. - - A turn-boundary flush writes the whole turn (user + assistant + tool - rows, typically 3-8 messages) as one BEGIN IMMEDIATE / commit pair - instead of one transaction (and, off WAL, one fsync) per row. - - Atomicity contract: all rows land or none do (the caller re-flushes - unstamped messages on the next attempt). The same admission guards - as :meth:`append_message` run once for the batch — same session, - same instant. - - ``chunk_rows`` bounds the transaction size for LARGE copies (branch - seeds can be thousands of rows; measured: 10k rows ≈ 2.4s inside one - BEGIN IMMEDIATE because the FTS triggers run per row, which would - monopolize the write lock and starve concurrent writers). When set, - the batch commits in chunks of at most that many rows — same - recovery semantics as the old per-row loops (a mid-copy failure - leaves a partial seed), just with bounded lock holds. A turn flush - never needs it. Returns the inserted row count. - """ - if not messages: - return 0 - - if chunk_rows is not None and len(messages) > chunk_rows: - inserted_total = 0 - for start in range(0, len(messages), chunk_rows): - inserted_total += self.append_messages_batch( - session_id, - messages[start:start + chunk_rows], - compression_lock_holder=compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - return inserted_total - - def _do(conn): - self._check_transcript_write_guards( - conn, - session_id, - compression_lock_holder, - turn_lease_holder=turn_lease_holder, - turn_lease_ttl_seconds=turn_lease_ttl_seconds, - ) - from agent.transcript_repair import resolve_and_repair_transcript_batch - - inserted_rows = resolve_and_repair_transcript_batch( - conn, - session_id, - messages, - encode_content_fn=self._encode_content, - decode_content_fn=self._decode_content, - ) - inserted = 0 - tool_calls_total = 0 - if inserted_rows: - inserted, tool_calls_total = self._insert_message_rows( - conn, session_id, inserted_rows - ) - - # One aggregated counter update for the newly inserted rows. - if tool_calls_total > 0: - conn.execute( - """UPDATE sessions SET message_count = message_count + ?, - tool_call_count = tool_call_count + ? WHERE id = ?""", - (inserted, tool_calls_total, session_id), - ) - elif inserted > 0: - conn.execute( - "UPDATE sessions SET message_count = message_count + ? WHERE id = ?", - (inserted, session_id), - ) - return inserted - - # Same criticality as append_message: this IS the turn's transcript. - return self._execute_write( - _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S - ) - - def set_latest_matching_message_display_kind( - self, session_id: str, *, role: str, content: str, display_kind: str, - display_metadata: Optional[Dict[str, Any]] = None, - ) -> bool: - """Stamp presentation metadata on this turn's freshly persisted row. - - The model still receives ``role`` and ``content`` unchanged. Gateway and - CLI synthetic inputs call this immediately after their serial turn has - flushed, preserving producer provenance without classifying by content - during transcript rendering. - """ - if not session_id or not content or not display_kind: - return False - - def _do(conn): - row = conn.execute( - "SELECT id FROM messages WHERE session_id = ? AND role = ? " - "AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1", - (session_id, role, self._encode_content(content)), - ).fetchone() - if row is None: - return False - conn.execute( - "UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?", - ( - _scrub_surrogates(display_kind), - self._encode_display_metadata(display_metadata), - row[0], - ), - ) - return True - - return bool(self._execute_write(_do)) #: Key under which message reactions live inside ``display_metadata``. #: Reactions share the existing per-message JSON column rather than a side #: table so they survive rewind/compaction row rewrites with the row itself. REACTIONS_METADATA_KEY = "reactions" - def set_message_reaction( - self, - session_id: str, - message_row_id: int, - emoji: Optional[str], - *, - author: str = "user", - ) -> Optional[List[Dict[str, Any]]]: - """Set (or with ``emoji=None`` clear) *author*'s reaction on one message. - - iOS Tapback semantics: one reaction per author per message. Re-sending - the same emoji clears it, a different emoji replaces it. Returns the - message's full reaction list after the write, or ``None`` when the row - doesn't exist or isn't part of *session_id*. - """ - if not session_id or message_row_id is None: - return None - - def _do(conn): - row = conn.execute( - "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", - (message_row_id, session_id), - ).fetchone() - if row is None: - return None - - meta = self._decode_display_metadata(row[0]) or {} - existing = meta.get(self.REACTIONS_METADATA_KEY) - reactions = [ - r - for r in (existing if isinstance(existing, list) else []) - if isinstance(r, dict) and r.get("author") != author - ] - previous = next( - ( - r - for r in (existing if isinstance(existing, list) else []) - if isinstance(r, dict) and r.get("author") == author - ), - None, - ) - # Tapping the live reaction again retracts it. - toggling_off = ( - emoji is not None and previous is not None and previous.get("emoji") == emoji - ) - if emoji and not toggling_off: - reactions.append( - {"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()} - ) - - if reactions: - meta[self.REACTIONS_METADATA_KEY] = reactions - else: - meta.pop(self.REACTIONS_METADATA_KEY, None) - - conn.execute( - "UPDATE messages SET display_metadata = ? WHERE id = ?", - (self._encode_display_metadata(meta) if meta else None, message_row_id), - ) - return reactions - - return self._execute_write(_do) - - def get_message_reactions( - self, session_id: str, message_row_id: int - ) -> List[Dict[str, Any]]: - """Return the reaction list persisted on one message row (never ``None``).""" - if not session_id or message_row_id is None: - return [] - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", - (message_row_id, session_id), - ).fetchone() - - if row is None: - return [] - - meta = self._decode_display_metadata(row[0]) or {} - reactions = meta.get(self.REACTIONS_METADATA_KEY) - - return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else [] - - def take_unseen_reactions( - self, session_id: str, *, author: str = "user" - ) -> List[Dict[str, Any]]: - """Return *author*'s not-yet-surfaced reactions and mark them seen. - - Powers the cache-safe model-context path: reactions are announced on the - NEXT user turn (never by rewriting the message that was reacted to), and - the ``seen`` stamp guarantees each one is announced exactly once. - """ - if not session_id: - return [] - - def _do(conn): - rows = conn.execute( - "SELECT id, role, content, display_metadata FROM messages " - "WHERE session_id = ? AND active = 1 AND display_metadata IS NOT NULL " - "ORDER BY id", - (session_id,), - ).fetchall() - - pending = [] - for row in rows: - meta = self._decode_display_metadata(row["display_metadata"]) - if not meta: - continue - reactions = meta.get(self.REACTIONS_METADATA_KEY) - if not isinstance(reactions, list): - continue - - changed = False - for reaction in reactions: - if ( - not isinstance(reaction, dict) - or reaction.get("author") != author - or reaction.get("seen") - ): - continue - reaction["seen"] = True - changed = True - content = self._decode_content(row["content"]) - pending.append( - { - "row_id": row["id"], - "role": row["role"], - "emoji": reaction.get("emoji") or "", - "text": content if isinstance(content, str) else "", - } - ) - - if changed: - conn.execute( - "UPDATE messages SET display_metadata = ? WHERE id = ?", - (self._encode_display_metadata(meta), row["id"]), - ) - - return pending - - return self._execute_write(_do) or [] - - def latest_message_row_id( - self, session_id: str, *, role: str = "user", offset: int = 0, require_text: bool = True - ) -> Optional[int]: - """Row id of the most recent active message with *role*, or ``None``. - - Two callers, same need — "the message I mean, without an id": the agent - defaulting to the turn that triggered it, and the desktop reacting to a - live message that hasn't round-tripped through a resume yet. - ``offset`` steps to earlier turns (1 = the one before the latest) so a - reaction can land retroactively — "two messages ago" is how the caller - thinks about it. - - ``require_text`` (default) skips rows with no plain-text content — - tool-call-only assistant turns and attachment stubs don't render as - bubbles, so "the latest message" as a HUMAN means it must never - resolve to one (a reaction landing on an invisible row looks dropped, - and its annotation quotes an empty string). - """ - if not session_id or role not in {"user", "assistant"} or offset < 0: - return None - - text_filter = ( - "AND content IS NOT NULL AND TRIM(content) != '' " if require_text else "" - ) - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT id FROM messages WHERE session_id = ? AND role = ? " - f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?", - (session_id, role, int(offset)), - ).fetchone() - - return row[0] if row else None - - def latest_user_message_row_id(self, session_id: str) -> Optional[int]: - """Row id of the most recent active user message, or ``None``. - - The agent's default reaction target: "the message that triggered me", - so the model never has to thread row ids through a tool call (mirrors - the photon adapter's ``_record_last_inbound``). - """ - return self.latest_message_row_id(session_id, role="user") - - def get_message_role(self, session_id: str, row_id: int) -> Optional[str]: - """Role of the active message at *row_id* in *session_id*, or ``None``. - - Lets a reaction event carry the target's role so a renderer can match - a live message that doesn't know its durable row id yet. - """ - if not session_id: - return None - - with self._read_ctx() as conn: - row = conn.execute( - "SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1", - (int(row_id), session_id), - ).fetchone() - - return row[0] if row else None - - def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]: - """Insert *messages* as fresh active rows for *session_id*. - - Shared by :meth:`replace_messages` (delete-then-insert) and - :meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the - caller's write transaction (takes the live ``conn``). Returns - ``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters - — the caller owns that, since the two flows reconcile counts differently. - """ - now_ts = time.time() - inserted = 0 - tool_calls_total = 0 - for msg in messages: - role = msg.get("role", "unknown") - tool_calls = msg.get("tool_calls") - message_timestamp = now_ts - if msg.get("timestamp") is not None: - try: - ts_value = msg.get("timestamp") - if hasattr(ts_value, "timestamp"): - message_timestamp = float(ts_value.timestamp()) - else: - message_timestamp = float(ts_value) - except (TypeError, ValueError): - logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp")) - reasoning_details = msg.get("reasoning_details") if role == "assistant" else None - codex_reasoning_items = ( - msg.get("codex_reasoning_items") if role == "assistant" else None - ) - codex_message_items = ( - msg.get("codex_message_items") if role == "assistant" else None - ) - reasoning_details_json = self._reasoning_json_text(reasoning_details) - codex_items_json = self._reasoning_json_text(codex_reasoning_items) - codex_message_items_json = self._reasoning_json_text(codex_message_items) - # tool_calls may arrive as a Python list (from the live agent) - # or as a JSON string (from import_sessions / export_session, - # which store it as TEXT). json.dumps on an already-serialized - # string double-encodes it, so parse first. - if isinstance(tool_calls, str): - try: - tool_calls = json.loads(tool_calls) - except (json.JSONDecodeError, TypeError): - tool_calls = [] - tool_calls_json = json.dumps(tool_calls) if tool_calls else None - # Accept either `platform_message_id` (new explicit name) or - # `message_id` (yuanbao's existing convention on message dicts). - platform_msg_id = ( - msg.get("platform_message_id") or msg.get("message_id") - ) - - api_content = msg.get("api_content") - - cur = conn.execute( - """INSERT INTO messages (session_id, role, content, tool_call_id, - tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, - reasoning, reasoning_content, reasoning_details, codex_reasoning_items, - codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", - ( - session_id, - role, - self._encode_content(msg.get("content")), - msg.get("tool_call_id"), - tool_calls_json, - _scrub_surrogates(msg.get("tool_name")), - msg.get("effect_disposition"), - message_timestamp, - msg.get("token_count"), - msg.get("finish_reason"), - _scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None, - _scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None, - reasoning_details_json, - codex_items_json, - codex_message_items_json, - platform_msg_id, - 1 if msg.get("observed") else 0, - 1 if msg.get("_compressed_summary") else 0, - 1, - _scrub_surrogates(api_content) if isinstance(api_content, str) else None, - _scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None, - self._encode_display_metadata(msg.get("display_metadata")), - ), - ) - if isinstance(msg, dict) and cur.lastrowid is not None: - msg["_row_id"] = cur.lastrowid - inserted += 1 - if tool_calls is not None: - tool_calls_total += ( - len(tool_calls) if isinstance(tool_calls, list) else 1 - ) - now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6) - return inserted, tool_calls_total - - def replace_messages( - self, - session_id: str, - messages: List[Dict[str, Any]], - active_only: bool = False, - archive_dropped: bool = False, - reject_active_turn_lease: bool = False, - ) -> None: - """Atomically replace the stored messages for a session. - - Used by transcript-rewrite flows such as /retry, /undo, and /compress. - The delete + reinsert sequence must commit as one transaction so a - mid-rewrite failure does not leave SQLite with a partial transcript. - - DESTRUCTIVE by default: every row for the session is DELETEd (and drops - out of the FTS index). For compaction that must preserve the - pre-compaction transcript under the same id, use - :meth:`archive_and_compact` instead. - - Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows, - leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1`` - turns that :meth:`archive_and_compact` keeps on disk for #38763 - durability, or rewind/undo rows) untouched. Callers that share a session - id with an agent already running in-place compaction must use this so a - full-history rewrite doesn't wipe the rows the agent deliberately - archived. ``message_count``/``tool_call_count`` then track the live set, - matching :meth:`archive_and_compact`. - - Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of - DELETEing them: the replaced turns stay on disk with ``active = 0``, - ``compacted = 0`` — the same "the user took it back" marking - :meth:`rewind_to_message` applies — and stay readable via - :meth:`get_messages` with ``include_inactive=True``. This is the mode a - rewind/edit/regenerate must use: those flows overwrite a transcript the - user may not have meant to drop, and a plain DELETE also evicts the rows - from the FTS index, leaving nothing to recover from (#82756). It implies - active-only handling — already-archived rows are never touched — so - ``active_only`` is redundant with it. The rewritten set is inserted as - fresh active rows exactly as in the destructive path, so the live view - is identical either way; only the durability of the dropped turns - differs. - - Pass ``reject_active_turn_lease=True`` for user-initiated rewrites that - do not already own the cross-process turn lease. The lease check and - transcript mutation then share one write transaction, so a second - process cannot archive or replace a turn that is still being produced. - """ - - active_clause = " AND active = 1" if active_only else "" - - def _do(conn): - if reject_active_turn_lease: - self._check_transcript_write_guards( - conn, - session_id, - None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - else: - session = conn.execute( - "SELECT ended_at, end_reason FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if ( - session is not None - and session["ended_at"] is not None - and session["end_reason"] == "compression" - ): - raise CompressionSessionClosedError(session_id) - if archive_dropped: - # Content-preserving UPDATE: the rows keep their FTS entries - # (the messages_fts triggers fire on INSERT / DELETE / UPDATE - # of content columns, not on `active`), so the replaced turns - # stay readable via get_messages(include_inactive=True) and - # searchable with include_inactive=True after the rewrite. - conn.execute( - "UPDATE messages SET active = 0 " - "WHERE session_id = ? AND active = 1", - (session_id,), - ) - else: - conn.execute( - f"DELETE FROM messages WHERE session_id = ?{active_clause}", - (session_id,), - ) - conn.execute( - "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", - (session_id,), - ) - total_messages, total_tool_calls = self._insert_message_rows( - conn, session_id, messages - ) - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (total_messages, total_tool_calls, session_id), - ) - - self._execute_write(_do) - - def has_archived_messages(self, session_id: str) -> bool: - """Return True if the session has any soft-archived (``active = 0``) rows. - - Cheap existence probe — does not load rows. NOTE: production rewrite - paths no longer branch on this (they pass ``active_only=True`` - unconditionally — a probe can fail open or race a concurrent - ``archive_and_compact``, #80216); kept for tests and diagnostics. - """ - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1", - (session_id,), - ) - return cursor.fetchone() is not None - - def get_active_message_watermark(self, session_id: str) -> int: - """MAX(id) of the session's active rows — the compression watermark. - - Captured at compression START (before the slow provider summary call). - Every active row with ``id > watermark`` at commit time arrived - concurrently and must survive the compaction verbatim. Returns 0 for - an empty/unknown session. - """ - if not session_id: - return 0 - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COALESCE(MAX(id), 0) FROM messages " - "WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchone() - return int(row[0]) if row else 0 - - def archive_and_compact( - self, - session_id: str, - compacted_messages: List[Dict[str, Any]], - model_config_patch: Optional[Dict[str, Any]] = None, - watermark: Optional[int] = None, - lock_holder: Optional[str] = None, - tail_count: int = 0, - ) -> int: - """Non-destructive in-place compaction for a single durable session id. - - Soft-archives the active messages (``active = 0``) and inserts - *compacted_messages* as fresh active rows — atomically, in one write - transaction. The conversation keeps ONE session id for life (#38763) - WITHOUT destroying history: - - - The live-context load (:meth:`get_messages_as_conversation`, - :meth:`get_messages`) filters ``active = 1`` by default, so the model - reloads ONLY the compacted set. - - The archived pre-compaction turns stay on disk (active=0) and stay - DISCOVERABLE: they are marked compacted=1, and search_messages() - includes compacted=1 rows by default — so session_search still finds - them, unlike rewind/undo rows (active=0, compacted=0) which stay - hidden. They remain in the FTS index (the messages_fts* triggers - index on INSERT / drop on DELETE and don't key on active/compacted; - flipping to active=0 is a content-preserving UPDATE) and are - recoverable via get_messages(..., include_inactive=True). - - Concurrent-append safety (#75316): when *watermark* is provided (the - value of :meth:`get_active_message_watermark` captured at compression - START), rows that arrived during the slow provider summary call - (``id > watermark``) are NOT summarized away. They are re-sequenced - after the compacted set by a pure-SQL column clone (every column - except ``id`` — content, api_content, platform_message_id, token - counts, reasoning sidecars all survive byte-exact, and the FTS - triggers index the clones naturally), and the originals are archived. - NOTE: re-sequencing assigns the tail rows fresh ids; consumers that - reference durable row ids re-resolve by content (see 3e8ab0610). - ``watermark=None`` preserves the historical archive-everything - behavior. - - Commit-fence safety: when *lock_holder* is provided, the commit - verifies INSIDE the transaction that the compression lock is still - held by that holder and unexpired — a compression whose lease was - reclaimed (crash cleanup, TTL expiry, competing writer) fails the - commit instead of clobbering the winner's transcript. - - *tail_count* (default 0) names how many of the LAST rows of - *compacted_messages* are the verbatim carried-forward tail the - compressor protected rather than summarized (#86366). Those rows' - ORIGINALS — which this call archives as a side effect of the blanket - soft-archive — are superseded byte-identical duplicates, not - "summarized away" content, so they are stamped rewind-style - (``active=0, compacted=0``, hidden from search_messages) instead of - ``compacted=1``. Without this the tail originals satisfy the recall - filter alongside their live clones and session_search returns every - carried-forward message once per compaction. Callers that cannot know - their tail shape keep the historical archive-everything behavior. - - ``message_count`` is set to the ACTIVE count after commit, matching - what the live load returns. ``model_config_patch`` is merged into the - session's JSON config in the same transaction; a ``None`` value - removes that key. Returns the new active count. - """ - - def _do(conn): - if lock_holder is not None: - lock_row = conn.execute( - "SELECT holder, expires_at FROM compression_locks " - "WHERE session_id = ?", - (session_id,), - ).fetchone() - if ( - lock_row is None - or lock_row["holder"] != lock_holder - or float(lock_row["expires_at"]) <= time.time() - ): - raise SessionCompressionInProgressError( - f"Compression lease for {session_id!r} lost before " - "commit; refusing to publish a stale compaction" - ) - - patched_model_config = None - if model_config_patch is not None: - # on_missing="raise": a prune/compaction must not commit - # against a vanished session row (the compressor's caller - # converts the raised error into a safe keep-the-original - # no-op), unlike the flag setters which tolerate missing rows. - patched_model_config = self._merge_model_config_json( - conn, session_id, model_config_patch, on_missing="raise" - ) - - # Concurrent tail: active rows that arrived after the watermark. - # Snapshot their ids and tool_calls now — the clone below needs a - # stable id list, and the tool-call count keeps sessions.* honest. - tail_ids: list[int] = [] - tail_tool_calls = 0 - if watermark is not None: - for row in conn.execute( - "SELECT id, tool_calls FROM messages " - "WHERE session_id = ? AND active = 1 AND id > ? " - "ORDER BY id", - (session_id, int(watermark)), - ).fetchall(): - tail_ids.append(int(row["id"])) - raw = row["tool_calls"] - if raw: - try: - parsed = json.loads(raw) if isinstance(raw, str) else raw - tail_tool_calls += len(parsed) if isinstance(parsed, list) else 0 - except (TypeError, ValueError): - pass - - # Soft-archive the live turns: active=0 hides them from the live - # context load, compacted=1 marks them as "summarized away" (vs - # rewind/undo's active=0+compacted=0, which means "user took it - # back"). search_messages includes compacted=1 rows by default so - # the pre-compaction transcript stays discoverable; live-context - # loads (active=1 only) still exclude them. Tail originals whose - # verbatim clones ride inside *compacted_messages* (tail_count) - # are superseded duplicates instead (#86366): they get the - # rewind-style flags so they stop matching the recall filter. - # Rewind-target ids: the originals of the carried-forward tail - # rows (tail_count), captured BEFORE any flag flips. Named apart - # from the watermark `tail_ids` below on purpose — the two are - # different sets (#86366): rewind targets sit AT/BELOW the - # watermark (the compressor only saw rows up to it), while - # `tail_ids` are concurrent appends ABOVE it. Without the bound, - # a concurrent append would steal a LIMIT slot and leave a real - # carried-forward original stamped compacted=1. - rewind_tail_ids: Optional[list[int]] = None - if tail_count > 0: - if watermark is not None: - tail_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 AND id <= ? " - "ORDER BY id DESC LIMIT ?", - (session_id, int(watermark), int(tail_count)), - ).fetchall() - else: - tail_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id DESC LIMIT ?", - (session_id, int(tail_count)), - ).fetchall() - rewind_tail_ids = [int(row["id"]) for row in tail_rows] - - # The watermark clone below re-inserts `tail_ids` rows byte-exact - # as live rows — their originals are the SAME superseded-duplicate - # class as the carried-forward tail (#86366), so they take the - # rewind flags too instead of double-matching the recall filter. - rewind_ids = [*(rewind_tail_ids or []), *tail_ids] - - if rewind_ids: - placeholders = ",".join("?" for _ in rewind_ids) - conn.execute( - "UPDATE messages SET active = 0, compacted = 0 " - f"WHERE session_id = ? AND id IN ({placeholders})", - [session_id, *rewind_ids], - ) - conn.execute( - "UPDATE messages SET active = 0, compacted = 1 " - "WHERE session_id = ? AND active = 1 " - f"AND id NOT IN ({placeholders})", - [session_id, *rewind_ids], - ) - else: - conn.execute( - "UPDATE messages SET active = 0, compacted = 1 " - "WHERE session_id = ? AND active = 1", - (session_id,), - ) - inserted, tool_calls_total = self._insert_message_rows( - conn, session_id, compacted_messages - ) - - if tail_ids: - # Re-sequence the concurrent tail after the compacted set via - # a pure-SQL column clone: no decode/re-encode round trip, no - # field drift — new id, active=1, compacted=0, all else exact. - placeholders = ",".join("?" for _ in tail_ids) - clone_cols = [ - c for c in self._message_column_names(conn) - if c not in ("id", "active", "compacted") - ] - col_list = ", ".join(clone_cols) - conn.execute( - f"INSERT INTO messages ({col_list}, active, compacted) " - f"SELECT {col_list}, 1, 0 FROM messages " - f"WHERE id IN ({placeholders}) ORDER BY id", - tail_ids, - ) - inserted += len(tail_ids) - tool_calls_total += tail_tool_calls - - # message_count / tool_call_count reflect the LIVE (active) set — - # the archived rows are still on disk but not part of the live count. - if model_config_patch is None: - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", - (inserted, tool_calls_total, session_id), - ) - else: - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ?, " - "model_config = ? WHERE id = ?", - (inserted, tool_calls_total, patched_model_config, session_id), - ) - return inserted - - return self._execute_write(_do) - - def _message_column_names(self, conn) -> List[str]: - """Column names of the messages table, cached per-connection era.""" - cached = getattr(self, "_message_columns_cache", None) - if cached: - return cached - cols = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()] - self._message_columns_cache = cols - return cols - - def set_latest_user_api_content( - self, session_id: str, content: Any, api_content: str - ) -> int: - """Backfill the ``api_content`` sidecar onto the newest ACTIVE user row. - - In-place preflight compaction (:meth:`archive_and_compact`) inserts the - current turn's user row BEFORE the turn prologue composes the - prefetch/plugin sidecar, and the subsequent crash persist identity-skips - every compacted dict — without this backfill the stamped sidecar would - never land in the DB and any reload would replay clean content, - re-introducing the prompt-cache divergence the sidecar exists to close. - - The ``content`` match is a defensive guard: if the newest active user - row is not the message the caller stamped (racing rewrite, unexpected - tail shape), nothing is written. Returns the number of rows updated - (0 or 1). - """ - encoded = self._encode_content(content) - - def _do(conn): - cursor = conn.execute( - "UPDATE messages SET api_content = ? WHERE id = (" - "SELECT id FROM messages " - "WHERE session_id = ? AND role = 'user' AND active = 1 " - "ORDER BY id DESC LIMIT 1" - ") AND content IS ?", - (_scrub_surrogates(api_content), session_id, encoded), - ) - return cursor.rowcount - - return self._execute_write(_do) - - def _dedupe_display_generations(self, rows): - """Collapse compaction generations so each message appears once. - - Compaction epochs copy the protected tail into each new generation, so - one logical message can exist as several rows (identical - role/content/timestamp) with different ``active`` flags and ids. A - display read must surface each exactly once: prefer the live row, then - the newest generation. - - This is the ONE definition shared by every display projection — - :meth:`get_messages` (REST), :meth:`get_resume_conversations` and - :meth:`get_ancestor_display_prefix` (gateway resume), and - :meth:`get_messages_as_conversation` (warm-session payload) — so the - surfaces cannot disagree about the same transcript. *rows* must already - be ordered by ``id``; the returned list keeps that order. - """ - seen: Dict[Tuple[Any, ...], Any] = {} - for row in rows: - dedupe_content = row["content"] - if row["role"] == "user": - from agent.context_compressor import split_user_originated_turn - - candidate = { - "role": "user", - "content": self._decode_content(row["content"]), - "display_kind": row["display_kind"], - "display_metadata": self._decode_display_metadata( - row["display_metadata"] - ), - } - handoff, live_view = split_user_originated_turn(candidate) - if handoff is not None and live_view is not None: - dedupe_content = self._encode_content(live_view.get("content")) - # Tool fields participate in the dedupe key: compaction copies them - # verbatim, so identical tool messages across generations still - # collapse, while distinct tool calls that happen to share - # role/content/timestamp are never merged. - key = ( - row["role"], - dedupe_content, - row["timestamp"], - row["tool_call_id"], - row["tool_calls"], - row["tool_name"], - ) - cur = seen.get(key) - if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): - seen[key] = row - return sorted(seen.values(), key=lambda r: r["id"]) - - def get_messages( - self, - session_id: str, - include_inactive: bool = False, - include_compacted: bool = False, - limit: Optional[int] = None, - offset: int = 0, - latest: bool = False, - after_id: Optional[int] = None, - ) -> List[Dict[str, Any]]: - """Load messages for a session in insertion order. - - By default only active messages are returned. Pass - ``include_inactive=True`` to load soft-deleted rows (e.g. for - audit / debug views of rewound history). See - :meth:`rewind_to_message` for the soft-delete mechanic. - - Pass ``include_compacted=True`` to additionally load rows preserved - by in-place context compaction (``active=0, compacted=1``). Those are - durable display history, not soft-deleted rows — a user-visible - transcript read must not drop them, or earlier turns silently become - unreachable once the UI exhausts its active-only window. Soft-deleted - Undo/Rewind rows (``active=0, compacted=0``) stay excluded; use - ``include_inactive`` for those. - - Ordered by AUTOINCREMENT id (true insertion order) rather than - timestamp — see c03acca50 for the WSL2 clock-regression rationale. - - When ``limit`` is provided, returns at most ``limit`` messages - starting from ``offset`` (0-based, in insertion order). Enables - pagination for the API endpoint to avoid loading entire transcripts. - With ``latest=True``, the offset is measured back from the newest - message and the selected page is still returned in chronological - order. ``offset`` alone (without ``limit``) also pages — SQLite - requires a LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1`` - (unbounded). - - ``after_id`` enables keyset pagination (``id > after_id``): O(1) - page seeks on huge transcripts where OFFSET degrades to O(n) per - page. Ascending order only (incompatible with ``latest``/``offset``). - """ - if after_id is not None and (latest or offset): - raise ValueError("after_id is incompatible with latest/offset paging") - if after_id is not None and include_compacted: - raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)") - if include_inactive: - # Audit / debug reads: every row, including soft-deleted. - active_clause = "" - elif include_compacted: - # Display history: active rows plus rows preserved by in-place - # compaction (active=0, compacted=1), but never soft-deleted - # Undo/Rewind rows (active=0, compacted=0). - active_clause = " AND (active = 1 OR compacted = 1)" - else: - active_clause = " AND active = 1" - keyset_clause = " AND id > ?" if after_id is not None else "" - sql = ( - "SELECT * FROM messages WHERE session_id = ?" - f"{active_clause}{keyset_clause} ORDER BY id {'DESC' if latest else 'ASC'}" - ) - params: list = [session_id] - if after_id is not None: - params.append(after_id) - if include_compacted: - # Read the full display set (a session's rows are bounded; the - # UI-level 500-row cap lives in the endpoint, not here), dedupe - # generations, then apply paging. - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT * FROM messages WHERE session_id = ?" + active_clause - + " ORDER BY id ASC", - [session_id], - ) - all_rows = cursor.fetchall() - rows = self._dedupe_display_generations(all_rows) - if latest: - rows = rows[::-1] - rows = rows[offset:] - if limit is not None: - rows = rows[:limit] - if latest: - rows = rows[::-1] - else: - if limit is not None or offset: - # SQLite's OFFSET requires LIMIT; -1 means "no limit". - sql += " LIMIT ? OFFSET ?" - params.extend([-1 if limit is None else limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - rows = cursor.fetchall() - if latest: - rows.reverse() - result = [] - for row in rows: - msg = dict(row) - if msg.pop("_compressed_summary", 0): - msg["_compressed_summary"] = True - if "content" in msg: - msg["content"] = self._decode_content(msg["content"]) - if msg.get("tool_calls"): - try: - msg["tool_calls"] = json.loads(msg["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []") - msg["tool_calls"] = [] - if msg.get("display_metadata") is not None: - msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) - result.append(msg) - return result - - def find_pr_url_messages(self, session_ids: List[str]) -> List[Dict[str, Any]]: - """Tool results in these sessions that mention a GitHub PR url. - - A candidate scan, deliberately loose: it hands back every tool result - containing ``/pull/`` and leaves the caller to decide which ones make a - claim (see the desktop's PR recovery, which only accepts an output that - is a bare PR url — the signature of ``gh pr create``). Ordered - oldest-first per session so the caller can take the last match. - """ - found: List[Dict[str, Any]] = [] - ids = [s for s in session_ids if s] - for start in range(0, len(ids), 900): # SQLite's bound-variable ceiling. - chunk = ids[start : start + 900] - placeholders = ",".join("?" * len(chunk)) - with self._read_ctx() as conn: - rows = conn.execute( - f"""SELECT session_id, content FROM messages - WHERE session_id IN ({placeholders}) - AND role = 'tool' AND content LIKE '%/pull/%' - ORDER BY id ASC""", - chunk, - ).fetchall() - found.extend({"session_id": row[0], "content": row[1]} for row in rows) - return found - - def get_messages_around( - self, - session_id: str, - around_message_id: int, - window: int = 5, - ) -> Dict[str, Any]: - """Load a window of messages anchored on a specific message id. - - Returns a dict with: - - ``window``: up to ``window`` messages before the anchor, the anchor - itself, and up to ``window`` messages after, ordered by id ascending. - - ``messages_before``: count of messages strictly before the anchor - still in the session (== window unless we hit the start). - - ``messages_after``: count of messages strictly after the anchor - still in the session (== window unless we hit the end). - - Used by ``session_search`` for both the discovery shape (anchored on the - FTS5 match) and the scroll shape (anchored on any message id). The - ``messages_before`` / ``messages_after`` counts let the caller detect - session boundaries: when either is less than ``window``, the agent has - reached one end of the session. - - Returns an empty window when ``around_message_id`` is not a real id in - ``session_id`` — callers decide how to surface that. - """ - if window < 0: - window = 0 - with self._read_ctx() as conn: - # Confirm the anchor exists in this session. - anchor_exists = conn.execute( - "SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1", - (around_message_id, session_id), - ).fetchone() - if not anchor_exists: - return {"window": [], "messages_before": 0, "messages_after": 0} - - # Two queries: anchor + before (DESC, take window+1), and after - # (ASC, take window). Final order is id ASC. - before_rows = conn.execute( - "SELECT * FROM messages " - "WHERE session_id = ? AND id <= ? " - "ORDER BY id DESC LIMIT ?", - (session_id, around_message_id, window + 1), - ).fetchall() - after_rows = conn.execute( - "SELECT * FROM messages " - "WHERE session_id = ? AND id > ? " - "ORDER BY id ASC LIMIT ?", - (session_id, around_message_id, window), - ).fetchall() - - # before_rows is DESC; reverse so it's ASC, then concatenate after_rows. - rows = list(reversed(before_rows)) + list(after_rows) - result = [] - for row in rows: - msg = dict(row) - if "content" in msg: - msg["content"] = self._decode_content(msg["content"]) - if msg.get("tool_calls"): - try: - msg["tool_calls"] = json.loads(msg["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning( - "Failed to deserialize tool_calls in get_messages_around, falling back to []" - ) - msg["tool_calls"] = [] - if msg.get("display_metadata") is not None: - msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) - result.append(msg) - - # before_rows includes the anchor itself; subtract 1 for the count of - # messages strictly before the anchor in the returned slice. - messages_before = max(0, len(before_rows) - 1) - messages_after = len(after_rows) - return { - "window": result, - "messages_before": messages_before, - "messages_after": messages_after, - } - - def resolve_resume_session_id(self, session_id: str) -> str: - """Redirect a resume target to the descendant session that holds the messages. - - Context compression ends the current session and forks a new child session - (linked via ``parent_session_id``). The flush cursor is reset, so the - child is where new messages actually land — the parent ends up with - ``message_count = 0`` rows unless messages had already been flushed to - it before compression. See #15000. - - This helper walks ``parent_session_id`` forward from ``session_id`` and - returns the descendant in the chain that has the **most recent** messages. - Unlike the original logic, it does NOT short-circuit when the starting - session already has messages — a descendant that was created by - compression may hold the continuation content and should be preferred - by the WebUI and gateway for ``--resume`` and session loading. - - If no descendant (including the starting session) has any messages, - the original ``session_id`` is returned unchanged. - - The chain is always walked via the child whose ``started_at`` is - latest; that matches the single-chain shape that compression creates. - A depth cap (32) guards against accidental loops in malformed data. - """ - if not session_id: - return session_id - - # Follow the compression-continuation chain forward to the live tip - # FIRST. Auto-compression ends the current session and forks a - # continuation child, but a long-lived parent keeps its own flushed - # message rows — so the empty-head walk below never redirects it, and - # resuming the parent id reloads the pre-compression transcript while - # the turns generated *after* compression (and their responses) sit in - # the continuation. ``get_compression_tip`` is lineage-aware: it only - # follows children whose parent ended with ``end_reason='compression'`` - # (created after the parent was ended), so delegation / branch children - # never hijack the resume. This is the fix for the desktop "I came back - # and the reply isn't there" report on large sessions. - try: - tip = self.get_compression_tip(session_id) - except Exception: - tip = session_id - if tip and tip != session_id: - session_id = tip - - with self._read_ctx() as conn: - current = session_id - seen = {current} - best = None # tracks the last (deepest) node with messages - - for _ in range(32): - # Check if the current node has messages. - try: - row = conn.execute( - "SELECT 1 FROM messages WHERE session_id = ? LIMIT 1", - (current,), - ).fetchone() - except Exception: - return session_id - if row is not None: - best = current - - # Walk to the most-recently-started child — but skip explicit - # branch (`_branched_from`), delegate/subagent (`_delegate_from`), - # reset-continuation (`_reset_from` or the legacy same-key - # heuristic — a post-reset conversation must never be reached - # by resuming the parent the user reset away), and tool - # children. They also carry a ``parent_session_id`` yet - # are NOT compression continuations; following them would hijack - # the resume target to an unrelated session (e.g. a subagent - # run). This mirrors the child-exclusion in ``get_compression_tip``. - try: - child_row = conn.execute( - "SELECT id FROM sessions AS child " - "WHERE child.parent_session_id = ? " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL " - " AND json_extract(COALESCE(child.model_config, '{}'), '$._reset_from') IS NULL " - f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} " - " AND COALESCE(child.source, '') != 'tool' " - "ORDER BY child.started_at DESC, child.id DESC LIMIT 1", - (current,), - ).fetchone() - except Exception: - return session_id - if child_row is None: - break - child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0] - if not child_id or child_id in seen: - break - seen.add(child_id) - current = child_id - - return best if best is not None else session_id - - def get_messages_as_conversation( - self, - session_id: str, - include_ancestors: bool = False, - include_inactive: bool = False, - repair_alternation: bool = False, - include_row_ids: bool = False, - include_compacted: bool = False, - ) -> List[Dict[str, Any]]: - """ - Load messages in the OpenAI conversation format (role + content dicts). - Used by the gateway to restore conversation history. - - By default only active messages are returned. Pass - ``include_inactive=True`` to load soft-deleted (rewound) rows - as well. See :meth:`rewind_to_message`. - - ``include_compacted=True`` additionally loads rows preserved by - in-place compaction (``active=0, compacted=1``), deduped by - :meth:`_dedupe_display_generations`. DISPLAY reads want this; the - model-fed restore must NOT pass it, or a resumed session regrows the - very history compaction just summarized away. - - ``repair_alternation=True`` runs ``repair_message_sequence`` over the - loaded list before returning it. Callers that restore a session for - LIVE REPLAY should pass it: a durable alternation violation (e.g. a - ``user;user`` pair left by a turn that persisted no assistant row) - otherwise re-triggers the pre-request defensive repair on every - single request for the rest of the session's life — the repair - mutates only the per-request list, never the stored transcript. - Inspection/export consumers keep the default and see the transcript - verbatim. - """ - session_ids = [session_id] - if include_ancestors and not self._is_explicit_branch_session(session_id): - session_ids = self._session_lineage_root_to_tip(session_id) - - if include_inactive: - active_clause = "" - elif include_compacted: - active_clause = " AND (active = 1 OR compacted = 1)" - else: - active_clause = " AND active = 1" - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders})" - # Order by AUTOINCREMENT id (true insertion order), NOT timestamp: - # append_message stamps rows with time.time(), which is not - # monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later - # row can carry an earlier timestamp than its predecessor, and - # ORDER BY timestamp would then sort an assistant tool_calls row - # after its tool response, breaking tool-call/response adjacency - # and triggering an HTTP 400 on replay. This matches get_messages - # — see c03acca50 for the original fix. - f"{active_clause} ORDER BY id", - tuple(session_ids), - ).fetchall() - - if include_compacted: - rows = self._dedupe_display_generations(rows) - - return self._rows_to_conversation( - rows, - session_id=session_id, - include_ancestors=include_ancestors, - repair_alternation=repair_alternation, - include_row_ids=include_row_ids, - ) # Columns every conversation projection decodes. Shared by # get_messages_as_conversation and get_resume_conversations so a single @@ -14004,367 +5595,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) "api_content, display_kind, display_metadata" ) - def _rows_to_conversation( - self, - rows, - *, - session_id: str, - include_ancestors: bool, - repair_alternation: bool, - include_row_ids: bool = False, - include_summary_markers: bool = False, - ) -> List[Dict[str, Any]]: - """Decode fetched message rows into the OpenAI conversation format. - - Extracted from get_messages_as_conversation so get_resume_conversations - can build the model-fed and display views from one SELECT. ``rows`` must - already be ordered by ``id`` (insertion order) and filtered to the - desired session set / active state by the caller. - """ - messages = [] - # Watermark rotation column-clones concurrent tail rows into the child - # after the new summary, so the copies need not be adjacent. Index the - # exact durable clone identity while decoding instead of rescanning the - # whole accumulated lineage for every user row. - exact_user_clones: Dict[Tuple[Any, str], Dict[str, Any]] = {} - for row in rows: - content = self._decode_content(row["content"]) - if row["role"] in {"user", "assistant"} and isinstance(content, str): - content = sanitize_context(content).strip() - msg = {"role": row["role"], "content": content} - # Born durable (#92231): this dict is materialized FROM a durable - # row, so stamp the persistence marker at the source instead of - # relying on every restore caller to thread the loaded list back - # through a flush as ``conversation_history=`` — any - # identity-losing handoff (compression's durable-snapshot - # adoption, incremental persists with no history arg) would - # otherwise re-append the ENTIRE transcript on flush. - # Underscore-prefixed like ``_row_id``: every transport strips it - # before the wire, and compression's assembly copies deliberately - # strip it so rotated child handoffs still flush (see - # _fresh_compaction_message_copy). - msg[_DB_PERSISTED_MARKER_KEY] = True - # Durable per-message identity for surfaces that need to address a - # specific row later (desktop reactions). OPT-IN: only the gateway - # asks for it — every other consumer (ACP restore, export, - # inspection) gets the transcript in its historical shape. - # Underscore-prefixed so every transport's convert_messages() - # strips it before the wire. - if include_row_ids and row["id"] is not None: - msg["_row_id"] = row["id"] - # api_content is the byte-fidelity sidecar: the exact string sent - # to the API when it differed from the clean content. Returned - # VERBATIM — no sanitize_context, no strip — because the replay - # path substitutes it for content to keep the provider prompt - # cache prefix byte-stable across turns. Cleaning it here would - # re-introduce the divergence it exists to remove. - if row["api_content"]: - msg["api_content"] = row["api_content"] - if row["display_kind"]: - msg["display_kind"] = row["display_kind"] - if row["display_metadata"]: - decoded = self._decode_display_metadata(row["display_metadata"]) - if decoded is not None: - msg["display_metadata"] = decoded - if include_summary_markers and row["_compressed_summary"]: - msg["_compressed_summary"] = True - if row["timestamp"]: - msg["timestamp"] = row["timestamp"] - if row["tool_call_id"]: - msg["tool_call_id"] = row["tool_call_id"] - if row["tool_name"]: - msg["tool_name"] = row["tool_name"] - if row["effect_disposition"]: - msg["effect_disposition"] = row["effect_disposition"] - if row["tool_calls"]: - try: - msg["tool_calls"] = json.loads(row["tool_calls"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []") - msg["tool_calls"] = [] - # Surface the platform-side message id (e.g. yuanbao msg_id, - # telegram update_id) so platform-specific flows like recall - # can match by external identifier instead of having to fall - # back to content-match heuristics. Exposed as ``message_id`` - # for backward compatibility with the JSONL transcript shape. - if row["platform_message_id"]: - msg["message_id"] = row["platform_message_id"] - if row["observed"]: - msg["observed"] = True - # Restore reasoning fields on assistant messages so providers - # that replay reasoning (OpenRouter, OpenAI, Nous) receive - # coherent multi-turn reasoning context. - if row["role"] == "assistant": - if row["finish_reason"]: - msg["finish_reason"] = row["finish_reason"] - if row["reasoning"]: - msg["reasoning"] = row["reasoning"] - if row["reasoning_content"] is not None: - msg["reasoning_content"] = row["reasoning_content"] - if row["reasoning_details"]: - try: - msg["reasoning_details"] = json.loads(row["reasoning_details"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize reasoning_details, falling back to None") - msg["reasoning_details"] = None - if row["codex_reasoning_items"]: - try: - msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize codex_reasoning_items, falling back to None") - msg["codex_reasoning_items"] = None - if row["codex_message_items"]: - try: - msg["codex_message_items"] = json.loads(row["codex_message_items"]) - except (json.JSONDecodeError, TypeError): - logger.warning("Failed to deserialize codex_message_items, falling back to None") - msg["codex_message_items"] = None - if include_ancestors: - canonical_content, _is_composite = ( - self._canonical_replayed_user_content(msg) - ) - exact_clone_key = self._exact_replayed_user_clone_key( - msg.get("timestamp"), canonical_content - ) - previous_exact = ( - exact_user_clones.get(exact_clone_key) - if exact_clone_key is not None - else None - ) - duplicate = None - if previous_exact is not None: - previous_index = next( - ( - index - for index, candidate in enumerate(messages) - if candidate is previous_exact - ), - None, - ) - if previous_index is not None: - duplicate = (previous_index, True) - if duplicate is None: - duplicate = self._find_duplicate_replayed_user_message( - messages, msg - ) - if duplicate is not None: - duplicate_index, prefer_current = duplicate - if prefer_current: - # A rotated compression child can carry the same live - # ask as the parent row plus the only surviving summary - # scaffold. Keep the child carrier (and its durable row - # id), not the simpler ancestor copy. - messages.pop(duplicate_index) - else: - continue - messages.append(msg) - if include_ancestors and exact_clone_key is not None: - exact_user_clones[exact_clone_key] = msg - # DEFENSE-IN-DEPTH against background-review session pollution: a forked - # skill/memory review that (in older builds, before the _persist_disabled - # fix) shared the parent's session_id wrote its harness turn into this - # real session. The harness is a user/system message instructing the - # agent to "Review the conversation above and update the skill library / - # save to memory" under a hard tool restriction; re-loading it as live - # history makes the agent adopt the curator role and refuse the user's - # actual task. Strip any such harness message AND the curator-mode - # assistant reply immediately following it, so a polluted session - # resumes clean even if stray rows exist. - messages = _strip_background_review_harness(messages) - # DEFENSE-IN-DEPTH against #78148: before that fix, a bare tool-call - # marker (e.g. "[memory]") could get cached as a fallback and - # persisted as if it were the model's real answer. Sessions written - # before the fix can still carry those rows — clear the stray - # content on load so replaying history doesn't re-teach the model - # to keep emitting the marker. No-op for unaffected sessions. - messages = _strip_stale_tool_call_markers(messages) - if repair_alternation and messages: - # Lazy import: hermes_state already depends on agent.* (see - # sanitize_context above), but keep this optional path from - # widening the import surface at module load. - from agent.agent_runtime_helpers import repair_message_sequence - - repaired = repair_message_sequence(None, messages) - if repaired: - logger.info( - "Repaired %d message-alternation violation(s) while " - "restoring session %s — durable transcript kept them, " - "see repair_message_sequence", - repaired, - session_id, - ) - return messages - - def get_resume_conversations( - self, session_id: str - ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: - """Return ``(model_history, display_history)`` for a session resume in ONE SELECT. - - ``session.resume`` needs two projections of the same lineage: - - - ``model_history`` — the tip session's active rows, alternation-repaired - (the live-replay working conversation). Equivalent to - ``get_messages_as_conversation(session_id, repair_alternation=True)``. - - ``display_history`` — the full compression lineage (ancestors → tip), - verbatim, with replayed-user dedup. Explicit ``/branch`` sessions are - excluded from this lineage because their own rows already contain the - copied transcript; including the live parent's rows would let messages - written to the original after the fork leak into the branch. - - The display projection also includes rows preserved by IN-PLACE - compaction (``active=0, compacted=1``), deduped by - :meth:`_dedupe_display_generations`. Without them a compacted - conversation resumes showing only its summary plus the carried-forward - tail — the user's own turns read as deleted even though every row is - still on disk, and the REST transcript read (which has always included - them) disagreed with this one about the same session (#92080). - - The display fetch already reads a superset of the model fetch (the tip - rows are part of the lineage), so serving both from one lineage SELECT - halves the resume's DB work versus two separate calls, with byte-identical - output (see test_get_resume_conversations_matches_separate_reads). - """ - session_ids = self._resume_lineage_ids(session_id) - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) " - # Compaction-archived rows (active=0, compacted=1) are display - # history; Undo/Rewind rows (active=0, compacted=0) are not. - "AND (active = 1 OR compacted = 1) " - # ORDER BY id (insertion order) — see get_messages_as_conversation - # for why timestamp ordering is unsafe. - "ORDER BY id", - tuple(session_ids), - ).fetchall() - - # Tip rows are exactly the model-fed set (get_messages_as_conversation - # with session_ids=[session_id]); filtering the lineage fetch preserves - # their relative id order. The model projection stays active-only — it - # is the compressed working context and must not regrow the history - # compaction just summarized away. - tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]] - model_history = self._rows_to_conversation( - tip_rows, - session_id=session_id, - include_ancestors=False, - repair_alternation=True, - include_row_ids=True, - # Pre-compress checkpointing: the resumed model history must keep - # the summary marker so checkpoint providers can exclude derivative - # summaries after a process restart (marker survives restart). - include_summary_markers=True, - ) - display_history = self._rows_to_conversation( - self._dedupe_display_generations(rows), - session_id=session_id, - include_ancestors=True, - repair_alternation=False, - include_row_ids=True, - ) - return model_history, display_history - - def _resume_lineage_ids(self, session_id: str) -> List[str]: - """Session ids a full (display) resume materializes for *session_id*. - - Compression continuations need their ended ancestors' rows for the - display transcript; an explicit ``/branch`` copy already owns its - transcript, so its lineage is itself alone. This is the ONE definition - shared by the resume readers (``get_resume_conversations``, - ``get_ancestor_display_prefix``) and the resume guard - (``assert_resume_safe`` / ``get_resume_message_count``) — the guard must - count exactly the rows a resume would load, never a superset. - """ - if self._is_explicit_branch_session(session_id): - return [session_id] - return self._session_lineage_root_to_tip(session_id) - - def get_resume_message_count( - self, session_id: str, *, tip_only: bool = False - ) -> int: - """Count the rows that a resume would materialize. - - ``tip_only=True`` counts the tip segment's ACTIVE rows — the set a - model-history restore loads (``get_messages_as_conversation`` without - ancestors, or the deferred Desktop resume that pages the display - transcript over REST and never materializes the ancestor prefix in - memory). - - Otherwise this counts the full-lineage DISPLAY set — active rows plus - the compaction-archived rows ``get_resume_conversations`` now loads - for the transcript. Counting only active rows here would let a - heavily-compacted conversation pass a limit sized for a handful of - live rows and then materialize tens of thousands. - """ - session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) - active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" - placeholders = ",".join("?" for _ in session_ids) - with self._read_ctx() as conn: - row = conn.execute( - f"SELECT COUNT(*) FROM messages " - f"WHERE session_id IN ({placeholders}) AND {active_clause}", - tuple(session_ids), - ).fetchone() - return int(row[0] if row else 0) - - def assert_resume_safe( - self, - session_id: str, - max_messages: Optional[int] = None, - *, - tip_only: bool = False, - ) -> int: - """Return resume row count or reject a transcript too large to load. - - ``max_messages=None`` resolves the limit from config - (``sessions.max_resume_messages``); 0 disables the guard and returns - the (bounded) count without raising. - - ``tip_only=True`` bounds only the tip segment's ACTIVE rows, for - callers that never materialize the ancestor lineage or the - compaction archive in memory (tip-only model restore, deferred - Desktop resume whose display history is REST-paginated). A - heavily-compressed conversation — 85 compaction segments and ~29k - lineage rows behind a ~700-row tip — is exactly the shape compression - is supposed to produce; counting its whole lineage against a limit - sized for in-memory materialization rejected the healthiest sessions - (Desktop Bot Chat stuck on "Waking up…" with code 4130) while the - process would only ever have held the tip. - - The full (non-``tip_only``) bound counts the DISPLAY set — active plus - compaction-archived rows — because that is what - ``get_resume_conversations`` materializes for the transcript. - """ - if max_messages is None: - max_messages = resolved_max_resume_messages() - if max_messages < 0: - raise ValueError("max_messages must be non-negative") - if max_messages == 0: - # Guard disabled by config — skip counting entirely. Every live - # caller invokes this for its raise side effect and ignores the - # return value, and an unbounded lineage COUNT here would do the - # exact pathological work the disable exists to avoid. - return 0 - session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) - active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" - placeholders = ",".join("?" for _ in session_ids) - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COUNT(*) FROM (" - f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) " - f"AND {active_clause} LIMIT ?" - ")", - (*session_ids, max_messages + 1), - ).fetchone() - message_count = int(row[0] if row else 0) - if message_count > max_messages: - raise SessionResumeTooLargeError( - message_count, - max_messages, - scope="in its tip segment" if tip_only else "across its lineage", - ) - return message_count def assert_export_safe( self, @@ -14390,75 +5620,17 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # this for its raise side effect only (and skip calling it # entirely when the limit is 0). return 0 - with self._read_ctx() as conn: - row = conn.execute( - "SELECT COUNT(*) FROM (" - "SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?" - ")", - (session_id, max_messages + 1), - ).fetchone() + row = self._read_one( + "SELECT COUNT(*) FROM (" + "SELECT 1 FROM messages WHERE session_id = ? AND active = 1 LIMIT ?" + ")", + (session_id, max_messages + 1), + ) message_count = int(row[0] if row else 0) if message_count > max_messages: raise SessionExportTooLargeError(session_id, message_count, max_messages) return message_count - def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]: - """Return the ancestor-only display messages for a session lineage. - - These are messages from parent/grandparent sessions (compression - ancestors) that appear in the display transcript but NOT in the - tip session's model-fed history. Used by ``session.resume`` to - build the ``display_history_prefix`` that ``_live_session_payload`` - prepends to the live model history. - - Previously the prefix was calculated as - ``display_history[:len(display) - len(raw)]``, but that overcounts - when ``repair_message_sequence`` removes messages from the MIDDLE - of the tip history (e.g. verification candidates collapsed by the - consecutive-assistant merge) — the length difference includes both - ancestor messages AND repair-removed tip messages, but the slice - only captures the first N display messages (which are tip messages - when there are no ancestors), causing duplication. This method - returns ONLY the genuine ancestor messages, identified by - ``session_id != tip_session_id``. (#65919) - """ - session_ids = self._resume_lineage_ids(session_id) - if len(session_ids) <= 1: - return [] - with self._read_ctx() as conn: - placeholders = ",".join("?" for _ in session_ids) - rows = conn.execute( - f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " - f"FROM messages WHERE session_id IN ({placeholders}) " - # Display read: compaction-archived rows included, Undo/Rewind - # rows excluded (see get_resume_conversations). - "AND (active = 1 OR compacted = 1) " - "ORDER BY id", - tuple(session_ids), - ).fetchall() - rows = self._dedupe_display_generations(rows) - ancestor_ids = { - int(row["id"]) - for row in rows - if row["session_id"] != session_id and row["id"] is not None - } - if not ancestor_ids: - return [] - lineage = self._rows_to_conversation( - rows, - session_id=session_id, - include_ancestors=True, - repair_alternation=False, - include_row_ids=True, - ) - prefix: List[Dict[str, Any]] = [] - for message in lineage: - if message.get("_row_id") not in ancestor_ids: - continue - projected = message.copy() - projected.pop("_row_id", None) - prefix.append(projected) - return prefix def _is_explicit_branch_session(self, session_id: str) -> bool: """Return whether *session_id* is a copied user-facing branch. @@ -14471,11 +5643,10 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) """ if not session_id: return False - with self._read_ctx() as conn: - row = conn.execute( - "SELECT model_config FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() + row = self._read_one( + "SELECT model_config FROM sessions WHERE id = ?", + (session_id,), + ) if row is None: return False raw_config = row["model_config"] if hasattr(row, "keys") else row[0] @@ -14487,19 +5658,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) return False return isinstance(config, dict) and bool(config.get("_branched_from")) - def get_conversation_root(self, session_id: str) -> str: - """Return the ROOT id of *session_id*'s lineage chain. - - The root is the stable "conversation id": context compression - rotates ``session_id`` to a new segment linked via - ``parent_session_id``, and delegate subagents hang off their - parent the same way. Walking to the root gives every segment of - one user-facing conversation (and its delegation tree) a single - identifier — used for Nous Portal ``conversation=`` usage tagging. - Returns *session_id* unchanged when it has no recorded parent. - """ - chain = self._session_lineage_root_to_tip(session_id) - return (chain[0] if chain and chain[0] else session_id) def _session_lineage_root_to_tip(self, session_id: str) -> List[str]: if not session_id: @@ -14523,324 +5681,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) current = row["parent_session_id"] if hasattr(row, "keys") else row[0] return list(reversed(chain)) or [session_id] - @staticmethod - def _canonical_replayed_user_content( - msg: Dict[str, Any], - ) -> Tuple[Any, bool]: - """Return canonical live content and whether *msg* is composite.""" - if msg.get("role") != "user": - return None, False - - from agent.context_compressor import split_user_originated_turn - - handoff, live_view = split_user_originated_turn(msg) - is_composite = handoff is not None and live_view is not None - return ( - live_view.get("content") - if is_composite and live_view is not None - else msg.get("content"), - is_composite, - ) - - @staticmethod - def _exact_replayed_user_clone_key( - timestamp: Any, content: Any - ) -> Optional[Tuple[Any, str]]: - """Return a hashable key for a column-exact rotation clone.""" - if timestamp is None or content in (None, "", []): - return None - try: - encoded = json.dumps( - content, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - except (TypeError, ValueError): - return None - return timestamp, encoded - - @staticmethod - def _find_duplicate_replayed_user_message( - messages: List[Dict[str, Any]], msg: Dict[str, Any] - ) -> Optional[Tuple[int, bool]]: - """Return an adjacent replay duplicate and whether *msg* must win. - - Compression rotation may persist the current ask once in the parent - and again inside a composite child carrier. Compare the canonical live - payload for that carrier, while retaining the historical exact-string - dedupe for ordinary replayed users. The child carrier wins because it - owns both the current durable row identity and the retained scaffold. - """ - if msg.get("role") != "user": - return None - - content, prefer_current = SessionDB._canonical_replayed_user_content(msg) - if content in (None, "", []): - return None - - for index in range(len(messages) - 1, -1, -1): - prev = messages[index] - if prev.get("role") == "user": - prev_content, prev_is_composite = ( - SessionDB._canonical_replayed_user_content(prev) - ) - if prev_content == content and ( - prefer_current - or prev_is_composite - or isinstance(content, str) - ): - return index, prefer_current - if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")): - return None - return None - - @staticmethod - def _is_duplicate_replayed_user_message( - messages: List[Dict[str, Any]], msg: Dict[str, Any] - ) -> bool: - return SessionDB._find_duplicate_replayed_user_message(messages, msg) is not None # ========================================================================= # Rewind (soft-delete) — see /rewind slash command + issue #21910 # ========================================================================= - def get_active_message_ids(self, session_id: str) -> List[int]: - """Return the ordered physical ids pinned by rewind CAS checks. - - Conversation projections intentionally omit legacy background-review - harness rows. Destructive rewinds must nevertheless pin every active - physical row so the caller snapshot matches the transaction-local - comparison in :meth:`rewind_to_message`. - """ - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id", - (session_id,), - ).fetchall() - return [int(row[0]) for row in rows] - - @staticmethod - def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]: - """Return active message/tool-call counts inside the caller's txn.""" - rows = conn.execute( - "SELECT tool_calls FROM messages " - "WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchall() - tool_call_count = 0 - for row in rows: - raw = row[0] - if not raw: - continue - try: - decoded = json.loads(raw) if isinstance(raw, str) else raw - except (json.JSONDecodeError, TypeError): - continue - if isinstance(decoded, list): - tool_call_count += len(decoded) - elif decoded: - tool_call_count += 1 - return len(rows), tool_call_count - - def rewind_to_message( - self, - session_id: str, - target_message_id: int, - *, - preserve_compaction_handoff: bool = False, - expected_active_ids: Optional[List[int]] = None, - expected_target_content: Any = None, - ) -> Dict[str, Any]: - """Soft-delete all messages with id >= ``target_message_id`` in *session_id*. - - The target message itself becomes inactive as well so the caller - can pre-fill it as the next user prompt without it appearing - twice in the replayed transcript. Rewound rows are kept on - disk with ``active=0`` for audit / forensic inspection — use - :meth:`get_messages` with ``include_inactive=True`` to see them. - - Returns a dict:: - - { - "rewound_count": int, # number of rows newly flipped to active=0 - "target_message": dict, # full row dict of the target - "new_head_id": int|None # id of the last still-active row, or None - } - - Raises ``ValueError`` if the target message does not exist in - *session_id* or if its role is not ``"user"``. With - ``preserve_compaction_handoff=True``, a composite summary carrier is - split inside the same write transaction: its original row is archived - and its canonical hidden handoff scaffold is inserted as the new head. - That opt-in result also contains ``replacement_message_id``. - - ``expected_active_ids`` optionally pins the ordered active row set. - ``expected_target_content`` additionally pins the selected canonical - live-user payload. Both checks run inside the write transaction before - any row or counter mutation. Presentation-only metadata changes (for - example Desktop reactions) deliberately do not invalidate a rewind. - A live cross-process turn lease always refuses the rewind; expired or - provably dead holders are reclaimed inside the mutation transaction. - - Always increments ``sessions.rewind_count`` — even when the - target is already inactive — so the counter accurately reflects - the number of rewind operations performed against the session. - Idempotent on the ``active`` flag: re-rewinding past the same - target is a no-op on row state but still bumps the counter. - """ - - def _do(conn): - # Rewind changes the active transcript and must honor the same - # compression/closed-parent and cross-process turn guards as - # append writers. - self._check_transcript_write_guards( - conn, - session_id, - None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - ) - - if expected_active_ids is not None: - active_rows = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND active = 1 ORDER BY id", - (session_id,), - ).fetchall() - active_ids = [int(active_row[0]) for active_row in active_rows] - if active_ids != expected_active_ids: - raise RuntimeError( - "active transcript changed before the rewind could be persisted" - ) - - row = conn.execute( - "SELECT * FROM messages WHERE id = ? AND session_id = ?", - (target_message_id, session_id), - ).fetchone() - if row is None: - raise ValueError( - f"message {target_message_id} not found in session {session_id}" - ) - target_row = dict(row) - if target_row.get("role") != "user": - raise ValueError( - f"rewind target must be a 'user' message (got role=" - f"{target_row.get('role')!r}, id={target_message_id})" - ) - - replacement_message_id: Optional[int] = None - replacement: Optional[Dict[str, Any]] = None - if preserve_compaction_handoff or expected_target_content is not None: - if not target_row.get("active"): - raise ValueError("rewind target is not active") - from agent.context_compressor import split_user_originated_turn - - split_target = target_row.copy() - split_target["content"] = self._decode_content( - split_target.get("content") - ) - split_target["display_metadata"] = self._decode_display_metadata( - split_target.get("display_metadata") - ) - handoff, live_view = split_user_originated_turn(split_target) - if live_view is None: - raise ValueError("rewind target is not a user-originated turn") - live_content = live_view.get("content") - if isinstance(live_content, str): - live_content = sanitize_context(live_content).strip() - if ( - expected_target_content is not None - and live_content != expected_target_content - ): - raise RuntimeError( - "rewind target changed before it could be persisted" - ) - if preserve_compaction_handoff and handoff is None: - raise ValueError( - "preserve_compaction_handoff requires an active composite carrier" - ) - replacement = handoff if preserve_compaction_handoff else None - - cursor = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND id >= ? AND active = 1", - (session_id, target_message_id), - ) - ids = [r[0] for r in cursor.fetchall()] - if ids: - placeholders = ",".join("?" for _ in ids) - conn.execute( - f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})", - ids, - ) - if replacement is not None: - self._insert_message_rows(conn, session_id, [replacement]) - inserted = conn.execute("SELECT last_insert_rowid()").fetchone() - replacement_message_id = int(inserted[0]) - conn.execute( - "UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 " - "WHERE id = ?", - (session_id,), - ) - message_count, tool_call_count = self._active_transcript_counts( - conn, session_id - ) - conn.execute( - "UPDATE sessions SET message_count = ?, tool_call_count = ? " - "WHERE id = ?", - (message_count, tool_call_count, session_id), - ) - head_row = conn.execute( - "SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1", - (session_id,), - ).fetchone() - new_head_id = ( - head_row[0] if head_row and head_row[0] is not None else None - ) - return target_row, ids, new_head_id, replacement_message_id - - target_row, rewound, new_head_id, replacement_message_id = ( - self._execute_write(_do) - ) - - # Decode content for callers (prefill the prompt buffer) without a - # second fallible database operation after the transaction commits. - target_row["content"] = self._decode_content(target_row.get("content")) - - result = { - "rewound_count": len(rewound), - "target_message": target_row, - "new_head_id": new_head_id, - } - if preserve_compaction_handoff: - result["replacement_message_id"] = replacement_message_id - return result - - def restore_rewound(self, session_id: str, since_message_id: int) -> int: - """Mark inactive messages with id >= *since_message_id* active again. - - Returns the number of rows flipped back to ``active=1``. - Intended for undo-of-rewind and test cleanup; not wired to a - slash command in v1. - """ - def _do(conn): - cursor = conn.execute( - "SELECT id FROM messages " - "WHERE session_id = ? AND id >= ? AND active = 0", - (session_id, since_message_id), - ) - ids = [r[0] for r in cursor.fetchall()] - if ids: - placeholders = ",".join("?" for _ in ids) - conn.execute( - f"UPDATE messages SET active = 1 WHERE id IN ({placeholders})", - ids, - ) - return len(ids) - - return self._execute_write(_do) # ========================================================================= # Search @@ -14882,14 +5727,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) params.extend(ws_params) where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else "" params.extend([limit, offset]) - with self._read_ctx() as conn: - cursor = conn.execute( - f"{select_with_last_active}" - f"{where_sql} " - "ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?", - params, - ) - return [self._session_row_dict(row) for row in cursor.fetchall()] + return [self._session_row_dict(row) for row in self._read_all( + f"{select_with_last_active}" + f"{where_sql} " + "ORDER BY last_active DESC, s.started_at DESC, s.id DESC LIMIT ? OFFSET ?", + params, + )] # ========================================================================= # Utility @@ -14952,9 +5795,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) where_sql = f" WHERE {' AND '.join(where_clauses)}" if where_clauses else "" - with self._read_ctx() as conn: - cursor = conn.execute(f"SELECT COUNT(*) FROM sessions s{where_sql}", params) - return cursor.fetchone()[0] + return self._read_one(f"SELECT COUNT(*) FROM sessions s{where_sql}", params)[0] def session_count_ge(self, n: int = 1) -> bool: """Check if at least N sessions exist (archived included). @@ -14967,9 +5808,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) Use this instead of ``session_count() >= n`` when the exact count is irrelevant. """ - with self._read_ctx() as conn: - cursor = conn.execute("SELECT 1 FROM sessions LIMIT ?", (n,)) - rows = cursor.fetchall() + rows = self._read_all("SELECT 1 FROM sessions LIMIT ?", (n,)) return len(rows) >= n def session_count_by_source( @@ -15017,79 +5856,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchall() return {str(row["source"]): int(row["count"] or 0) for row in rows} - def message_count(self, session_id: str = None) -> int: - """Count messages, optionally for a specific session.""" - with self._read_ctx() as conn: - if session_id: - cursor = conn.execute( - "SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,) - ) - else: - cursor = conn.execute("SELECT COUNT(*) FROM messages") - return cursor.fetchone()[0] - - def has_platform_message_id( - self, session_id: str, platform_message_id: str - ) -> bool: - """Check if a message with the given platform_message_id exists. - - Uses the idx_messages_platform_msg_id partial index for efficient - lookup. Used by the gateway's transient-failure dedupe guard (#47237) - to skip re-persisting a user message that was already saved on a - prior retry of the same inbound platform message. - """ - with self._read_ctx() as conn: - cursor = conn.execute( - "SELECT 1 FROM messages " - "WHERE session_id = ? AND platform_message_id = ? LIMIT 1", - (session_id, platform_message_id), - ) - return cursor.fetchone() is not None # ========================================================================= # Export and cleanup # ========================================================================= - def _is_explicit_fork_child_row(self, session: Dict[str, Any]) -> bool: - """True when ``session`` is a branch, delegate, or tool child of its parent. - - Markers only count as a fork when they point at ``parent_session_id``. - Compression copies ``model_config`` onto the continuation - (``publish_compression_child`` callers pass - ``agent._session_init_model_config``), so a delegate's continuation - carries ``_delegate_from=``. Presence-only - matching would treat that real continuation as a fork — the same - misclassification ``_NON_CONTINUATION_CHILD_FILTER_SQL`` already - avoids by binding both markers to the queried parent. - """ - if session.get("source") == "tool": - return True - raw = session.get("model_config") - if not raw: - return False - try: - cfg = json.loads(raw) if isinstance(raw, str) else raw - except (TypeError, json.JSONDecodeError): - return False - if not isinstance(cfg, dict): - return False - parent_id = session.get("parent_session_id") - branched = cfg.get("_branched_from") - delegated = cfg.get("_delegate_from") - if parent_id: - return branched == parent_id or delegated == parent_id - return branched is not None or delegated is not None - - def is_explicit_fork_child(self, session_id: str) -> bool: - """True when ``session_id`` is a /branch, delegate, or tool child row. - - Read-only public view of :meth:`_is_explicit_fork_child_row` for - callers that must respect the fork boundary without re-implementing - its marker rules (``agent/prompt_cache_scope.py`` keeps a declared - conversation key from crossing it). A missing row is not a fork. - """ - session = self.get_session(session_id) - return bool(session and self._is_explicit_fork_child_row(session)) def declared_scope_identity(self, session_id: str) -> Tuple[bool, str]: """Fork verdict and recorded ``source`` for *session_id*, in ONE read. @@ -15115,124 +5886,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) str(session.get("source") or "").strip(), ) - def latest_conversation_boundary( - self, session_key: str, source: str - ) -> Optional[int]: - """How many conversation boundaries this routing peer has crossed. - - A boundary is a row this peer ended at an intentional conversation - break — the ``_RESET_END_REASONS`` set (``/new``, ``/switch``, idle, - daily, suspended, resume_pending_expired). That is the same fence - :meth:`find_latest_gateway_session_for_peer` refuses to reach behind, - so the two agree on where one conversation stops and the next begins - and cannot drift. - - The peer is ``(session_key, source)``, the SAME identity tuple recovery - uses — never the key alone. ``X-Hermes-Session-Key`` accepts any - authenticated caller-supplied string, so an API conversation may - legally carry the same key as a Telegram row in one database; keying - on the string alone would let a ``/new`` on that unrelated row rotate - this conversation's affinity identity while recovery correctly refuses - to cross the same line. - - Returns the count, or ``None`` when this peer has never been reset. - - The value comes from ``conversation_generations``, which - :meth:`_bump_conversation_generation` advances inside the transaction - that writes each boundary — NOT from an aggregate over the session - rows. An aggregate cannot prove non-reuse: ``delete_session()`` - orphans children and deletes the row, and bulk prune selects ended - rows, so ``COUNT``/``MAX`` over boundaries can return a pair it already - emitted and hand a new conversation a retired affinity identity. It is - also wall-clock-free, so a backwards NTP correction cannot reorder it. - - Databases upgraded mid-conversation start at no generation and take - their first one from the next boundary written; a conversation that - reset before the upgrade shares its predecessor's scope once, which - costs a warm prompt-cache bucket and never crosses an identity. - - These rows are never garbage-collected, by design: dropping one resets - the peer to "no generation", so its next boundary writes ``1`` again - and re-issues a scope a retired conversation already used — the ABA - this counter exists to prevent. See the schema comment in - ``hermes_state_common.py``. - """ - if not session_key or not source: - return None - with self._read_ctx() as conn: - row = conn.execute( - "SELECT generation FROM conversation_generations " - "WHERE source = ? AND session_key = ?", - (source, session_key), - ).fetchone() - if row is None or row["generation"] is None: - return None - generation = int(row["generation"]) - return generation if generation > 0 else None - - def _is_compression_child_row(self, child: Dict[str, Any]) -> bool: - parent_id = child.get("parent_session_id") - if not parent_id or self._is_explicit_fork_child_row(child): - return False - parent = self.get_session(parent_id) - return bool(parent and parent.get("end_reason") == "compression") - - def get_compression_lineage(self, session_id: str) -> List[str]: - """Return compression ancestors through tip in chronological order.""" - session = self.get_session(session_id) - if not session or self._is_explicit_fork_child_row(session): - return [session_id] if session else [] - - root = session - ancestors = {root["id"]} - while self._is_compression_child_row(root): - parent = self.get_session(root["parent_session_id"]) - if not parent or parent["id"] in ancestors: - break - root = parent - ancestors.add(root["id"]) - - lineage = [root["id"]] - seen = {root["id"]} - current = root - while current.get("end_reason") == "compression": - with self._read_ctx() as conn: - rows = conn.execute( - """ - SELECT * FROM sessions - WHERE parent_session_id = ? - ORDER BY started_at ASC - """, - (current["id"],), - ).fetchall() - next_child = None - for row in rows: - candidate = dict(row) - if self._is_compression_child_row(candidate): - next_child = candidate - break - if not next_child or next_child["id"] in seen: - break - lineage.append(next_child["id"]) - seen.add(next_child["id"]) - current = next_child - if current["id"] == session_id: - # Continue to include later compression tips only when the - # requested session itself was compacted. - continue - return lineage if session_id in lineage else [session_id] - - def clear_messages(self, session_id: str) -> None: - """Delete all messages for a session and reset its counters.""" - def _do(conn): - conn.execute( - "DELETE FROM messages WHERE session_id = ?", (session_id,) - ) - conn.execute( - "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", - (session_id,), - ) - self._execute_write(_do) @staticmethod def _remove_session_files(sessions_dir: Optional[Path], session_id: str) -> None: @@ -15507,11 +6160,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) to clean up, and pre-populate the confirm dialog with the actual count. """ - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions WHERE {self._EMPTY_SESSION_WHERE}" - ) - return cursor.fetchone()[0] + return self._read_one(f"SELECT COUNT(*) FROM sessions WHERE {self._EMPTY_SESSION_WHERE}")[0] def delete_empty_sessions( self, @@ -15579,254 +6228,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self._remove_session_files(sessions_dir, sid) return count - @staticmethod - def _prune_filter_where( - *, - last_active_before: Optional[float] = None, - last_active_after: Optional[float] = None, - started_before: Optional[float] = None, - started_after: Optional[float] = None, - source: Optional[str] = None, - title_like: Optional[str] = None, - end_reason: Optional[str] = None, - cwd_prefix: Optional[str] = None, - min_messages: Optional[int] = None, - max_messages: Optional[int] = None, - archived: Optional[bool] = None, - model_like: Optional[str] = None, - provider: Optional[str] = None, - user_id: Optional[str] = None, - chat_id: Optional[str] = None, - chat_type: Optional[str] = None, - branch_like: Optional[str] = None, - min_tokens: Optional[int] = None, - max_tokens: Optional[int] = None, - min_cost: Optional[float] = None, - max_cost: Optional[float] = None, - min_tool_calls: Optional[int] = None, - max_tool_calls: Optional[int] = None, - include_pinned: bool = False, - ) -> Tuple[str, list]: - """Build the shared WHERE clause for bulk prune/archive selection. - - All filters AND together. Only ended sessions are ever candidates - (``ended_at IS NOT NULL``) so a live session is never selected. - ``archived`` is a tri-state: ``None`` = both, ``True`` = only - archived rows, ``False`` = only unarchived rows. - - String matching conventions: ``model_like`` / ``branch_like`` / - ``title_like`` are case-insensitive substring matches (model slugs - and branch names vary in prefix format); ``provider`` / ``user_id`` - / ``chat_id`` / ``chat_type`` / ``source`` / ``end_reason`` are - exact (case-insensitive for provider). Token bounds apply to - ``input_tokens + output_tokens``; cost bounds apply to - ``COALESCE(actual_cost_usd, estimated_cost_usd)``. - - The clause references the ``s`` table alias — callers must select - ``FROM sessions s``. - """ - clauses = ["s.ended_at IS NOT NULL"] - params: list = [] - if last_active_before is not None: - clauses.append( - """COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) < ?""" - ) - params.append(last_active_before) - # An automatic orphan sweep closes a stale open row so the user can - # still recover it. Age those rows from the sweep, not from their old - # activity, or the next prune pass can delete them immediately. - clauses.append( - "(COALESCE(s.end_reason, '') != 'startup_orphan_reap' " - "OR s.ended_at < ?)" - ) - params.append(last_active_before) - if last_active_after is not None: - clauses.append( - """COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) >= ?""" - ) - params.append(last_active_after) - if started_before is not None: - clauses.append("s.started_at < ?") - params.append(started_before) - if started_after is not None: - clauses.append("s.started_at >= ?") - params.append(started_after) - if source: - clauses.append("s.source = ?") - params.append(source) - if title_like: - clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(title_like.lower())}%") - if end_reason: - clauses.append("s.end_reason = ?") - params.append(end_reason) - if cwd_prefix: - clause, clause_params = _cwd_prefix_clause(cwd_prefix) - clauses.append(clause) - params.extend(clause_params) - if min_messages is not None: - clauses.append("s.message_count >= ?") - params.append(min_messages) - if max_messages is not None: - clauses.append("s.message_count <= ?") - params.append(max_messages) - if model_like: - clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(model_like.lower())}%") - if provider: - clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?") - params.append(provider.lower()) - if user_id: - clauses.append("s.user_id = ?") - params.append(user_id) - if chat_id: - clauses.append("s.chat_id = ?") - params.append(chat_id) - if chat_type: - clauses.append("s.chat_type = ?") - params.append(chat_type) - if branch_like: - clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'") - params.append(f"%{_escape_like(branch_like.lower())}%") - if min_tokens is not None: - clauses.append( - "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?" - ) - params.append(min_tokens) - if max_tokens is not None: - clauses.append( - "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?" - ) - params.append(max_tokens) - if min_cost is not None: - clauses.append( - "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?" - ) - params.append(min_cost) - if max_cost is not None: - clauses.append( - "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?" - ) - params.append(max_cost) - if min_tool_calls is not None: - clauses.append("COALESCE(s.tool_call_count, 0) >= ?") - params.append(min_tool_calls) - if max_tool_calls is not None: - clauses.append("COALESCE(s.tool_call_count, 0) <= ?") - params.append(max_tool_calls) - if archived is True: - clauses.append("s.archived = 1") - elif archived is False: - clauses.append("s.archived = 0") - # Pinned sessions are a durable "keep" flag (exempt from the stale - # auto-archive sweep). Bulk prune/delete/archive must honor that too: - # exclude pinned rows unless the caller explicitly opts in. Without - # this, `sessions prune`/`delete`/`archive` with a filter silently - # destroyed pinned conversations (round-3 QA SES-01, data loss). - if not include_pinned: - clauses.append("COALESCE(s.pinned, 0) = 0") - return " AND ".join(clauses), params - - @staticmethod - def _apply_prune_age_filter( - older_than_days: Optional[float], filters: Dict[str, Any] - ) -> None: - """Translate the legacy age window into the shared activity filter.""" - if ( - filters.get("last_active_before") is None - and filters.get("started_before") is None - and older_than_days is not None - ): - filters["last_active_before"] = time.time() - ( - older_than_days * 86400 - ) - - def list_prune_candidates( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> List[Dict[str, Any]]: - """Return the sessions a matching :meth:`prune_sessions` / - :meth:`archive_sessions` call would touch, without modifying anything. - - Backs ``--dry-run`` and pre-confirmation counts. Accepts the same - keyword filters as :meth:`_prune_filter_where` (unknown names raise - ``TypeError`` there). Rows are ordered oldest-first and carry - ``id, source, title, model, started_at, last_active, ended_at, - message_count, archived``. ``older_than_days`` is an inactivity - threshold: it uses the latest message timestamp, falling back to - ``started_at`` for sessions without messages. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - with self._read_ctx() as conn: - cursor = conn.execute( - f"""SELECT s.id, s.source, s.title, s.model, s.started_at, - COALESCE( - (SELECT MAX(m.timestamp) FROM messages m - WHERE m.session_id = s.id), - s.started_at - ) AS last_active, - s.ended_at, s.message_count, s.archived - FROM sessions s WHERE {where} - ORDER BY last_active ASC, s.started_at ASC""", - params, - ) - return [dict(row) for row in cursor.fetchall()] - - def count_prune_matches( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> int: - """Count sessions a matching prune/archive would touch. - - Same filter surface as :meth:`list_prune_candidates` (including the - ``include_pinned`` tri-state), but returns only a count. Used by the - CLI to report how many pinned sessions are being spared. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions s WHERE {where}", params - ) - return int(cursor.fetchone()[0]) - - def count_open_prune_matches( - self, - older_than_days: Optional[float] = None, - source: str = None, - **filters, - ) -> int: - """Count open sessions excluded from a matching bulk prune. - - This applies every normal prune filter, but inverts only the - ``ended_at`` safety guard. It is visibility-only: callers can explain - why an otherwise matching session was skipped without making live - sessions eligible for destructive pruning. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, params = self._prune_filter_where(source=source, **filters) - ended_guard = "s.ended_at IS NOT NULL" - if not where.startswith(ended_guard): - raise RuntimeError("prune filter lost its ended-session safety guard") - open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}" - with self._read_ctx() as conn: - cursor = conn.execute( - f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params - ) - return int(cursor.fetchone()[0]) def archive_sessions( self, @@ -15854,256 +6255,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) self.set_session_archived(row["id"], True) return len(rows) - def archive_stale_sessions( - self, idle_days: float, *, exclude_pinned: bool = True - ) -> int: - """Archive every session untouched for at least ``idle_days`` days. - - "Touched" is the freshest of ``last_activity_at`` and the latest - message timestamp (else ``started_at``) — i.e. real recency, not - creation time — so a session - created long ago but active yesterday is spared, while an old - abandoned one (even a still-open one) is swept. Unlike - :meth:`archive_sessions`, this method can also archive unended - sessions. - - Guards: - * ``pinned = 0`` when ``exclude_pinned`` (the Desktop "keep" flag). - * ``archived = 0`` so repeat runs are idempotent no-ops. - * only lineage *tips* / standalone rows are candidates - (``end_reason <> 'compression'``); a stale tip archives its whole - chain via :meth:`set_session_archived`, so we never resurrect an - active conversation by matching an old compressed-away root whose - live continuation is recent. - - Returns the number of sessions archived. Never raises for an empty or - non-positive ``idle_days`` — it simply archives nothing. - """ - if idle_days is None or idle_days < 0: - return 0 - cutoff = time.time() - float(idle_days) * 86400.0 - pin_clause = "AND s.pinned = 0" if exclude_pinned else "" - with self._read_ctx() as conn: - rows = conn.execute( - f""" - SELECT s.id FROM sessions s - WHERE s.archived = 0 - AND COALESCE(s.end_reason, '') <> 'compression' - {pin_clause} - AND {_sql_session_last_active("s")} < ? - ORDER BY s.started_at ASC - """, - (cutoff,), - ).fetchall() - ids = [(r["id"] if isinstance(r, sqlite3.Row) else r[0]) for r in rows] - for sid in ids: - self.set_session_archived(sid, True) - return len(ids) - - def prune_sessions( - self, - older_than_days: Optional[float] = 90, - source: str = None, - sessions_dir: Optional[Path] = None, - exclude_active_write_guards: bool = False, - **filters, - ) -> int: - """Delete sessions matching the filters. Returns count deleted. - - By default, delete ended sessions inactive for - ``older_than_days`` days, optionally restricted to ``source``. - Activity is the latest message timestamp, falling back to - ``started_at`` for sessions without messages. Additional keyword - filters AND together — the full set is defined by - :meth:`_prune_filter_where`: - - * ``last_active_before`` / ``last_active_after`` — epoch bounds on - the latest message timestamp (falling back to ``started_at``). - * ``started_before`` / ``started_after`` — epoch bounds on - ``started_at``. An explicit ``started_before`` overrides the - default ``older_than_days`` inactivity cutoff; pass - ``older_than_days=None`` for no implicit upper age bound. - * ``title_like`` / ``model_like`` / ``branch_like`` — - case-insensitive substring matches. - * ``end_reason`` / ``provider`` / ``user_id`` / ``chat_id`` / - ``chat_type`` — exact matches (provider case-insensitive, against - ``billing_provider``). - * ``cwd_prefix`` — session cwd equals or is under this path. - * ``min_messages`` / ``max_messages`` — bounds on message_count. - * ``min_tokens`` / ``max_tokens`` — bounds on input+output tokens. - * ``min_cost`` / ``max_cost`` — bounds on USD cost - (actual, falling back to estimated). - * ``min_tool_calls`` / ``max_tool_calls`` — bounds on tool_call_count. - * ``archived`` — tri-state: None = both (default), True = only - archived, False = only unarchived. - - Only prunes ended sessions (not active ones). Child sessions outside - the prune window are orphaned (parent_session_id set to NULL) rather - than cascade-deleted. When *sessions_dir* is provided, also removes - on-disk transcript files (``.json`` / ``.jsonl`` / - ``request_dump_*``) for every pruned session, outside the DB - transaction. - - ``exclude_active_write_guards`` is for destructive automatic - maintenance: rows protected by a live turn lease or compression lock - are skipped, while expired or provably dead holders are reclaimed and - fenced in the same write transaction. - """ - self._apply_prune_age_filter(older_than_days, filters) - where, where_params = self._prune_filter_where(source=source, **filters) - removed_ids: list[str] = [] - - def _do(conn): - cursor = conn.execute( - f"SELECT s.id FROM sessions s WHERE {where}", where_params - ) - session_ids = {row["id"] for row in cursor.fetchall()} - - if exclude_active_write_guards: - protected = set() - for sid in session_ids: - try: - self._check_transcript_write_guards( - conn, - sid, - compression_lock_holder=None, - turn_lease_holder=None, - reject_active_turn_lease=True, - reject_active_compression_lock=True, - allow_closed_compression_parent=True, - ) - except ( - SessionCompressionInProgressError, - SessionTurnLeaseLostError, - ): - protected.add(sid) - session_ids.difference_update(protected) - - if not session_ids: - return 0 - - # Orphan any sessions whose parent is about to be deleted - placeholders = ",".join("?" * len(session_ids)) - conn.execute( - f"UPDATE sessions SET parent_session_id = NULL " - f"WHERE parent_session_id IN ({placeholders})", - list(session_ids), - ) - - for sid in session_ids: - conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,)) - conn.execute("DELETE FROM sessions WHERE id = ?", (sid,)) - removed_ids.append(sid) - self._delete_unreferenced_system_prompts(conn) - return len(session_ids) - - count = self._execute_write(_do) - # Clean up on-disk files outside the DB transaction - for sid in removed_ids: - self._remove_session_files(sessions_dir, sid) - return count - - def purge_stale_tool_call_markers( - self, *, dry_run: bool = False, backup: bool = True - ) -> Dict[str, Any]: - """Permanently clear bare tool-call marker content (e.g. "[memory]") - left in the ``messages`` table by sessions persisted before the - #78148 fix in ``agent.conversation_loop``. - - ``_strip_stale_tool_call_markers`` already repairs this in memory on - every session load (see ``_rows_to_conversation``), so running this - is optional — but for long-lived sessions the same rows get - re-scanned and re-repaired on every resume, which is wasted work - and keeps the contaminated bytes sitting in the DB (and in any - downstream cache/backup snapshot of it) indefinitely. This rewrites - the affected rows once, in place. - - Only the ``content`` column is touched — ``role``, ``tool_calls``, - and every other column on the row are left exactly as they are, so - provider tool_call/tool_result pairing is unaffected. - - Unlike the in-memory repair, this UPDATE is permanent and can't be - undone from within the DB. Since ``backup`` defaults to True, a - timestamped full snapshot is taken via ``VACUUM INTO`` (safe against - a live connection, unlike the raw-copy ``_backup_db_file`` used for - malformed-schema repair) before any row is touched — mirroring - ``repair_state_db_schema``'s backup-by-default convention for - destructive state.db operations. No snapshot is taken when there is - nothing to change. - - With ``dry_run=True``, reports the affected row count/ids without - writing or backing up (read-only, no write lock taken). - - Returns ``{"dry_run": bool, "rows_affected": int, "row_ids": [...], - "backup_path": str|None}``. - """ - - def _find_affected(conn) -> List[int]: - cursor = conn.execute( - "SELECT id, content FROM messages " - "WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''" - ) - affected: List[int] = [] - for row in cursor.fetchall(): - content = row["content"] - if isinstance(content, str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()): - affected.append(row["id"]) - return affected - - with self._read_ctx() as conn: - affected_ids = _find_affected(conn) - - if dry_run: - return { - "dry_run": True, - "rows_affected": len(affected_ids), - "row_ids": affected_ids, - "backup_path": None, - } - - if not affected_ids: - return { - "dry_run": False, - "rows_affected": 0, - "row_ids": [], - "backup_path": None, - } - - backup_path: Optional[str] = None - if backup: - import datetime - - stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - dest = self.db_path.with_name( - f"{self.db_path.name}.pre-clean-markers-backup-{stamp}" - ) - with self._lock: - self._conn.execute("VACUUM INTO ?", (str(dest),)) - backup_path = str(dest) - logger.info("Backed up state.db to %s before clean-markers write", backup_path) - - def _do(conn): - ids = _find_affected(conn) - if ids: - placeholders = ",".join("?" * len(ids)) - conn.execute( - f"UPDATE messages SET content = '' WHERE id IN ({placeholders})", - ids, - ) - return ids - - affected_ids = self._execute_write(_do) - if affected_ids: - logger.info( - "Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)", - len(affected_ids), - ) - return { - "dry_run": False, - "rows_affected": len(affected_ids), - "row_ids": affected_ids, - "backup_path": backup_path, - } # ── Meta key/value (for scheduler bookkeeping) ── @@ -16121,7 +6272,7 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ).fetchone() if row is None: return None - return row["value"] if isinstance(row, sqlite3.Row) else row[0] + return row[0] def set_meta( self, key: str, value: str, *, cursor: Optional[sqlite3.Cursor] = None @@ -16142,13 +6293,11 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) ) return - def _do(conn): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (key, value), - ) - self._execute_write(_do) + self._write_sql( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) def retag_kanban_worker_sessions(self, workspaces_root: str) -> int: """Retag legacy kanban worker rows from ``cli`` to ``kanban``. @@ -16196,567 +6345,12 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) if not prefix: return [] escaped = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'", - (escaped + "%",), - ).fetchall() + rows = self._read_all( + "SELECT key, value FROM state_meta WHERE key LIKE ? ESCAPE '\\'", + (escaped + "%",), + ) return [(row[0], row[1]) for row in rows] - def apply_telegram_topic_migration(self) -> None: - """Create Telegram DM topic-mode tables on explicit /topic opt-in. - - This migration is deliberately not part of automatic SessionDB startup - reconciliation. Operators must be able to upgrade Hermes, keep the old - Telegram bot behavior running, and only mutate topic-mode state when the - user executes /topic to opt into the feature. - - Schema versions: - v1 — initial shape (no ON DELETE CASCADE on session_id FK) - v2 — session_id FK gets ON DELETE CASCADE so session pruning - automatically clears bindings. - v3 — ``profile_name`` dimension on both tables so multiplexed - gateways (shared ``state.db``) isolate topic mode/bindings - per Hermes profile (issue #76423). - """ - # (table, column list, DDL body). ``profile_name`` leads the primary - # key so multiplexed profiles sharing one state.db never collide on a - # private chat_id (which is the user id, identical across bots). - tables = ( - ( - "telegram_dm_topic_mode", - "profile_name, chat_id, user_id, enabled, activated_at, updated_at, " - "has_topics_enabled, allows_users_to_create_topics, " - "capability_checked_at, intro_message_id, pinned_message_id", - """ - profile_name TEXT NOT NULL DEFAULT 'default', - chat_id TEXT NOT NULL, - user_id TEXT NOT NULL, - enabled INTEGER NOT NULL DEFAULT 1, - activated_at REAL NOT NULL, - updated_at REAL NOT NULL, - has_topics_enabled INTEGER, - allows_users_to_create_topics INTEGER, - capability_checked_at REAL, - intro_message_id TEXT, - pinned_message_id TEXT, - PRIMARY KEY (profile_name, chat_id) - """, - ), - ( - "telegram_dm_topic_bindings", - "profile_name, chat_id, thread_id, user_id, session_key, " - "session_id, managed_mode, linked_at, updated_at", - """ - profile_name TEXT NOT NULL DEFAULT 'default', - chat_id TEXT NOT NULL, - thread_id TEXT NOT NULL, - user_id TEXT NOT NULL, - session_key TEXT NOT NULL, - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - managed_mode TEXT NOT NULL DEFAULT 'auto', - linked_at REAL NOT NULL, - updated_at REAL NOT NULL, - PRIMARY KEY (profile_name, chat_id, thread_id) - """, - ), - ) - - def _do(conn): - for table, columns, ddl in tables: - # Fresh installs get the v3 shape immediately. - conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})") - have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")} - if "profile_name" in have: - continue - # Pre-profile shape (v1 or v2) → v3. SQLite can't ALTER a - # primary key (or a foreign key), so rebuild; this also - # supplies the v2 ON DELETE CASCADE for v1 bindings tables. - # Legacy rows land in the "default" namespace only — never - # replicated across profiles. - legacy_columns = columns.replace("profile_name, ", "", 1) - conn.executescript( - f""" - CREATE TABLE {table}_new ({ddl}); - INSERT INTO {table}_new ({columns}) - SELECT 'default', {legacy_columns} FROM {table}; - DROP TABLE {table}; - ALTER TABLE {table}_new RENAME TO {table}; - """ - ) - - # Indexes after any rebuild so they always target the v3 shape - # (a legacy table lacking profile_name can't take the user index). - conn.executescript( - """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session - ON telegram_dm_topic_bindings(session_id); - - CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user - ON telegram_dm_topic_bindings(profile_name, user_id, chat_id); - """ - ) - - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - ("telegram_dm_topic_schema_version", "3"), - ) - self._execute_write(_do) - - def enable_telegram_topic_mode( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - has_topics_enabled: Optional[bool] = None, - allows_users_to_create_topics: Optional[bool] = None, - ) -> None: - """Enable Telegram DM topic mode for one private chat/user. - - This method intentionally owns the explicit topic migration. Ordinary - SessionDB startup must not create these side tables. - - ``profile_name`` namespaces rows under a shared multiplex ``state.db`` - (issue #76423). Callers handling a multiplexed event must pass the - routed profile from ``source.profile``, not the process-global active - profile. - """ - self.apply_telegram_topic_migration() - now = time.time() - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _to_int(value: Optional[bool]) -> Optional[int]: - if value is None: - return None - return 1 if value else 0 - - def _do(conn): - conn.execute( - """ - INSERT INTO telegram_dm_topic_mode ( - profile_name, chat_id, user_id, enabled, activated_at, updated_at, - has_topics_enabled, allows_users_to_create_topics, - capability_checked_at - ) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?) - ON CONFLICT(profile_name, chat_id) DO UPDATE SET - user_id = excluded.user_id, - enabled = 1, - updated_at = excluded.updated_at, - has_topics_enabled = excluded.has_topics_enabled, - allows_users_to_create_topics = excluded.allows_users_to_create_topics, - capability_checked_at = excluded.capability_checked_at - """, - ( - profile_name, - str(chat_id), - str(user_id), - now, - now, - _to_int(has_topics_enabled), - _to_int(allows_users_to_create_topics), - now, - ), - ) - self._execute_write(_do) - - def disable_telegram_topic_mode( - self, - *, - chat_id: str, - profile_name: str = "default", - clear_bindings: bool = True, - ) -> None: - """Disable Telegram DM topic mode for one private chat. - - When ``clear_bindings`` is True (default) the (chat_id, thread_id) - bindings for this chat are also cleared so re-enabling later - starts from a clean slate. Set to False if the operator wants to - preserve bindings for a later re-enable. - - Never creates the topic-mode tables from scratch; if they don't - exist there is nothing to disable and the call is a no-op. - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _do(conn): - try: - conn.execute( - "UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? " - "WHERE profile_name = ? AND chat_id = ?", - (time.time(), profile_name, str(chat_id)), - ) - if clear_bindings: - conn.execute( - "DELETE FROM telegram_dm_topic_bindings " - "WHERE profile_name = ? AND chat_id = ?", - (profile_name, str(chat_id)), - ) - except sqlite3.OperationalError: - # Tables don't exist yet — nothing to disable. - return - self._execute_write(_do) - - def is_telegram_topic_mode_enabled( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - ) -> bool: - """Return whether Telegram DM topic mode is enabled for this chat/user.""" - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT enabled FROM telegram_dm_topic_mode - WHERE profile_name = ? AND chat_id = ? AND user_id = ? - """, - (profile_name, str(chat_id), str(user_id)), - ).fetchone() - except sqlite3.OperationalError: - return False - if row is None: - return False - enabled = row["enabled"] if isinstance(row, sqlite3.Row) else row[0] - return bool(enabled) - - def get_telegram_topic_binding( - self, - *, - chat_id: str, - thread_id: str, - profile_name: str = "default", - ) -> Optional[Dict[str, Any]]: - """Return the session binding for a Telegram DM topic, if present.""" - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT * FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? AND thread_id = ? - """, - (profile_name, str(chat_id), str(thread_id)), - ).fetchone() - except sqlite3.OperationalError: - return None - return dict(row) if row else None - - def list_telegram_topic_bindings_for_chat( - self, - *, - chat_id: str, - profile_name: str = "default", - ) -> List[Dict[str, Any]]: - """All Telegram DM topic bindings for one chat, newest first. - - Read-only; returns [] if the bindings table doesn't exist yet - (does not trigger the topic-mode migration). - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - with self._read_ctx() as conn: - try: - rows = conn.execute( - "SELECT * FROM telegram_dm_topic_bindings " - "WHERE profile_name = ? AND chat_id = ? " - "ORDER BY updated_at DESC", - (profile_name, str(chat_id)), - ).fetchall() - except sqlite3.OperationalError: - return [] - return [dict(row) for row in rows] - - def get_telegram_topic_binding_by_session( - self, - *, - session_id: str, - ) -> Optional[Dict[str, Any]]: - """Return the Telegram DM topic binding for a given session_id, if present. - - Uses the UNIQUE INDEX on telegram_dm_topic_bindings(session_id) for an - efficient reverse lookup. Returns None when the session has no binding or - the table does not exist yet. - """ - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT * FROM telegram_dm_topic_bindings - WHERE session_id = ? - """, - (str(session_id),), - ).fetchone() - except sqlite3.OperationalError: - return None - return dict(row) if row else None - - def delete_telegram_topic_binding( - self, - *, - chat_id: str, - thread_id: str, - profile_name: str = "default", - ) -> int: - """Remove the binding row for a single (chat, thread) pair. - - Called when the Telegram Bot API confirms a topic was deleted - externally (``Thread not found`` after the same-thread retry - already failed). Without this prune, the stale row keeps - living in ``telegram_dm_topic_bindings`` and the - recovery logic in ``gateway.run._recover_telegram_topic_thread_id`` - cheerfully redirects future inbound messages to the deleted - topic, causing tool progress, approvals, and replies to land - in the wrong place. Issue #31501. - - When this prune removes the chat's *last* remaining binding, - the chat's row in ``telegram_dm_topic_mode`` is also flipped to - ``enabled = 0`` in the same transaction. Otherwise the chat - would be left in topic mode with zero lanes — and - ``gateway.run._recover_telegram_topic_thread_id`` keeps treating - the chat as topic-enabled, lobby messages keep hunting for a - binding that no longer exists, and a user who disabled topics in - the Telegram client (rather than via ``/topic off``) stays stuck - until the next send happens to fail. Clearing the flag makes - recovery fully stand down once the dead topics are gone. - - Returns the number of binding rows deleted (0 when the binding - was already absent or the topic-mode tables haven't been - migrated yet — both are silent no-ops; we never raise from - a cleanup hot path). - """ - chat_id = str(chat_id) - thread_id = str(thread_id) - profile_name = _normalize_telegram_topic_profile_name(profile_name) - deleted = {"count": 0} - - def _do(conn): - try: - cursor = conn.execute( - """ - DELETE FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? AND thread_id = ? - """, - (profile_name, chat_id, thread_id), - ) - deleted["count"] = cursor.rowcount or 0 - except sqlite3.OperationalError: - # Tables don't exist yet — nothing to prune. - deleted["count"] = 0 - return - if not deleted["count"]: - return - # If that was the chat's last binding, disable topic mode for - # the chat so recovery stops steering lobby messages at a now - # empty lane set. Same transaction → no read-after-prune race. - try: - remaining = conn.execute( - """ - SELECT 1 FROM telegram_dm_topic_bindings - WHERE profile_name = ? AND chat_id = ? LIMIT 1 - """, - (profile_name, chat_id), - ).fetchone() - if remaining is None: - conn.execute( - "UPDATE telegram_dm_topic_mode " - "SET enabled = 0, updated_at = ? " - "WHERE profile_name = ? AND chat_id = ?", - (time.time(), profile_name, chat_id), - ) - except sqlite3.OperationalError: - # telegram_dm_topic_mode absent — binding prune still stands. - pass - - self._execute_write(_do) - return deleted["count"] - - def bind_telegram_topic( - self, - *, - chat_id: str, - thread_id: str, - user_id: str, - session_key: str, - session_id: str, - managed_mode: str = "auto", - profile_name: str = "default", - ) -> None: - """Bind one Telegram DM topic thread to one Hermes session. - - A Hermes session may only be linked to one Telegram topic in MVP. - Rebinding the same topic to the same session is idempotent; trying to - link the same session to a different topic raises ValueError. - """ - self.apply_telegram_topic_migration() - now = time.time() - chat_id = str(chat_id) - thread_id = str(thread_id) - user_id = str(user_id) - session_key = str(session_key) - session_id = str(session_id) - profile_name = _normalize_telegram_topic_profile_name(profile_name) - - def _do(conn): - existing_session = conn.execute( - """ - SELECT profile_name, chat_id, thread_id - FROM telegram_dm_topic_bindings - WHERE session_id = ? - """, - (session_id,), - ).fetchone() - if existing_session is not None: - if isinstance(existing_session, sqlite3.Row): - linked_profile = existing_session["profile_name"] - linked_chat = existing_session["chat_id"] - linked_thread = existing_session["thread_id"] - else: - linked_profile, linked_chat, linked_thread = existing_session - if ( - str(linked_profile) != profile_name - or str(linked_chat) != chat_id - or str(linked_thread) != thread_id - ): - raise ValueError("session is already linked to another Telegram topic") - - conn.execute( - """ - INSERT INTO telegram_dm_topic_bindings ( - profile_name, chat_id, thread_id, user_id, session_key, session_id, - managed_mode, linked_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET - user_id = excluded.user_id, - session_key = excluded.session_key, - session_id = excluded.session_id, - managed_mode = excluded.managed_mode, - updated_at = excluded.updated_at - """, - ( - profile_name, - chat_id, - thread_id, - user_id, - session_key, - session_id, - managed_mode, - now, - now, - ), - ) - self._execute_write(_do) - - def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool: - """Return True if a Hermes session is already bound to any Telegram DM topic. - - Read-only: does NOT trigger the telegram-topic migration. If the - topic-mode tables have not been created yet (i.e. nobody has run - ``/topic`` in this profile), the session is by definition unbound - and we return False. - """ - with self._read_ctx() as conn: - try: - row = conn.execute( - """ - SELECT 1 FROM telegram_dm_topic_bindings - WHERE session_id = ? - LIMIT 1 - """, - (str(session_id),), - ).fetchone() - except sqlite3.OperationalError: - return False - return row is not None - - def list_unlinked_telegram_sessions_for_user( - self, - *, - chat_id: str, - user_id: str, - profile_name: str = "default", - limit: int = 10, - ) -> List[Dict[str, Any]]: - """List previous Telegram sessions for this user that are not bound to a topic. - - Read-only: does NOT trigger the telegram-topic migration. If the - topic-mode tables are absent, fall back to a simpler query that - just returns this user's Telegram sessions — there can't be any - bindings yet. - - Scoped by ``profile_name`` so multiplexed profiles do not surface - each other's unlinked sessions (issue #76423). - """ - profile_name = _normalize_telegram_topic_profile_name(profile_name) - # sessions.profile_name is NULL/empty for legacy rows → treat as default. - profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?" - with self._read_ctx() as conn: - try: - rows = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - {_sql_session_last_active("s")} AS last_active - FROM sessions s - LEFT JOIN system_prompts sp - ON sp.hash = s.system_prompt_hash - WHERE s.source = 'telegram' - AND s.user_id = ? - {profile_clause} - AND NOT EXISTS ( - SELECT 1 FROM telegram_dm_topic_bindings b - WHERE b.session_id = s.id - ) - ORDER BY last_active DESC, s.started_at DESC - LIMIT ? - """, - (str(user_id), profile_name, int(limit)), - ).fetchall() - except sqlite3.OperationalError: - # telegram_dm_topic_bindings doesn't exist yet — no bindings - # means every telegram session for this user is "unlinked". - rows = conn.execute( - f""" - SELECT s.*, - COALESCE(sp.prompt, s.system_prompt) - AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, - {_sql_session_last_active("s")} AS last_active - FROM sessions s - LEFT JOIN system_prompts sp - ON sp.hash = s.system_prompt_hash - WHERE s.source = 'telegram' - AND s.user_id = ? - ORDER BY last_active DESC, s.started_at DESC - LIMIT ? - """, - (str(user_id), int(limit)), - ).fetchall() - - sessions: List[Dict[str, Any]] = [] - for row in rows: - session = self._session_row_dict(row) - session["preview"] = _shape_preview(session.pop("_preview_raw", "")) - sessions.append(session) - return sessions # ── Space reclamation ── @@ -16766,269 +6360,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # is present — so we probe each before touching it (see optimize_fts). _FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") - def logical_size_bytes(self) -> Optional[int]: - """Database size in bytes as SQLite itself accounts for it. - - ``page_count * page_size`` — the size the main DB file will have once - the WAL is checkpointed back into it. - - Prefer this over ``os.path.getsize(db_path)`` when reporting the effect - of a VACUUM. In WAL mode a VACUUM's rewrite lands in the ``-wal`` file, - and the checkpoint that folds it back is refused while any other - connection (a live gateway) holds a read-mark. Until that happens the - main file on disk still carries its pre-VACUUM size and keeps growing, - so a stat()-based before/after delta understates the win and can go - negative — the "reclaimed -3820.1 MB" report on a database that had - actually shrunk 60%. - - Returns None if the pragmas cannot be read. - """ - try: - with self._read_ctx() as conn: - if self._conn is None: - return None - page_count = conn.execute("PRAGMA page_count").fetchone()[0] - page_size = conn.execute("PRAGMA page_size").fetchone()[0] - return int(page_count) * int(page_size) - except Exception as exc: - logger.debug("Could not read logical DB size: %s", exc) - return None - - def _freelist_ratio(self) -> Optional[float]: - """Fraction of database pages that are on the freelist (reclaimable). - - ``PRAGMA freelist_count / PRAGMA page_count`` read over the existing - connection (never a byte-level probe of the live file — see - ``sqlite_safe_read``). This is what VACUUM would actually give back; - it is the gate :meth:`maybe_auto_prune_and_vacuum` uses to decide - whether a full rewrite pays off (#54189). - - Returns None if the pragmas cannot be read (callers treat that as - "unknown" and fall back to the time throttle alone). - """ - try: - with self._read_ctx() as conn: - if self._conn is None: - return None - page_count = int(conn.execute("PRAGMA page_count").fetchone()[0]) - freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0]) - if page_count <= 0: - return 0.0 - return freelist / page_count - except Exception as exc: - logger.debug("Could not read freelist ratio: %s", exc) - return None - - def vacuum(self) -> int: - """Run VACUUM to reclaim disk space after large deletes. - - SQLite does not shrink the database file when rows are deleted — - freed pages just get reused on the next insert. After a prune that - removed hundreds of sessions, the file stays bloated unless we - explicitly VACUUM. - - VACUUM rewrites the entire DB, so it's expensive (seconds per - 100MB) and cannot run inside a transaction. It also acquires an - exclusive lock, so callers must ensure no other writers are - active. Safe to call at startup before the gateway/CLI starts - serving traffic. - - FTS5 segments are merged first via :meth:`optimize_fts` so the - subsequent VACUUM reclaims the pages freed by the merge. This is a - layout-only optimization — search results are unchanged. - - Returns the number of FTS indexes that were optimized (0 if the - merge step failed or no FTS tables exist). - """ - # Merge FTS5 segments before VACUUM so the freed pages are returned - # to the OS in the same pass. optimize_fts() manages its own lock. - optimized = 0 - try: - optimized = self.optimize_fts() - except Exception as exc: - logger.warning("FTS optimize before VACUUM failed: %s", exc) - # VACUUM cannot be executed inside a transaction. - with self._lock: - # Best-effort WAL checkpoint first, then VACUUM. PASSIVE, not - # TRUNCATE: a manual `hermes sessions vacuum` runs in a transient - # CLI process, and a TRUNCATE reset here would race a live gateway - # writer and tear B-tree pages (#45383). VACUUM folds the WAL back - # itself; journal_size_limit bounds the file. - try: - self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") - except Exception as exc: - logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc) - self._conn.execute("VACUUM") - # ...and again afterwards. VACUUM rewrites every page THROUGH the - # WAL, so the pre-VACUUM checkpoint above does nothing for the - # slack VACUUM itself creates: on a 3.0 GB database it left a - # 3.07 GB state.db-wal behind, so `sessions optimize` reported - # "reclaimed -11.2 MB" while actually consuming 3 GB of disk and - # filling the host to 100%. Truncating here is what makes the - # command a net win instead of a net loss on large databases. - try: - self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") - except Exception as exc: - logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc) - # TRUNCATE may replace the WAL inode; adopt the post-VACUUM - # sidecars so the write-path generation guard does not halt a - # healthy exclusive maintenance connection. - self._record_db_file_identity() - return optimized - - def maybe_auto_prune_and_vacuum( - self, - retention_days: int = 90, - min_interval_hours: int = 24, - vacuum: bool = True, - sessions_dir: Optional[Path] = None, - min_vacuum_interval_days: int = 30, - min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO, - ) -> Dict[str, Any]: - """Idempotent auto-maintenance: prune inactive sessions + optional VACUUM. - - Records the last run timestamp in state_meta so subsequent calls - within ``min_interval_hours`` no-op. VACUUM has its own, typically - longer, throttle controlled by ``min_vacuum_interval_days`` so routine - pruning does not repeatedly rewrite the database, and is additionally - gated on the reclaimable fraction of the file: it only runs when - ``PRAGMA freelist_count / PRAGMA page_count`` exceeds - ``min_vacuum_freelist_ratio`` (default - :data:`AUTO_VACUUM_MIN_FREELIST_RATIO`, 25%), so pruning a few small - sessions on a dense multi-GB database never triggers a full rewrite - (#54189). Designed to be called once at startup from long-lived - entrypoints (CLI, gateway, cron scheduler). - - When *sessions_dir* is provided, on-disk transcript files - (``.json`` / ``.jsonl`` / ``request_dump_*``) for pruned sessions - are removed as part of the same sweep (issue #3015). - - Stale-open reconciliation (#54189): several state-owned producers - (cron, kanban workers, subagents, one-shot CLI runs) never set - ``ended_at`` when their process dies, and ``prune_sessions`` only - deletes ended rows — so retention was a no-op exactly where growth - concentrates. After pruning, this pass closes open rows from - :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` whose activity is older than - ``retention_days`` (``end_reason='startup_orphan_reap'``). Closed rows - stay resumable and are aged from their close, so they get one more - full retention window before a later pass deletes them. Messaging - and UI sources are never touched here. - - Never raises. On any failure, logs a warning and returns a dict - with ``"error"`` set. - - Returns a dict with keys: - - ``"skipped"`` (bool) — true if within min_interval_hours of last run - - ``"pruned"`` (int) — number of sessions deleted - - ``"closed"`` (int) — stale open state-owned sessions marked ended - - ``"vacuumed"`` (bool) — true if VACUUM ran - - ``"freelist_ratio"`` (float|None) — reclaimable fraction measured - when a VACUUM was considered (absent when it was not) - - ``"error"`` (str, optional) — present only on failure - """ - result: Dict[str, Any] = { - "skipped": False, - "pruned": 0, - "closed": 0, - "vacuumed": False, - } - maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path) - if maintenance_lock is None: - result["skipped"] = True - return result - try: - # Skip if another process/call did maintenance recently. - last_raw = self.get_meta("last_auto_prune") - now = time.time() - if last_raw: - try: - last_ts = float(last_raw) - if now - last_ts < min_interval_hours * 3600: - result["skipped"] = True - return result - except (TypeError, ValueError): - pass # corrupt meta; treat as no prior run - - # Delete only sessions that were already explicitly closed. A - # startup orphan discovered by this pass is closed *after* pruning, - # preserving a full retention window in which it can be resumed. - pruned = self.prune_sessions( - older_than_days=retention_days, - sessions_dir=sessions_dir, - exclude_active_write_guards=True, - ) - result["pruned"] = pruned - - # Reap stale state-owned rows only. Runtime-owned messaging sources - # are intentionally outside this automatic destructive scope. - closed = self.sweep_orphaned_sessions( - max_idle_seconds=float(retention_days) * 86400.0, - sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, - exclude_pinned=True, - # These sources are owned by state.db lifecycles, not by the - # dashboard/TUI gateway heartbeats used by startup recovery. - respect_gateway_heartbeats=False, - ) - result["closed"] = len(closed) - # Only VACUUM if we actually freed rows, no more often than once - # every min_vacuum_interval_days, AND only when the rewrite pays - # off: the reclaimable fraction of the file (freelist_count / - # page_count) must exceed AUTO_VACUUM_MIN_FREELIST_RATIO (#54189). - # A large prune (e.g. the first one to cross retention_days on a - # DB with tens of thousands of rows) can free enough pages that - # pruned > 0 fires on every subsequent startup even though a - # VACUUM already ran recently; and pruning one tiny session on a - # dense multi-GB DB would otherwise rewrite the whole file to - # reclaim a few MB. VACUUM on this DB's size (FTS5 shadow tables) - # is not cheap -- it holds an exclusive lock for the full rewrite. - # The time throttle says "not too often"; the ratio gate says - # "only when it pays off". Both must pass. - last_vacuum_raw = self.get_meta("last_vacuum") - vacuum_due = True - if last_vacuum_raw: - try: - vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400 - except (TypeError, ValueError): - vacuum_due = True - if vacuum and pruned > 0 and vacuum_due: - ratio = self._freelist_ratio() - result["freelist_ratio"] = ratio - if ratio is None or ratio > min_vacuum_freelist_ratio: - try: - self.vacuum() - result["vacuumed"] = True - self.set_meta("last_vacuum", str(now)) - except Exception as exc: - logger.warning("state.db VACUUM failed: %s", exc) - else: - logger.debug( - "state.db auto-maintenance: skipping VACUUM, only " - "%.1f%% of pages reclaimable (threshold %.0f%%)", - ratio * 100.0, - min_vacuum_freelist_ratio * 100.0, - ) - - # Record the attempt even if pruned == 0, so we don't retry - # every startup within the min_interval_hours window. - self.set_meta("last_auto_prune", str(now)) - - if closed or pruned > 0: - logger.info( - "state.db auto-maintenance: closed %d stale open session(s), " - "pruned %d session(s) inactive for %d days%s", - len(closed), - pruned, - retention_days, - " + VACUUM" if result["vacuumed"] else "", - ) - except Exception as exc: - # Maintenance must never block startup. Log and return error marker. - logger.warning("state.db auto-maintenance failed: %s", exc) - result["error"] = str(exc) - finally: - _release_auto_maintenance_lock(maintenance_lock) - - return result def maybe_auto_archive( self, @@ -17095,172 +6426,6 @@ class SessionDB(SessionSearchMixin, SessionSchemaMixin, SessionPortabilityMixin) # The CLI writes "pending" then poll-waits for terminal state. The gateway # watcher transitions pending→running→{completed,failed}. - def request_handoff(self, session_id: str, platform: str) -> bool: - """Mark a session as pending handoff to the given platform. - - Returns True if the row was found and not already in flight; False if - the session is already in a non-terminal handoff state. - """ - def _do(conn): - cur = conn.execute( - "UPDATE sessions " - "SET handoff_state = 'pending', " - " handoff_platform = ?, " - " handoff_error = NULL " - "WHERE id = ? AND (handoff_state IS NULL " - " OR handoff_state IN ('completed', 'failed'))", - (platform, session_id), - ) - return cur.rowcount > 0 - return self._execute_write(_do) - - def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]: - """Read the current handoff state for a session. - - Returns ``{"state", "platform", "error"}`` or None if the session has - no handoff record. - """ - try: - with self._read_ctx() as conn: - row = conn.execute( - "SELECT handoff_state, handoff_platform, handoff_error " - "FROM sessions WHERE id = ?", - (session_id,), - ).fetchone() - if not row: - return None - return { - "state": row["handoff_state"], - "platform": row["handoff_platform"], - "error": row["handoff_error"], - } - except Exception: - return None - - def list_pending_handoffs(self) -> List[Dict[str, Any]]: - """Return all sessions in handoff_state='pending', oldest first. - - Used by the gateway's handoff watcher. - """ - try: - with self._read_ctx() as conn: - rows = conn.execute( - "SELECT s.*, " - "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " - "FROM sessions s " - "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " - "WHERE s.handoff_state = 'pending' " - "ORDER BY s.started_at ASC" - ).fetchall() - return [self._session_row_dict(r) for r in rows] - except Exception: - return [] - - def claim_handoff(self, session_id: str) -> bool: - """Atomically transition pending → running. Returns True if claimed.""" - def _do(conn): - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'running' " - "WHERE id = ? AND handoff_state = 'pending'", - (session_id,), - ) - return cur.rowcount > 0 - return self._execute_write(_do) - - def complete_handoff(self, session_id: str) -> None: - """Mark a handoff as completed.""" - def _do(conn): - conn.execute( - "UPDATE sessions SET handoff_state = 'completed', " - "handoff_error = NULL WHERE id = ?", - (session_id,), - ) - self._execute_write(_do) - - def fail_handoff( - self, - session_id: str, - error: str, - *, - only_states: Optional[Tuple[str, ...]] = None, - ) -> bool: - """Mark a handoff as failed and record the reason. - - ``only_states`` makes the write a compare-and-swap: the row is only - failed when its current ``handoff_state`` is in the given tuple. - Waiters that give up (CLI 60s poll, Desktop bounded poll) MUST pass - ``only_states=("pending",)`` — once the gateway watcher has claimed - the row (``running``) it owns the terminal state, and a waiter-side - unconditional fail races the dispatch: the gateway later overwrites - ``failed`` → ``completed`` while the user was already told the - gateway is down (split-brain — the handoff actually delivered and - ``switch_session`` re-pointed the session). - - The gateway watcher itself fails its OWN claimed row unconditionally - (no ``only_states``) — it is the owner while the row is ``running``. - - Returns True when a row was transitioned to ``failed``. - """ - def _do(conn): - if only_states: - placeholders = ", ".join("?" for _ in only_states) - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - f"handoff_error = ? WHERE id = ? AND handoff_state IN ({placeholders})", - (error[:500], session_id, *only_states), - ) - else: - cur = conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - "handoff_error = ? WHERE id = ?", - (error[:500], session_id), - ) - return cur.rowcount > 0 - return bool(self._execute_write(_do)) - - def reclaim_stale_running_handoffs(self, error: str) -> List[str]: - """Fail every handoff stuck in ``running``. Returns the ids reclaimed. - - Only the gateway's watcher ever sets ``running``, and it does so for - the duration of a single in-process dispatch. So any row still in - ``running`` when a watcher starts up belongs to a PREVIOUS gateway - that died mid-dispatch (crash, kill, machine reboot). - - Such a row is unrecoverable *and* poisonous: ``request_handoff`` only - accepts a new request when the state is NULL/``completed``/``failed``, - so a stranded ``running`` row makes that session permanently unable to - hand off again — with no error surfaced anywhere. - - Failing (rather than re-queueing as ``pending``) is deliberate: the - dead gateway may have already switched the session key and dispatched - the synthetic turn before dying, so a blind retry risks double - delivery. The user's CLI has long since timed out; the right outcome - is a clean terminal state they can retry from explicitly. - """ - def _do(conn): - cur = conn.execute( - "SELECT id FROM sessions WHERE handoff_state = 'running'" - ) - ids = [r[0] for r in cur.fetchall()] - if ids: - conn.execute( - "UPDATE sessions SET handoff_state = 'failed', " - "handoff_error = ? WHERE handoff_state = 'running'", - (error[:500],), - ) - return ids - try: - return self._execute_write(_do) or [] - except Exception: - # Swallow but never silently: a persistently failing reclaim - # leaves poisonous 'running' rows in place (sessions that can - # never hand off again), so the operator needs a trace of it. - logger.warning( - "reclaim_stale_running_handoffs failed; stranded 'running' " - "handoff rows (if any) were left in place", exc_info=True, - ) - return [] - class AsyncSessionDB: """Async door onto SessionDB: offloads each call via asyncio.to_thread so a blocking SQLite call never freezes the event loop. Generic forwarder — the audit confirms no method returns a live cursor/generator.""" diff --git a/hermes_state_common.py b/hermes_state_common.py index af12d322d3..88faf2e3ba 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -1,9 +1,7 @@ -"""Shared module-level constants for the SessionDB family of modules. +"""Shared constants and helpers for the SessionDB family of modules. -Extracted verbatim from hermes_state.py so the SessionDB mixin modules -(hermes_state_search / hermes_state_schema / hermes_state_portability) can -reference them without importing hermes_state (which would be a cycle). -hermes_state re-imports every name here for backward compatibility. +Lives outside hermes_state so the mixin modules can import it without a +cycle; hermes_state re-exports every name for backward compatibility. """ import contextlib @@ -29,16 +27,11 @@ from agent.context_compressor import ( ) -# Session preview = the head of the first user message, shown wherever a -# session has no title (sidebar rows, pickers, exports, the desktop's -# `sessionTitle` fallback). -# -# A /skill invocation expands into a message that embeds the whole skill body, -# so the plain head of it previews the SKILL's opening prose as if the user had -# written it. Scaffolded rows therefore carry a wider excerpt so -# ``_shape_preview`` can hand it to ``describe_skill_invocation`` and recover -# ``/work — fix the title leak``: the whole message while it stays under the -# budget, and head + tail (where the typed instruction lands) once it doesn't. +# Session preview = head of the first user message, shown wherever a session +# has no title. A /skill invocation embeds the whole skill body, so its plain +# head would preview the SKILL's prose; scaffolded rows carry a wider excerpt +# (whole message under budget, else head + tail where the typed instruction +# lands) so ``_shape_preview`` can recover ``/work — fix the title leak``. _PREVIEW_HEAD_CHARS = 63 @@ -49,13 +42,9 @@ _PREVIEW_MAX_CHARS = 60 def escape_like(text: str) -> str: - """Escape SQL LIKE wildcards so operator/session-derived text matches - literally. Pair with ``ESCAPE '\\'`` in the clause. - - ``%`` and ``_`` are wildcards to LIKE, and ``_`` in particular is common - in the values these patterns run against (branch names, session titles, - filesystem paths). A match documented as substring/prefix must not - silently widen. + """Escape LIKE wildcards (``%``, ``_``) so derived text matches literally; + pair with ``ESCAPE '\\'``. ``_`` is common in branch names, titles and + paths, and a documented substring/prefix match must not silently widen. """ return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -90,10 +79,9 @@ def _sql_starts_with(expression: str, prefixes: tuple[str, ...]) -> str: return "(" + " OR ".join(checks) + ")" -# Current and historical long-form prefixes share this complete introduction; -# their stale-item guidance diverges only after it. Matching the whole intro -# avoids treating an ordinary user message that merely starts with the short -# bracketed label as a compaction carrier. +# Current and legacy long-form prefixes share this whole introduction; matching +# all of it keeps an ordinary message that merely starts with the bracketed +# label from counting as a compaction carrier. _PREVIEW_LONG_FORM_PREFIX = SUMMARY_PREFIX.split("Do NOT answer", 1)[0] _PREVIEW_SUMMARY_PREFIXES = ( _PREVIEW_LONG_FORM_PREFIX, @@ -127,9 +115,8 @@ _PREVIEW_FORCE_USER_REMAINDER_SQL = ( f" + {len(_SUMMARY_END_MARKER)})" ) -# Session preview subqueries select their first eligible user-authored content. -# Pure compaction rows are ineligible; force-user-leading and merged carriers -# remain eligible only when authentic content survives the wire boundary. +# Pure compaction rows are ineligible for previews; force-user-leading and +# merged carriers are eligible only when authentic content survives. _PREVIEW_ELIGIBLE_SQL = ( f"((NOT {_PREVIEW_STANDALONE_SUMMARY_SQL} AND NOT {_PREVIEW_MERGED_SUMMARY_SQL})" f" OR ({_PREVIEW_STANDALONE_SUMMARY_SQL}" @@ -140,10 +127,8 @@ _PREVIEW_ELIGIBLE_SQL = ( ) -# The shared ``_preview_raw`` SELECT expression, interpolated by every listing -# query. A scaffolded row gets a wider excerpt: the whole message while it fits -# the budget, else head + tail (where the typed instruction lands) spliced -# around SKILL_EXCERPT_JOINT. +# Shared ``_preview_raw`` SELECT expression for every listing query (scaffolded +# rows: head + tail spliced around SKILL_EXCERPT_JOINT when over budget). _PREVIEW_RAW_SELECT = ( f"CASE WHEN {_PREVIEW_STANDALONE_SUMMARY_SQL}" f" THEN {_PREVIEW_FORCE_USER_REMAINDER_SQL}" @@ -173,8 +158,17 @@ def _shape_preview(raw: Any) -> str: return text -# A child session counts as a /branch (kept visible, never cascade-deleted) if -# it carries the stable marker OR the legacy end_reason heuristic holds. +# Correlated ``_preview_raw`` column for a ``sessions s`` row. +_PREVIEW_RAW_SUBQUERY_SQL = ( + f"COALESCE((SELECT {_PREVIEW_RAW_SELECT} FROM messages m" + f" WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL" + f" AND {_PREVIEW_ELIGIBLE_SQL}" + f" ORDER BY m.timestamp, m.id LIMIT 1), '') AS _preview_raw" +) + + +# A /branch child (kept visible, never cascade-deleted): stable marker OR the +# legacy end_reason heuristic. _BRANCH_CHILD_SQL = ( "json_extract(COALESCE({a}.model_config, '{{}}'), '$._branched_from') IS NOT NULL" " OR EXISTS (SELECT 1 FROM sessions p" @@ -193,12 +187,10 @@ _COMPRESSION_CHILD_SQL = ( _RESET_END_REASONS = ( "session_reset", - # switch_session() never creates a child row, but pre-marker DBs can hold - # legacy reset children whose parent later ended with 'session_switch' - # (resumed then switched away before reopen-time stamping existed). Also - # keeps this set identical to the recovery fence in - # find_latest_gateway_session_for_peer, which interpolates - # _RESET_END_REASONS_SQL so the two cannot drift. + # switch_session() creates no child row, but pre-marker DBs hold legacy + # reset children whose parent later ended 'session_switch'. Also keeps + # this set identical to the recovery fence in + # find_latest_gateway_session_for_peer (which interpolates the SQL form). "session_switch", "idle", "daily", @@ -207,34 +199,25 @@ _RESET_END_REASONS = ( ) _RESET_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RESET_END_REASONS) -# Accidental end reasons that recovery treats as resumable (see -# docs/session-lifecycle.md "recoverable accidental reasons"). Interpolated -# into the recovery SQL below AND exposed as SessionDB.RECOVERABLE_END_REASONS -# so the tuple is the single source of truth — literals cannot drift. +# Accidental end reasons recovery treats as resumable (docs/session-lifecycle.md). +# Single source of truth: interpolated into recovery SQL AND exposed as +# SessionDB.RECOVERABLE_END_REASONS. _RECOVERABLE_END_REASONS = ( "agent_close", "ws_orphan_reap", - # A stale sentinel-parked runtime quietly superseded by a fresh - # session.resume of the same stored session (no reclaimed broadcast); - # the stored session stays resumable like any accidental end. + # Stale sentinel-parked runtime superseded by a fresh session.resume. "superseded_by_resume", - # Startup sweep of rows orphaned by a dead gateway process (#65194): - # the in-process ws-orphan grace timer died with the process, so the - # row was closed at the next boot instead. Same accident class as - # ws_orphan_reap — kept distinct for forensics — and equally resumable. + # Startup sweep of rows orphaned by a dead gateway process: same accident + # class as ws_orphan_reap, kept distinct for forensics. "startup_orphan_reap", ) _RECOVERABLE_END_REASONS_SQL = ", ".join(f"'{reason}'" for reason in _RECOVERABLE_END_REASONS) -# End reasons written by AUTOMATIC infrastructure cleanup (server shutdown, -# orphan reapers, idle/LRU eviction) rather than by a deliberate conversation -# boundary (compression, session_reset, session_switch, explicit user close). -# An automatic stamp records "some runtime went away", NOT "this conversation -# ended" — so a writer that can prove the conversation is still live (e.g. an -# active compression rotation holding the lease, #88197) may treat the stamp -# as stale and clear it. Superset of the recoverable set: those are already -# resumable accidents; the extra TUI reasons are the same accident class but -# were historically only known to tui_gateway's _AUTOMATIC_SESSION_END_REASONS. +# End reasons written by AUTOMATIC cleanup (shutdown, orphan reapers, idle/LRU +# eviction), not by a deliberate conversation boundary. Such a stamp means +# "some runtime went away", NOT "this conversation ended", so a writer that can +# prove liveness (e.g. a compression rotation holding the lease) may clear it. +# Superset of the recoverable set plus the TUI gateway's automatic reasons. _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | { "tui_shutdown", "ws_disconnect", @@ -244,23 +227,17 @@ _AUTOMATIC_END_REASONS = frozenset(_RECOVERABLE_END_REASONS) | { def is_automatic_end_reason(reason) -> bool: - """True when *reason* is an automatic-cleanup end stamp (see above). - - Single owner of the "accidental vs deliberate end" predicate — every - compression-liveness site must call this instead of re-implementing the - reason taxonomy (#88197, never-patch-predicates). - """ + """True when *reason* is an automatic-cleanup end stamp (see above). Single + owner of the accidental-vs-deliberate predicate; compression-liveness + sites must call this rather than re-implement the taxonomy.""" return isinstance(reason, str) and reason in _AUTOMATIC_END_REASONS def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str: - """Pre-marker reset-continuation heuristic. - - A child is a legacy reset continuation when it rides its parent's exact - non-empty routing key and the parent ended at a reset boundary. Shared by - the listing predicate (``_RESET_CHILD_SQL``) and ``reopen_session()``'s - marker-stamping UPDATE so the two sites cannot drift; ``reasons_sql`` is - either the literal ``_RESET_END_REASONS_SQL`` or a bound-placeholder list. + """Pre-marker reset-continuation heuristic: child rides its parent's exact + non-empty routing key and the parent ended at a reset boundary. Shared by + ``_RESET_CHILD_SQL`` and ``reopen_session()``'s marker-stamping UPDATE so + the two cannot drift; ``reasons_sql`` is a literal or placeholder list. """ return ( f"EXISTS (SELECT 1 FROM sessions p" @@ -272,19 +249,17 @@ def _legacy_reset_child_sql(alias: str, reasons_sql: str) -> str: ) -# A reset starts a separate user-visible conversation even though gateway rows -# retain parent_session_id for durable lineage. New rows carry the stable -# marker; the same-key fallback recovers rows written before the marker existed. -# Requiring the exact non-empty routing key keeps ordinary child/subagent rows -# out even when their parent is later reset. +# A reset starts a separate user-visible conversation though rows keep +# parent_session_id for lineage. Stable marker, or the same-key fallback for +# pre-marker rows (the exact-key requirement keeps subagent children out). _RESET_CHILD_SQL = ( "json_extract(COALESCE({a}.model_config, '{{}}'), '$._reset_from') IS NOT NULL" " OR " + _legacy_reset_child_sql("{a}", _RESET_END_REASONS_SQL) ) -# Rows that surface in pickers: roots + branch/reset children. Subagent runs -# and compression continuations stay hidden. +# Picker-visible rows: roots + branch/reset children (not subagent runs or +# compression continuations). _LISTABLE_CHILD_SQL = ( f"(s.parent_session_id IS NULL OR {_BRANCH_CHILD_SQL.format(a='s')}" f" OR {_RESET_CHILD_SQL.format(a='s')})" @@ -305,14 +280,9 @@ def _ephemeral_child_sql(alias: str = "s") -> str: def _sql_session_last_active(alias: str = "s") -> str: - """SQL expression for session recency used by list/status surfaces. - - Freshest of ``last_activity_at`` (mid-turn agent activity heartbeat) and - the latest message timestamp, then fall back to ``started_at``. - - Must not prefer a stale heartbeat over a newer message: durable - heartbeats are rate-limited (~60s), so after a turn writes messages - ``last_activity_at`` can lag ``MAX(messages.timestamp)``. + """Session recency: freshest of ``last_activity_at`` and the latest message + timestamp, else ``started_at``. Heartbeats are rate-limited (~60s) so + ``last_activity_at`` can lag a newer message; never prefer it alone. """ msg_max = ( f"(SELECT MAX(_act_m.timestamp) FROM messages _act_m " @@ -356,24 +326,38 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: SCHEMA_VERSION = 28 +# Auto-maintenance VACUUMs only when at least this fraction of pages is on the +# freelist; below it a full rewrite costs more I/O than it returns. +AUTO_VACUUM_MIN_FREELIST_RATIO = 0.25 -# FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the -# state_meta key ``fts_storage_version``. The main schema version advances -# freely on open (so future migrations always land); the FTS *layout* only -# reaches the current version when a DB is either born fresh or explicitly -# optimized via ``hermes sessions optimize-storage``. A legacy DB sits at -# layout 0 (marker absent) with a working inline index until the user opts in. -# 1 = v23 external-content layout (content/tool_name/tool_calls, -# tool-row-excluded trigram) + +# FTS storage layout, tracked INDEPENDENTLY of SCHEMA_VERSION (state_meta +# ``fts_storage_version``): the schema version advances freely on open, but +# the FTS layout only changes when a DB is born fresh or explicitly optimized +# via ``hermes sessions optimize-storage``. Legacy DBs sit at 0 (marker +# absent) with a working inline index. 1 = v23 external-content layout. FTS_STORAGE_VERSION = 1 -# Cap on user-controlled FTS5 query input before regex/sanitizer processing. -# Search queries do not need to be arbitrarily large, and bounding them keeps -# sanitizer/runtime behavior predictable under adversarial input. +# Cap on user-controlled FTS5 query input before sanitizer processing. MAX_FTS5_QUERY_CHARS = 2_048 +# ── Helpers shared by SessionDB, its mixins and the registry ────────────── + +def stat_db_file_identity(path) -> "tuple[int, int] | None": + """``(st_dev, st_ino)`` for *path*, or None. st_ino=0 (Windows, some + network FS) would false-positive every replaced-file check, so it counts + as unknown.""" + try: + st = os.stat(path) + except OSError: + return None + if not st.st_dev or not st.st_ino: + return None + return (st.st_dev, st.st_ino) + + _FTS_TRIGGERS = ( "messages_fts_insert", "messages_fts_delete", @@ -628,10 +612,8 @@ CREATE INDEX IF NOT EXISTS idx_async_delegations_delivery """ -# Indexes that reference columns added in later schema versions must be -# created AFTER _reconcile_columns() has had a chance to ADD them on -# existing databases. SCHEMA_SQL above is run by sqlite executescript -# which would otherwise fail on legacy DBs ("no such column: active"). +# Indexes on columns added in later schema versions must run AFTER +# _reconcile_columns() adds them, or executescript fails on legacy DBs. DEFERRED_INDEX_SQL = """ CREATE INDEX IF NOT EXISTS idx_messages_session_active ON messages(session_id, active, timestamp); @@ -648,25 +630,17 @@ CREATE INDEX IF NOT EXISTS idx_sessions_system_prompt_hash """ -# ── Deferred FTS rebuild bookkeeping (schema v23) ── -# While a background index rebuild is pending, two state_meta keys define -# which message rows are currently IN the FTS indexes: +# ── Deferred FTS rebuild bookkeeping ── +# While a background rebuild is pending, two state_meta keys define which rows +# are IN the FTS indexes: H = fts_rebuild_high_water (MAX(messages.id) when the +# old indexes were dropped), P = fts_rebuild_progress (highest backfilled id). +# A row is indexed iff id <= P OR id > H (AUTOINCREMENT ids, so post-drop rows +# are indexed live by the insert triggers); rows in (P, H] are not. # -# fts_rebuild_high_water H — MAX(messages.id) at the moment the old -# indexes were dropped -# fts_rebuild_progress P — highest id the chunked backfill has indexed -# -# A row is indexed iff id <= P (backfilled) OR id > H (inserted after -# the drop; ids are AUTOINCREMENT so new rows are always > H and the insert -# triggers index them live). Rows in (P, H] are not yet indexed. -# -# Every trigger below gates on that same predicate: firing an FTS5 -# external-content 'delete' for a row that is NOT in the index corrupts the -# index, and skipping it for a row that IS indexed leaves a stale entry. -# When no rebuild is pending both keys are absent and COALESCE turns the -# predicate into a tautology (id > -1 OR id <= -1), i.e. normal operation. -# The two state_meta PK probes per write are negligible next to the FTS -# insert itself. +# Every trigger gates on that predicate: an FTS5 external-content 'delete' for +# a row NOT in the index corrupts it, and skipping one for an indexed row +# leaves a stale entry. With no rebuild pending both keys are absent and +# COALESCE makes the predicate a tautology. FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content, @@ -717,20 +691,12 @@ END; """ -# Trigram FTS5 table for CJK substring search. The default unicode61 -# tokenizer splits CJK characters into individual tokens, breaking phrase -# matching. The trigram tokenizer creates overlapping 3-byte sequences so -# substring queries work natively for any script (CJK, Thai, etc.). -# -# The trigram index is the most expensive index in state.db (~2.6x the size -# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes -# while being almost entirely machine noise (base64 payloads, file dumps, -# delegation transcripts). The index therefore reads through -# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay -# fully stored in ``messages`` and fully searchable via the standard -# ``messages_fts`` index; they just don't get trigram (CJK substring) -# treatment. ``search_messages`` routes CJK queries that filter on -# ``role='tool'`` to the LIKE fallback for the same reason. +# Trigram FTS5 table for CJK substring search (unicode61 splits CJK into single +# tokens, breaking phrase matching). The trigram index is ~2.6x the text it +# covers and ``role='tool'`` rows are ~90% of message bytes of machine noise, +# so it reads through the ``messages_fts_trigram_src`` view, which excludes +# tool rows; those remain searchable via ``messages_fts``, and +# ``search_messages`` routes CJK queries filtered on role='tool' to LIKE. FTS_TRIGRAM_SQL = """ CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS SELECT id, role, content, tool_name, tool_calls @@ -796,18 +762,15 @@ _FTS_CJK_TRIGGERS = ( ) -# state_meta breadcrumb set when a tokenizer-less process had to drop the -# cjk triggers to keep message writes alive: rows written from that moment -# on are missing from the cjk index, so it must not serve reads until +# Set when a tokenizer-less process dropped the cjk triggers to keep writes +# alive: the cjk index is missing rows and must not serve reads until # `hermes sessions optimize-storage` rebuilds it on a capable host. FTS_CJK_STALE_KEY = "fts_cjk_stale" -# Durable breadcrumb for a base/trigram FTS index that was detached from the -# canonical messages table after runtime corruption. While present, startup -# must rebuild the complete index before reinstalling sync triggers: rows may -# have been written while those triggers were absent, so merely recreating -# them would preserve an unknown index gap. +# Set when a base/trigram FTS index was detached after runtime corruption. +# While present, startup must rebuild the complete index before reinstalling +# sync triggers: rows written while they were absent leave an unknown gap. FTS_STALE_KEY = "fts_stale" # Durable diagnostic for stale FTS recovery blocked across process restarts. @@ -815,16 +778,11 @@ FTS_REBUILD_DEFERRAL_KEY = "fts_rebuild_deferral" # ── Legacy (v22 / inline-content) FTS DDL ────────────────────────────── -# Used ONLY to keep an existing pre-v23 install's search working and its -# triggers repairable UNTIL the user opts into `hermes db optimize`. This is -# the exact inline shape v11..v22 shipped: each virtual table stores its own -# copy of ``content || tool_name || tool_calls`` and the trigram table indexes -# every row (including role='tool'). We never CREATE these on a fresh install — -# fresh installs are born on the v23 external-content schema above. These -# constants exist so a legacy DB is never accidentally handed the v23 DDL -# (which would create the external-content trigram source VIEW and leave the -# DB in a mixed, broken state). `optimize_fts_storage()` is what migrates a -# legacy DB to the v23 shape. +# Used ONLY to keep a pre-v23 install's search working and its triggers +# repairable until `optimize_fts_storage()` migrates it: inline copies of +# content || tool_name || tool_calls, trigram over every row. Never created +# on a fresh install. Handing a legacy DB the v23 DDL would create the +# external-content trigram VIEW and leave it in a mixed, broken state. LEGACY_FTS_SQL = """ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( content @@ -882,39 +840,25 @@ END; # ── Cross-process full-FTS-rebuild admission (single authority) ────────────── # -# Several independent Hermes processes routinely share one state.db (gateway -# service, the Desktop app's `hermes serve` backend, interactive CLI sessions, -# the TUI slash worker). A full structural FTS rebuild — the FTS5 'rebuild' -# command or the drop/recreate script in `_recover_stale_fts` — must only ever -# run in ONE of them at a time: two concurrent rebuilds collide on write and -# have structurally corrupted state.db in production (PR #93200; the -# 2026-08-15 / 2026-08-23 incidents and issues #89293 / #90950). +# Several Hermes processes share one state.db. A full structural FTS rebuild +# (FTS5 'rebuild' or the drop/recreate in `_recover_stale_fts`) must run in +# ONE of them at a time: concurrent rebuilds have structurally corrupted +# state.db in production. This is the single admission authority for +# `rebuild_fts()`, `_rebuild_fts_indexes()` and `_recover_stale_fts()`. The +# chunked backfill (`fts_rebuild_step`) is deliberately NOT routed through it: +# it claims progress under SQLite transaction authority and is multi-process. # -# This is the single admission authority for every full structural rebuild -# entry point: `SessionSearchMixin.rebuild_fts()`, -# `SessionSchemaMixin._rebuild_fts_indexes()` (via `_init_schema`), and -# `SessionSchemaMixin._recover_stale_fts()`. The chunked deferred backfill -# (`fts_rebuild_step`) is deliberately NOT routed through it — it claims -# progress under `_execute_write`'s SQLite transaction authority and is -# intentionally multi-process. +# Semantics mirror `hermes_state._cross_process_repair_lock`: portable (msvcrt +# on Windows, flock elsewhere), bounded wait, FAIL CLOSED. The kernel drops +# the lock when the holder dies UNLESS a forked child inherited the fd (flock +# rides the open file description), which holds it forever; so the holder's +# pid + start time are recorded under the lock and a provably-dead holder's +# lock is broken by unlinking and retaking on a fresh inode. Indeterminate +# liveness still defers. Lives here because the mixins cannot import +# hermes_state (cycle). # -# Semantics mirror `hermes_state._cross_process_repair_lock` (the schema- -# surgery authority): portable (msvcrt on Windows, flock elsewhere), bounded -# wait, and FAIL CLOSED — a caller that cannot acquire the lock must NOT -# rebuild. The kernel drops both lock types when the holder dies — UNLESS a -# forked child inherited the lock fd (flock rides the open file description, -# which fork() duplicates), in which case the orphaned descriptor holds the -# lock forever (issue #100108). `_acquire_db_flock` therefore records the -# holder's pid + start time under the lock and, when the recorded holder is -# provably dead, breaks the orphaned lock by unlinking and retaking it on a -# fresh inode; indeterminate liveness still defers. It lives here (not -# hermes_state) because the search/schema mixins cannot import hermes_state -# (cycle). -# -# The lock file is `.fts_rebuild.lock`, distinct from `.repair.lock`: -# schema surgery runs on an EXCLUSIVE offline connection and can legitimately -# take minutes in VACUUM, while runtime rebuilds run on live connections. The -# timeout is sized for a full 'rebuild' of both indexes on a large DB. +# `.fts_rebuild.lock` is distinct from `.repair.lock`: schema surgery +# runs on an EXCLUSIVE offline connection and may take minutes in VACUUM. logger = logging.getLogger("hermes_state") @@ -922,31 +866,23 @@ _FTS_REBUILD_LOCK_TIMEOUT_SECONDS = 120.0 _FTS_REBUILD_LOCK_POLL_SECONDS = 0.1 _IS_WINDOWS = sys.platform == "win32" -# Post-break re-acquire budget: once a provably-orphaned lock has been broken -# the fresh inode is uncontended (or contended only by live processes), so a -# short bounded wait suffices — never re-enter the full timeout. +# Post-break re-acquire budget: the fresh inode is contended only by live +# processes, so a short wait suffices — never re-enter the full timeout. _LOCK_BREAK_REACQUIRE_SECONDS = 5.0 -# errno set for "another process holds this advisory lock". flock() reports -# contention as EWOULDBLOCK/EAGAIN; msvcrt.locking() as EACCES (and EDEADLK -# when its internal retry gives up). Anything else — ESTALE on a dropped NFS -# handle, ENOTSUP/ENOLCK on a filesystem without advisory locks, EIO — is a -# persistent environment failure that no amount of polling turns into an -# acquire. Treating every OSError as contention made such a failure look -# like a live holder and burned the full 120s admission timeout on every -# attempt (#100108, PR #100130). +# "Another process holds the lock": flock → EWOULDBLOCK/EAGAIN, msvcrt.locking +# → EACCES (EDEADLK when its retry gives up). Anything else (ESTALE, ENOTSUP, +# ENOLCK, EIO) is a persistent environment failure that polling cannot fix; +# treating it as contention burned the full timeout on every attempt. _LOCK_CONTENTION_ERRNOS = {errno.EAGAIN, errno.EACCES, errno.EWOULDBLOCK} if hasattr(errno, "EDEADLK"): _LOCK_CONTENTION_ERRNOS.add(errno.EDEADLK) def is_advisory_lock_contention(exc: BaseException) -> bool: - """True when *exc* means another process holds the advisory lock. - - False for every other ``OSError`` (ESTALE, ENOTSUP, ENOLCK, EIO, ...): - callers must fail closed IMMEDIATELY rather than poll to the deadline, - because retrying cannot succeed and the wait only stalls the caller. - """ + """True when *exc* means another process holds the advisory lock. For + any other ``OSError`` callers must fail closed IMMEDIATELY: retrying + cannot succeed and polling only stalls the caller.""" if isinstance(exc, BlockingIOError): return True if not isinstance(exc, OSError): @@ -955,13 +891,9 @@ def is_advisory_lock_contention(exc: BaseException) -> bool: def _proc_start_ticks(pid: int): - """Kernel start time of *pid* in clock ticks, or None when unknowable. - - Field 22 of ``/proc//stat`` (``starttime``) uniquely identifies a - process together with its PID: a recycled PID gets a different start - time. Returns None off Linux or on any read/parse failure — callers must - treat None as "unknowable" and FAIL CLOSED. - """ + """Kernel start time of *pid* (field 22 of ``/proc//stat``), which + with the PID uniquely identifies a process; None off Linux or on any + failure — callers must treat None as unknowable and FAIL CLOSED.""" try: with open(f"/proc/{pid}/stat", "rb") as fh: stat = fh.read() @@ -988,12 +920,8 @@ def _read_lock_holder_record(handle): def _write_lock_holder_record(handle) -> None: - """Record this process as the lock holder (advisory, best effort). - - Written under the flock so contenders that time out can tell an - orphaned-fd holder (recorded process dead, flock inherited by a forked - child — issue #100108) from a live wedged holder. - """ + """Record this process as holder (best effort), written under the flock so + timed-out contenders can tell an orphaned-fd holder from a live wedged one.""" try: record = { "pid": os.getpid(), @@ -1009,12 +937,8 @@ def _write_lock_holder_record(handle) -> None: def _clear_lock_holder_record(handle) -> None: - """Erase holder metadata before a normal release. - - Guarantees that a surviving record always describes an ABNORMAL exit - (holder died without releasing), which is the only condition under which - a contender may break the lock. - """ + """Erase holder metadata before a normal release, so a surviving record + always means an ABNORMAL exit — the only condition allowing a break.""" try: handle.seek(0) handle.truncate() @@ -1025,12 +949,8 @@ def _clear_lock_holder_record(handle) -> None: def _lock_holder_provably_dead(record) -> bool: """True ONLY when the recorded holder is provably dead or PID-recycled. - - Any indeterminate state (no record, malformed record, PID owned by - another user, /proc unavailable, start-time unknowable) returns False — - the caller must FAIL CLOSED and defer, never break a possibly-live - holder's lock. - """ + Anything indeterminate (no/malformed record, PID owned by another user, + /proc unavailable) is False — the caller must FAIL CLOSED and defer.""" if not isinstance(record, dict): return False try: @@ -1052,34 +972,27 @@ def _lock_holder_provably_dead(record) -> bool: current_ticks = _proc_start_ticks(pid) if current_ticks is None: return False - # Same PID, different kernel start time: the recorded holder is dead and - # its PID was recycled by an unrelated process. + # Same PID, different start time: recycled by an unrelated process. return current_ticks != recorded_ticks def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, description): """Bounded POSIX flock acquire with orphaned-holder staleness break. - Returns ``(acquired, handle)``; *handle* may have been re-opened (the - caller owns closing whichever handle comes back). *acquired* is True on - success, False when a holder kept the lock past the deadline, and None - when a non-contention ``OSError`` (ESTALE/ENOTSUP/EIO) made acquisition - impossible — already logged here; callers treat None as "not acquired" - without emitting the held-by-another-process warning. + Returns ``(acquired, handle)``; *handle* may have been re-opened and the + caller closes whichever comes back. *acquired* is True, False (a holder + kept the lock past the deadline), or None (non-contention ``OSError``, + already logged; callers treat it as not acquired without the + held-by-another-process warning). - Why breaking exists at all (issue #100108): ``flock`` belongs to the open - file DESCRIPTION, which ``fork()`` duplicates into every child. A holder - that forks (multiprocessing worker, daemonized helper) and then dies - leaves the flock held by a child that will never release it — the - kernel's holder-death release never triggers, and every contender defers - forever. The recorded-holder liveness check distinguishes exactly that - case: the process that ACQUIRED is provably dead (so its critical section - died with it), yet the flock is still held. Only then is the lock file - unlinked and retaken on a fresh inode; the orphan's flock stays on the - old unlinked inode where it blocks nobody. Every successful acquire - verifies its inode still names *lock_path*, so a racer that locked a dead - inode retries instead of running concurrently with the breaker. - Indeterminate liveness always defers (fail closed). + ``flock`` belongs to the open file DESCRIPTION, which ``fork()`` + duplicates: a holder that forks then dies leaves the lock held forever by + a child that never releases. When the process that ACQUIRED is provably + dead (its critical section died with it) yet the flock is held, the file + is unlinked and retaken on a fresh inode; the orphan's flock stays on the + old inode blocking nobody. Every successful acquire verifies its inode + still names *lock_path*, so a racer that locked a dead inode retries + instead of running alongside the breaker. Indeterminate liveness defers. """ import fcntl @@ -1090,9 +1003,7 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) except (BlockingIOError, OSError) as exc: if not is_advisory_lock_contention(exc): - # ESTALE / ENOTSUP / EIO: not a holder, and polling cannot - # fix it. Defer NOW instead of pretending a live process - # held the lock for the whole timeout (#100108). + # Not a holder and polling cannot fix it: defer NOW. logger.warning( "Could not acquire %s %s (%s) — deferring rather than " "waiting out the %.0fs holder timeout on a " @@ -1135,9 +1046,8 @@ def _acquire_db_flock(lock_path, handle, timeout_seconds, poll_seconds, descript broke_lock = True deadline = time.monotonic() + _LOCK_BREAK_REACQUIRE_SECONDS continue - # flock acquired — verify the path still names our inode: a breaker - # may have unlinked/replaced the file while we waited, and a lock on - # a dead inode excludes nobody. + # Verify the path still names our inode: a breaker may have replaced + # the file while we waited, and a lock on a dead inode excludes nobody. try: fd_stat = os.fstat(handle.fileno()) path_stat = os.stat(lock_path) @@ -1178,21 +1088,13 @@ def _describe_lock_holder(record) -> str: def fts_rebuild_admission(db_path, *, timeout_seconds=None): """Serialize full structural FTS rebuilds on *db_path* across processes. - Yields True when this process holds the rebuild authority, False when the - bounded acquire timed out or the lock file could not be opened at all. A - caller that gets False must NOT perform a full rebuild — proceeding is - exactly the concurrent-rebuild interleaving this lock exists to prevent - (fail closed). The deferred/stale breadcrumb machinery already guarantees - a skipped rebuild is retried later. + Yields True when this process holds the authority, False when the bounded + acquire timed out or the lock file could not be opened. On False the + caller must NOT rebuild (fail closed); the stale breadcrumb guarantees a + retry. ``db_path`` None (in-memory DB) yields True. - ``db_path`` may be a str or Path; None (in-memory DB / tests without a - file path) yields True — a private in-memory DB has no cross-process - surface. - - *timeout_seconds* defaults to ``_FTS_REBUILD_LOCK_TIMEOUT_SECONDS``. - Opportunistic in-process retries (``retry_deferred_fts_recovery``) pass - ``0`` so a live holder never stalls a long-lived writer for two minutes; - the orphaned-holder break still applies on the single attempt. + Opportunistic in-process retries pass ``timeout_seconds=0`` so a live + holder never stalls a long-lived writer; the orphan break still applies. """ if db_path is None: yield True @@ -1206,16 +1108,12 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None): try: handle = open(lock_path, "a+b") except OSError as exc: - # Fail closed, exactly as a timed-out acquire does. A lock file we - # cannot even open means the filesystem is out of space, inodes or - # descriptors — and a sibling process that opened ITS handle before - # the disk filled is still holding the authority and rebuilding. - # Yielding True here handed every process on a full disk a concurrent - # structural rebuild of the same live state.db with no cross-process - # authority at all: the disk-full trigger and the re-corruption on - # every multi-writer boot in #100368. Deferring costs nothing that - # was reachable anyway — the breadcrumb retries, and on a read-only - # directory the rebuild's own writes could not have committed either. + # Fail closed like a timed-out acquire. An unopenable lock file means + # the FS is out of space/inodes/descriptors, and a sibling that opened + # its handle earlier may still be rebuilding; yielding True here gave + # every process on a full disk a concurrent rebuild of the same DB. + # Deferring costs nothing: the breadcrumb retries, and the rebuild's + # own writes could not have committed either. logger.warning( "Could not open FTS rebuild lock %s (%s) — deferring this rebuild " "rather than running it without cross-process authority.", @@ -1257,14 +1155,13 @@ def fts_rebuild_admission(db_path, *, timeout_seconds=None): "FTS rebuild lock", ) if acquired is None: - # Non-contention failure: already logged with the real errno; - # a "held by another process" line here would be a lie. + # Already logged with the real errno; "held by another process" + # would be a lie. acquired = False elif not acquired: record = None if _IS_WINDOWS else _read_lock_holder_record(handle) if timeout <= 0: - # Non-blocking probe from an in-process retry: a busy lock - # is expected and will be tried again, so keep it quiet. + # Non-blocking probe from an in-process retry: keep it quiet. logger.info( "FTS rebuild lock %s is busy — deferring this retry " "(the stale-FTS breadcrumb keeps it retryable). " diff --git a/hermes_state_compression.py b/hermes_state_compression.py new file mode 100644 index 0000000000..cb91823f39 --- /dev/null +++ b/hermes_state_compression.py @@ -0,0 +1,1030 @@ +"""Compression lineage, cooldown/streak counters, locks and turn leases for SessionDB. + +Mixin bound onto ``SessionDB`` via the MRO, built on its ``_read_ctx`` / +``_execute_write`` / ``_write_sql`` / ``_read_one`` primitives.""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import time +from typing import Any, Dict, List, Optional + +from hermes_state_common import _sql_session_last_active, is_automatic_end_reason + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionCompressionMixin: + """Compression lineage, cooldown/streak counters, locks and turn leases.""" + + def find_live_compression_child( + self, parent_session_id: str + ) -> Optional[Dict[str, Any]]: + """Return the unique live direct child of a compression-ended session. + + A stale agent whose parent was rotated elsewhere may recover only when the + lineage names exactly one live continuation; more than one fails closed + rather than guessing which transcript owns later messages.""" + if not parent_session_id: + return None + with self._read_ctx() as conn: + parent = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (parent_session_id,), + ).fetchone() + if ( + parent is None + or parent["ended_at"] is None + or parent["end_reason"] != "compression" + ): + return None + rows = conn.execute( + """ + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.parent_session_id = ? + AND s.ended_at IS NULL + """ + + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="s.") + + """ + ORDER BY s.started_at ASC + LIMIT 2 + """, + (parent_session_id, parent_session_id, parent_session_id), + ).fetchall() + return self._session_row_dict(rows[0]) if len(rows) == 1 else None + + def reopen_orphaned_compression_session(self, session_id: str) -> bool: + """Reopen a compression parent only when no continuation was published. + + Publication is atomic now, but older builds could leave a closed parent + after an interrupted handoff. Conservative by design: an active lease or + any canonical child means another path owns the lineage — fail closed.""" + if not session_id: + return False + + def _do(conn): + parent = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + parent is None + or parent["ended_at"] is None + or parent["end_reason"] != "compression" + ): + return False + + # Any non-branch/non-delegate/non-tool child is a continuation, ended + # or not; reopening past it could give one lineage a second live head. + child = conn.execute( + """ + SELECT 1 + FROM sessions + WHERE parent_session_id = ? + """ + + self._NON_CONTINUATION_CHILD_FILTER_SQL.format(alias="") + + """ + LIMIT 1 + """, + (session_id, session_id, session_id), + ).fetchone() + if child is not None: + return False + + # refresh_compression_lock() lets an owner revive its own expired + # row, so reclaim it inside this write transaction before reopening: + # refresh-first makes the lease active and aborts recovery; + # recovery-first deletes the holder so a later refresh can't resurrect it. + now = time.time() + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if lock_row is not None: + expires_at = lock_row["expires_at"] + if expires_at is None or float(expires_at) >= now: + return False + deleted = conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ? AND expires_at = ?", + (session_id, lock_row["holder"], expires_at), + ) + if deleted.rowcount != 1: + return False + + updated = conn.execute( + "UPDATE sessions SET ended_at = NULL, end_reason = NULL " + "WHERE id = ? AND ended_at IS NOT NULL " + "AND end_reason = 'compression'", + (session_id,), + ) + # rowcount==1 is guaranteed by the parent SELECT in this same BEGIN + # IMMEDIATE transaction. If a False return is ever added past this + # point, raise instead: _execute_write commits the lease DELETE above unless _do raises. + return updated.rowcount == 1 + + return bool(self._execute_write(_do)) + + def publish_compression_child( + self, + *, + parent_session_id: str, + child_session_id: str, + source: str, + messages: List[Dict[str, Any]], + model: str = None, + model_config: Dict[str, Any] = None, + system_prompt: str = None, + cwd: str = None, + profile_name: str = None, + compression_lock_holder: str = None, + require_compression_lease: bool = True, + require_lease_refresh: bool = False, + lease_ttl_seconds: float = 300.0, + watermark: Optional[int] = None, + watermark_ceiling: Optional[int] = None, + ) -> None: + """Atomically close a parent and publish its durable compression child. + + Closure, child row, and handoff commit in one transaction: readers see + the live parent or a complete child, never an ended parent with a + missing/empty child. + + *watermark* (parent's ``get_active_message_watermark`` at compression + start): parent rows with ``id > watermark`` — appends landed during the + slow summary call — are column-cloned into the child AFTER the handoff + so they survive rotation. *watermark_ceiling* bounds the clone: the + rotation path flushes its OWN transcript to the parent just before + publishing and those rows are already in the handoff, so the caller + captures ``MAX(id)`` right BEFORE that flush and only + ``(watermark, watermark_ceiling]`` is foreign tail. ``None`` = unbounded. + + *require_lease_refresh* + *compression_lock_holder* refreshes the lease + on the same ``conn`` before the expiry check (no TOCTOU window), so a + refresher that died on transient DB errors gets one last chance.""" + from hermes_state import CompressionSessionBusyError + def _do(conn): + if require_lease_refresh and compression_lock_holder: + conn.execute( + "UPDATE compression_locks SET expires_at = ? " + "WHERE session_id = ? AND holder = ?", + (time.time() + lease_ttl_seconds, parent_session_id, + compression_lock_holder), + ) + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks WHERE session_id = ?", + (parent_session_id,), + ).fetchone() + if require_compression_lease and ( + lock_row is None + or not compression_lock_holder + or lock_row["holder"] != compression_lock_holder + or float(lock_row["expires_at"]) <= time.time() + ): + raise CompressionSessionBusyError( + f"Compression lease lost before publication: {parent_session_id}" + ) + parent = conn.execute( + """SELECT ended_at, end_reason, cwd, git_branch, git_repo_root, + user_id, session_key, chat_id, chat_type, + thread_id, display_name, origin_json, profile_name + FROM sessions WHERE id = ?""", + (parent_session_id,), + ).fetchone() + if parent is None: + raise RuntimeError(f"Compression parent not found: {parent_session_id}") + if parent["ended_at"] is not None: + # An ended stamp from AUTOMATIC cleanup (tui_shutdown, ws_disconnect, + # orphan reap, idle/LRU evict) is stale by construction — this lease + # holder is still continuing the conversation. Left alone it wedges + # rotation forever (every attempt aborts here; each pre-publish flush + # re-grows the parent until the provider rejects it). Clear it; the + # closure UPDATE below re-stamps end_reason='compression'. Deliberate + # boundaries (compression, session_reset, explicit close) still fail + # closed — another path owns the lineage. + if is_automatic_end_reason(parent["end_reason"]): + conn.execute( + "UPDATE sessions SET ended_at = NULL, end_reason = NULL " + "WHERE id = ?", + (parent_session_id,), + ) + else: + raise RuntimeError( + f"Compression parent already ended: {parent_session_id}" + ) + if not messages: + raise RuntimeError("Compression child handoff must not be empty") + system_prompt_hash = self._store_system_prompt(conn, system_prompt) + + conn.execute( + """INSERT INTO sessions ( + id, source, model, model_config, system_prompt, + system_prompt_hash, + parent_session_id, cwd, git_branch, git_repo_root, + profile_name, user_id, session_key, chat_id, chat_type, + thread_id, display_name, origin_json, started_at + ) VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + child_session_id, + source, + model, + json.dumps(model_config) if model_config else None, + system_prompt_hash, + parent_session_id, + cwd or parent["cwd"], + parent["git_branch"], + parent["git_repo_root"], + # Same contract as _insert_session_row's compression-fork backfill: + # child stays on the parent's profile and keeps gateway routing/ + # origin columns so peer recovery works after a boundary crash. No + # owner on either side (legacy NULL parent) → stamp this store's + # profile so the child doesn't extend the unowned lineage. + profile_name + or parent["profile_name"] + or self._own_profile_name(), + parent["user_id"], + parent["session_key"], + parent["chat_id"], + parent["chat_type"], + parent["thread_id"], + parent["display_name"], + parent["origin_json"], + time.time(), + ), + ) + total_messages, total_tool_calls = self._insert_message_rows( + conn, child_session_id, messages + ) + if watermark is not None: + # Clone the parent's concurrent tail (see docstring) into the + # child after the handoff: column-exact except id/session_id; + # originals stay in the closed parent for lineage recovery. + _ceiling_clause = "" + _params: list = [parent_session_id, int(watermark)] + if watermark_ceiling is not None: + _ceiling_clause = " AND id <= ?" + _params.append(int(watermark_ceiling)) + tail_rows = conn.execute( + "SELECT id, tool_calls FROM messages " + "WHERE session_id = ? AND active = 1 AND id > ?" + f"{_ceiling_clause} ORDER BY id", + _params, + ).fetchall() + if tail_rows: + tail_ids = [int(r["id"]) for r in tail_rows] + placeholders = ",".join("?" for _ in tail_ids) + clone_cols = [ + c for c in self._message_column_names(conn) + if c not in ("id", "session_id", "active", "compacted") + ] + col_list = ", ".join(clone_cols) + conn.execute( + f"INSERT INTO messages ({col_list}, session_id, active, compacted) " + f"SELECT {col_list}, ?, 1, 0 FROM messages " + f"WHERE id IN ({placeholders}) ORDER BY id", + [child_session_id, *tail_ids], + ) + total_messages += len(tail_ids) + for r in tail_rows: + raw = r["tool_calls"] + if raw: + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + total_tool_calls += len(parsed) if isinstance(parsed, list) else 0 + except (TypeError, ValueError): + pass + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (total_messages, total_tool_calls, child_session_id), + ) + updated = conn.execute( + "UPDATE sessions SET ended_at = ?, end_reason = 'compression' " + "WHERE id = ? AND ended_at IS NULL", + (time.time(), parent_session_id), + ) + if updated.rowcount != 1: + raise RuntimeError( + f"Compression parent changed during publication: {parent_session_id}" + ) + + self._execute_write(_do) + + def record_compression_failure_cooldown( + self, + session_id: str, + cooldown_until: float, + error: Optional[str] = None, + ) -> None: + """Persist the active compression-failure cooldown for a session.""" + if not session_id: + return + + try: + # Merge-max with any longer live deadline so a later shorter write + # can't reopen the thrash window; error always takes the latest diagnostic. + self._write_sql( + "UPDATE sessions SET compression_failure_cooldown_until = CASE " + "WHEN compression_failure_cooldown_until IS NOT NULL " + " AND compression_failure_cooldown_until > ? " + "THEN compression_failure_cooldown_until ELSE ? END, " + "compression_failure_error = ? WHERE id = ?", + (cooldown_until, cooldown_until, error, session_id), + ) + except sqlite3.Error as exc: + logger.warning( + "record_compression_failure_cooldown(%s) failed: %s", + session_id, exc, + ) + + def get_compression_failure_cooldown( + self, + session_id: str, + ) -> Optional[Dict[str, Any]]: + """Return the active compression-failure cooldown for ``session_id``.""" + if not session_id: + return None + now = time.time() + row = self._read_one( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if row is None: + return None + cooldown_until = row[0] + if cooldown_until is None: + return None + cooldown_until = float(cooldown_until) + if cooldown_until <= now: + return None + error = row[1] + return { + "cooldown_until": cooldown_until, + "remaining_seconds": cooldown_until - now, + "error": error, + } + + def get_compression_failure_cooldown_row( + self, + session_id: str, + ) -> Dict[str, Any]: + """Exact stored cooldown columns, no expiry filtering. Compression + cancellation uses this under its session lease so rollback preserves an + expired, partially-null, or absent row exactly instead of coercing it + through the active-cooldown API.""" + if not session_id: + return {"session_exists": False, "cooldown_until": None, "error": None} + row = self._read_one( + "SELECT compression_failure_cooldown_until, compression_failure_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if row is None: + return {"session_exists": False, "cooldown_until": None, "error": None} + cooldown_until = row[0] + error = row[1] + return { + "session_exists": True, + "cooldown_until": ( + float(cooldown_until) if cooldown_until is not None else None + ), + "error": error, + } + + def restore_compression_failure_cooldown_row( + self, + session_id: str, + snapshot: Dict[str, Any], + ) -> None: + """Restore and verify an exact cooldown-row snapshot. Unlike record/clear, + this rollback API propagates write and verification failures: cancellation + must not be reported mutation-free when compensation failed.""" + expected_exists = bool(snapshot.get("session_exists", False)) + if not expected_exists: + actual = self.get_compression_failure_cooldown_row(session_id) + if actual.get("session_exists", False): + raise RuntimeError( + "cannot restore absent compression cooldown row: session now exists" + ) + return + + deadline = snapshot.get("cooldown_until") + error = snapshot.get("error") + + def _do(conn): + cursor = conn.execute( + "UPDATE sessions SET compression_failure_cooldown_until = ?, " + "compression_failure_error = ? WHERE id = ?", + (deadline, error, session_id), + ) + if cursor.rowcount != 1: + raise RuntimeError( + f"compression cooldown rollback session missing: {session_id}" + ) + + self._execute_write(_do) + actual = self.get_compression_failure_cooldown_row(session_id) + expected = { + "session_exists": True, + "cooldown_until": float(deadline) if deadline is not None else None, + "error": error, + } + if actual != expected: + raise RuntimeError( + f"compression cooldown rollback verification failed: " + f"expected={expected!r}, actual={actual!r}" + ) + + def clear_compression_failure_cooldown(self, session_id: str) -> None: + """Clear any persisted compression-failure cooldown for a session.""" + if not session_id: + return + + try: + self._write_sql( + "UPDATE sessions SET compression_failure_cooldown_until = NULL, " + "compression_failure_error = NULL WHERE id = ?", + (session_id,), + ) + except sqlite3.Error as exc: + logger.warning( + "clear_compression_failure_cooldown(%s) failed: %s", + session_id, exc, + ) + + def _read_session_number(self, column: str, session_id: str, cast: type, zero: Any) -> Any: + """Read one numeric ``sessions`` column clamped at ``zero``; a missing + session, NULL, or unparsable value also reads as ``zero``.""" + if not session_id: + return zero + row = self._read_one( + f"SELECT {column} FROM sessions WHERE id = ?", (session_id,) + ) + if row is None: + return zero + try: + return max(zero, cast(row[0] or zero)) + except (TypeError, ValueError): + return zero + + def get_compression_fallback_streak(self, session_id: str) -> int: + """Return the persisted deterministic-fallback streak.""" + return self._read_session_number("compression_fallback_streak", session_id, int, 0) + + def set_compression_fallback_streak(self, session_id: str, streak: int) -> None: + """Persist the deterministic-fallback streak for one session.""" + if session_id: + self._write_sql( + "UPDATE sessions SET compression_fallback_streak = ? WHERE id = ?", + (max(0, int(streak)), session_id), + ) + + def get_compression_ineffective_count(self, session_id: str) -> int: + """Persisted ineffective-compaction strike count: the durable half of + the built-in compressor's anti-thrash guard, so a fresh compressor bound + to a resumed session inherits an armed/tripped guard across restarts.""" + return self._read_session_number("compression_ineffective_count", session_id, int, 0) + + def set_compression_ineffective_count(self, session_id: str, count: int) -> None: + """Persist the ineffective-compaction strike count for one session.""" + if session_id: + self._write_sql( + "UPDATE sessions SET compression_ineffective_count = ? WHERE id = ?", + (max(0, int(count)), session_id), + ) + + def get_compression_recovery_deadline(self, session_id: str) -> float: + """Persisted anti-thrash recovery deadline (epoch; ``0.0`` = not armed). + Durable because the gateway rebuilds the compressor every turn / cache + eviction: a process-local deadline restarted on each rebuild, so a + tripped session never earned its probe.""" + return self._read_session_number("compression_recovery_deadline", session_id, float, 0.0) + + def set_compression_recovery_deadline(self, session_id: str, deadline: float) -> None: + """Persist the anti-thrash recovery deadline; ``0`` / ``None`` disarms it.""" + if not session_id: + return + try: + normalized = max(0.0, float(deadline or 0.0)) + except (TypeError, ValueError): + normalized = 0.0 + stored = normalized if normalized > 0.0 else None + + self._write_sql( + "UPDATE sessions SET compression_recovery_deadline = ? WHERE id = ?", + (stored, session_id), + ) + + def refresh_compression_lock( + self, + session_id: str, + holder: str, + ttl_seconds: float = 300.0, + ) -> bool: + """Extend the compression lock lease if ``holder`` still owns it. + + Ownership is decided by ``holder`` alone, deliberately NOT ``expires_at``: + a live owner whose refresher stalled past its TTL (GC pause, loaded CI + runner, slow write escaping ``_execute_write``'s retry budget) must be + able to revive its still-unclaimed row. Requiring ``expires_at >= now`` + made such a stall permanent — every later refresh matched 0 rows and the + owner kept compressing/rotating with no lease, exactly the window in + which a competing path can fork the lineage. + + It cannot resurrect a lock someone else took: SQLite serialises writes, + so :meth:`try_acquire_compression_lock`'s reclaim (DELETE-expired + + INSERT-or-IGNORE) never interleaves with this UPDATE. Reclaim-first + replaces ``holder`` and this matches nothing; refresh-first pushes + ``expires_at`` forward and the reclaimer's DELETE matches nothing.""" + if not session_id or not holder: + return False + now = time.time() + expires_at = now + ttl_seconds + + try: + return self._write_rowcount( + "UPDATE compression_locks SET expires_at = ? " + "WHERE session_id = ? AND holder = ?", + (expires_at, session_id, holder), + ) > 0 + except sqlite3.Error as exc: + logger.warning( + "refresh_compression_lock(%s) failed: %s", + session_id, exc, + ) + return False + + def try_acquire_compression_lock( + self, + session_id: str, + holder: str, + ttl_seconds: float = 300.0, + ) -> bool: + """Try to atomically acquire the compression lock for ``session_id``. + + ``True``: caller owns the lock and must :meth:`release_compression_lock`. + ``False``: another holder owns a live lock and the caller MUST NOT + compress — its rotation would race the holder's and split the lineage. + Expired locks and structured holders whose local ``pid=`` is dead are + reclaimed transparently, so a gateway killed mid-compression doesn't + stall its replacement for the full TTL. Single-transaction DELETE-expired + + INSERT-or-IGNORE + SELECT-to-confirm; SQLite serialises writes, so it's atomic.""" + from hermes_state import _compression_lock_holder_process_is_dead + if not session_id: + return False + now = time.time() + expires_at = now + ttl_seconds + + def _do(conn): + reclaimed_holder = None + row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if row is not None: + current_holder = ( + row[0] + ) + current_expires_at = ( + row[1] + ) + if ( + current_expires_at < now + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, current_holder), + ) + reclaimed_holder = current_holder + # INSERT OR IGNORE gives no rowcount signal — verify ownership via SELECT. + conn.execute( + "INSERT OR IGNORE INTO compression_locks " + "(session_id, holder, acquired_at, expires_at) " + "VALUES (?, ?, ?, ?)", + (session_id, holder, now, expires_at), + ) + row = conn.execute( + "SELECT holder FROM compression_locks WHERE session_id = ?", + (session_id,), + ).fetchone() + acquired = row is not None and ( + row[0] + ) == holder + return acquired, reclaimed_holder + + try: + acquired, reclaimed_holder = self._execute_write(_do) + if reclaimed_holder: + logger.warning( + "Reclaimed stale compression lock for session=%s " + "(holder=%s)", + session_id, + reclaimed_holder, + ) + return bool(acquired) + except sqlite3.Error as exc: + logger.warning( + "try_acquire_compression_lock(%s) failed: %s", + session_id, exc, + ) + # False makes the caller skip compression — the safe behaviour + # when the lock subsystem is broken. + return False + + def release_compression_lock(self, session_id: str, holder: str) -> None: + """Release the compression lock for ``session_id`` iff we own it. Idempotent + when the lock is gone or reclaimed; the ``holder`` check stops a late + compressor clobbering someone else's fresh lock.""" + if not session_id: + return + + try: + self._write_sql( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, holder), + ) + except sqlite3.Error as exc: + logger.warning( + "release_compression_lock(%s) failed: %s", + session_id, exc, + ) + + def _session_turn_lease_key_on_conn(self, conn, session_id: str) -> str: + """Walk compression parents on ``conn`` to the conversation lease key. + + Must share the connection of the lease INSERT/UPDATE/DELETE: a failed + ``get_session`` must not yield a child id the write then persists + (refresh would walk to the parent and fail-close). Markers bind to + ``parent_session_id`` (as in ``_NON_CONTINUATION_CHILD_FILTER_SQL``). + Lock errors propagate so ``_execute_write`` / ``acquire_session_turn_lease`` can retry.""" + if not session_id: + return session_id + + def _row(sid: str): + row = conn.execute( + "SELECT id, parent_session_id, source, model_config, end_reason " + "FROM sessions WHERE id = ?", + (sid,), + ).fetchone() + return dict(row) if row else None + + current = _row(session_id) + seen = {session_id} + while current: + parent_id = current.get("parent_session_id") + if ( + not parent_id + or parent_id in seen + or self._is_explicit_fork_child_row(current) + ): + break + parent = _row(parent_id) + if not parent or parent.get("end_reason") != "compression": + break + seen.add(parent_id) + current = parent + return str(current.get("id") or session_id) if current else session_id + + def _session_turn_lease_key(self, session_id: str) -> str: + """Return the stable serialization key for every compression segment. + + Acquire/refresh/release resolve this inside their write transaction; this + is for tests/diagnostics. It does not swallow lock errors — a swallowed + walk plus a later successful write was the fail-open that replayed the + post-rotation refresh miss.""" + if not session_id: + return session_id + with self._read_ctx() as conn: + return self._session_turn_lease_key_on_conn(conn, session_id) + + def try_acquire_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + patience_s: Optional[float] = None, + ) -> bool: + """Atomically acquire the cross-process turn lease for a conversation. + Compression rotates a session into child segments, so the durable key is + the lineage root, not the current segment id. The walk, the INSERT, and + reclaim of expired or dead-local-PID leases share one write transaction.""" + from hermes_state import _compression_lock_holder_process_is_dead + if not session_id or not holder: + return False + now = time.time() + expires_at = now + max(0.1, float(ttl_seconds)) + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + row = conn.execute( + "SELECT holder, expires_at FROM session_turn_leases " + "WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + if row is not None: + current_holder = row["holder"] + if ( + float(row["expires_at"]) <= now + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, current_holder), + ) + conn.execute( + "INSERT OR IGNORE INTO session_turn_leases " + "(conversation_id, holder, acquired_at, expires_at) " + "VALUES (?, ?, ?, ?)", + (conversation_id, holder, now, expires_at), + ) + owner = conn.execute( + "SELECT holder FROM session_turn_leases WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + return owner is not None and owner["holder"] == holder + + return bool(self._execute_write(_do, patience_s=patience_s)) + + def acquire_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + wait_seconds: float = 1800.0, + poll_interval_seconds: float = 1.0, + on_wait=None, + wait_notice_interval_seconds: float = 15.0, + should_abort=None, + acquire_patience_s: float = 0.5, + ) -> bool: + """Wait for a cross-process turn lease without holding a SQLite lock. + + ``on_wait(elapsed)`` is best-effort: called when the first attempt fails + (elapsed ~0) and about every ``wait_notice_interval_seconds`` after, so + UIs can show another process holds the conversation. ``should_abort()`` + True (e.g. ``/stop``) returns False at once, not after ``wait_seconds``.""" + from hermes_state import classify_persistence_error + deadline = time.monotonic() + max(0.0, float(wait_seconds)) + wait_started = None + last_notice_at = None + notice_every = max(0.0, float(wait_notice_interval_seconds)) + while True: + if should_abort is not None: + try: + if should_abort(): + return False + except Exception: + logger.debug( + "session turn lease should_abort callback failed", + exc_info=True, + ) + try: + if self.try_acquire_session_turn_lease( + session_id, + holder, + ttl_seconds=ttl_seconds, + patience_s=acquire_patience_s, + ): + return True + except sqlite3.Error as exc: + # Long holder transactions (compression publish, large flushes) + # can exhaust one write-patience budget; keep polling until + # wait_seconds or should_abort. + if classify_persistence_error(exc) != "locked": + raise + now = time.monotonic() + remaining = deadline - now + if remaining <= 0: + return False + if wait_started is None: + wait_started = now + if on_wait is not None and ( + last_notice_at is None + or notice_every == 0.0 + or (now - last_notice_at) >= notice_every + ): + try: + on_wait(max(0.0, now - wait_started)) + except Exception: + logger.debug( + "session turn lease on_wait callback failed", + exc_info=True, + ) + last_notice_at = now + time.sleep(min(max(0.01, float(poll_interval_seconds)), remaining)) + + def refresh_session_turn_lease( + self, + session_id: str, + holder: str, + *, + ttl_seconds: float = 300.0, + ) -> bool: + """Extend a turn lease only while ``holder`` still owns it.""" + if not session_id or not holder: + return False + expires_at = time.time() + max(0.1, float(ttl_seconds)) + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + cursor = conn.execute( + "UPDATE session_turn_leases SET expires_at = ? " + "WHERE conversation_id = ? AND holder = ?", + (expires_at, conversation_id, holder), + ) + return cursor.rowcount > 0 + + return bool(self._execute_write(_do)) + + def release_session_turn_lease(self, session_id: str, holder: str) -> None: + """Release a turn lease iff ``holder`` still owns it; idempotent.""" + if not session_id or not holder: + return + + def _do(conn): + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, holder), + ) + + self._execute_write(_do) + + def get_compression_lock_holder(self, session_id: str) -> Optional[str]: + """Return the current (non-expired) holder for ``session_id``, or None. + Diagnostic only — not part of the locking protocol.""" + if not session_id: + return None + now = time.time() + row = self._read_one( + "SELECT holder FROM compression_locks " + "WHERE session_id = ? AND expires_at >= ?", + (session_id, now), + ) + if row is None: + return None + return row[0] + + def finalize_orphaned_compression_sessions(self) -> int: + """Mark orphaned compression continuations (parent ended by compression; + child has messages, no end_reason/ended_at, api_call_count=0) as + ``orphaned_compression``. Non-destructive: messages are preserved.""" + cutoff = time.time() - 604800 # 7 days + + def _do(conn): + now = time.time() + result = conn.execute( + """ + UPDATE sessions + SET ended_at = ?, + end_reason = 'orphaned_compression' + WHERE api_call_count = 0 + AND end_reason IS NULL + AND ended_at IS NULL + AND started_at < ? + AND parent_session_id IS NOT NULL + AND EXISTS ( + SELECT 1 FROM sessions p + WHERE p.id = sessions.parent_session_id + AND p.end_reason = 'compression' + AND p.ended_at IS NOT NULL + ) + AND EXISTS ( + SELECT 1 FROM messages m + WHERE m.session_id = sessions.id + ) + """, + (now, cutoff), + ) + return result.rowcount + + return self._execute_write(_do) or 0 + + def get_compression_chain(self, session_id: str) -> List[str]: + """Walk the compression-continuation chain forward and return every id. + + Root-first, ending at the tip; ``[session_id]`` when no continuation + exists. ``get_compression_tip`` is this walk's last element — one + implementation so the two can never disagree. + + A continuation is a child of a session with ``end_reason='compression'``. + Older builds also required ``child.started_at >= parent.ended_at``; + too brittle — gateway + compression races can insert the real + continuation before the parent's ``ended_at`` is written while a stale + websocket later creates a sibling that passes the timestamp test, so + desktop resume followed the sibling and recent messages looked "lost". + Instead: follow only children of compression-ended parents, exclude + explicit branch/delegate/tool children, and prefer children that continue + the chain (``end_reason='compression'``) or are still live over stale + closed siblings such as ``ws_orphan_reap``.""" + current = session_id + chain = [current] if current else [] + seen = {current} if current else set() + # Defensive bound; chains this deep are pathological. + for _ in range(100): + with self._read_ctx() as conn: + cursor = conn.execute( + f""" + SELECT child.id + FROM sessions parent + JOIN sessions child ON child.parent_session_id = parent.id + WHERE parent.id = ? + AND parent.end_reason = 'compression' + AND json_extract(COALESCE(child.model_config, '{{}}'), '$._branched_from') IS NULL + AND json_extract(COALESCE(child.model_config, '{{}}'), '$._delegate_from') IS NULL + AND COALESCE(child.source, '') != 'tool' + ORDER BY + CASE + WHEN child.end_reason = 'compression' THEN 0 + WHEN child.ended_at IS NULL THEN 1 + ELSE 2 + END, + {_sql_session_last_active("child")} DESC, + child.started_at DESC, + child.id DESC + LIMIT 1 + """, + (current,), + ) + row = cursor.fetchone() + if row is None: + return chain + child_id = row["id"] + if not child_id or child_id in seen: + return chain + seen.add(child_id) + current = child_id + chain.append(child_id) + return chain + + def get_compression_tip(self, session_id: str) -> Optional[str]: + """Live tip of a compression chain (walk semantics: ``get_compression_chain``); + the input id when no continuation exists.""" + chain = self.get_compression_chain(session_id) + return chain[-1] if chain else session_id + + def _is_compression_child_row(self, child: Dict[str, Any]) -> bool: + parent_id = child.get("parent_session_id") + if not parent_id or self._is_explicit_fork_child_row(child): + return False + parent = self.get_session(parent_id) + return bool(parent and parent.get("end_reason") == "compression") + + def get_compression_lineage(self, session_id: str) -> List[str]: + """Return compression ancestors through tip in chronological order.""" + session = self.get_session(session_id) + if not session or self._is_explicit_fork_child_row(session): + return [session_id] if session else [] + + root = session + ancestors = {root["id"]} + while self._is_compression_child_row(root): + parent = self.get_session(root["parent_session_id"]) + if not parent or parent["id"] in ancestors: + break + root = parent + ancestors.add(root["id"]) + + lineage = [root["id"]] + seen = {root["id"]} + current = root + while current.get("end_reason") == "compression": + rows = self._read_all( + """ + SELECT * FROM sessions + WHERE parent_session_id = ? + ORDER BY started_at ASC + """, + (current["id"],), + ) + next_child = None + for row in rows: + candidate = dict(row) + if self._is_compression_child_row(candidate): + next_child = candidate + break + if not next_child or next_child["id"] in seen: + break + lineage.append(next_child["id"]) + seen.add(next_child["id"]) + current = next_child + if current["id"] == session_id: + # Later tips are included only when the requested session itself was compacted. + continue + return lineage if session_id in lineage else [session_id] diff --git a/hermes_state_dbfile.py b/hermes_state_dbfile.py new file mode 100644 index 0000000000..ba0af9d837 --- /dev/null +++ b/hermes_state_dbfile.py @@ -0,0 +1,674 @@ +"""state.db file-level health helpers. + +Split out of ``hermes_state.py``: header probes (application_id / zeroed-file +detection), deleted-WAL-sidecar holder scans, quarantine of zeroed or +lock-poisoned databases, ``collect_state_db_stats`` and holder-process +classification. Every name is re-imported into ``hermes_state`` so +``hermes_state.`` keeps resolving — and tests that monkeypatch it keep +intercepting, because intra-module calls to patched helpers go through a +lazy ``from hermes_state import ...`` at call time. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import sqlite3 +import struct +import sys +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from hermes_state_common import ( + FTS_REBUILD_DEFERRAL_KEY, + stat_db_file_identity as _stat_db_file_identity, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +# _read_sqlite_application_id runs on EVERY write via _raise_if_db_replaced, +# against the LIVE state.db. A bare open()/read()/close() there is the +# howtocorrupt §2.2 bug: close() cancels every POSIX advisory lock this +# process holds on the file — measured on Linux/SQLite 3.53.1, one probe call +# drops the WAL-mode DMS shared lock the writer connection holds on state.db +# (see hermes_cli/sqlite_safe_read.py for the module built around this rule). +# With the DMS lock gone, a fresh opener in another process can treat this +# writer as dead and rerun WAL-index recovery underneath it. +# +# The probe therefore reads through a per-path fd cached for the life of the +# process: opening an fd never cancels locks (only close() does), and +# os.pread takes no shared file position. When the path is re-pointed at a +# new inode (the very replacement this probe exists to detect), the stale fd +# is RETIRED, never closed — closing it would cancel the live connection's +# locks on the old file, the exact bug being avoided. Replacement events are +# rare and halt writes anyway, so the leak is bounded. +_HEADER_PROBE_LOCK = threading.Lock() + + +_HEADER_PROBE_FDS: "dict[str, tuple[int, int, int]]" = {} # key -> (fd, dev, ino) + + +_RETIRED_HEADER_PROBE_FDS: "list[int]" = [] # intentionally never closed + + +def _pread_db_header(db_path: Path, length: int) -> "Optional[bytes]": + """Lock-safe raw header read of a possibly-live SQLite database. + + POSIX: pread from a cached, never-closed fd (rebound when the path names + a new inode). Windows: plain read — advisory-lock cancellation is a + POSIX-only hazard and msvcrt locks do not share the failure mode. + """ + from hermes_state import _IS_WINDOWS + if _IS_WINDOWS: + try: + with db_path.open("rb") as handle: + return handle.read(length) + except OSError: + return None + key = str(db_path) + try: + st = os.stat(db_path) + except OSError: + return None + with _HEADER_PROBE_LOCK: + cached = _HEADER_PROBE_FDS.get(key) + if cached is not None and (cached[1], cached[2]) != (st.st_dev, st.st_ino): + # Path re-pointed at a new file. Retire (never close) the old fd. + _RETIRED_HEADER_PROBE_FDS.append(cached[0]) + cached = None + del _HEADER_PROBE_FDS[key] + if cached is None: + try: + fd = os.open(db_path, os.O_RDONLY) + except OSError: + return None + try: + fst = os.fstat(fd) + except OSError: + _RETIRED_HEADER_PROBE_FDS.append(fd) + return None + cached = (fd, fst.st_dev, fst.st_ino) + _HEADER_PROBE_FDS[key] = cached + try: + return os.pread(cached[0], length, 0) + except OSError: + return None + + +def _read_sqlite_application_id(db_path: Path) -> "Optional[int]": + """Read application_id from the SQLite header without opening a connection. + + Safe against live databases: routed through :func:`_pread_db_header`, + which never issues a ``close()`` that would cancel this process's POSIX + locks on the file (howtocorrupt §2.2). + """ + from hermes_state import _STATE_DB_APPLICATION_ID_OFFSET + header = _pread_db_header(db_path, _STATE_DB_APPLICATION_ID_OFFSET + 4) + if header is None: + return None + if len(header) < _STATE_DB_APPLICATION_ID_OFFSET + 4: + return None + if header[:16] != b"SQLite format 3\x00": + return None + return int( + struct.unpack( + ">I", + header[_STATE_DB_APPLICATION_ID_OFFSET:_STATE_DB_APPLICATION_ID_OFFSET + 4], + )[0] + ) + + +def _stat_sqlite_sidecar_identity(db_path: Path) -> Dict[str, tuple]: + """Snapshot ``(st_dev, st_ino)`` for existing WAL/SHM sidecars.""" + identities: Dict[str, tuple] = {} + base = os.fspath(db_path) + for suffix in ("-wal", "-shm"): + ident = _stat_db_file_identity(Path(base + suffix)) + if ident is not None: + identities[suffix] = ident + return identities + + +def _canonical_sqlite_path(path: str) -> str: + """Normalize a /proc fd target, stripping the Linux `` (deleted)`` suffix.""" + return os.path.normcase(os.path.abspath(path.removesuffix(" (deleted)"))) + + +def _watched_sqlite_sidecar_paths(db_path) -> Set[str]: + base = os.path.abspath(os.fspath(db_path)) + return { + _canonical_sqlite_path(base + "-wal"), + _canonical_sqlite_path(base + "-shm"), + } + + +def iter_deleted_sqlite_sidecar_holders(db_path) -> List[Tuple[int, str]]: + """Return processes holding an unlinked ``state.db-wal`` / ``-shm``. + + Linux-only (``/proc//fd`` readlink). Windows and other hosts + return ``[]`` — Windows cannot unlink a sidecar another process still + holds, and macOS does not use the `` (deleted)`` suffix. + + The scan includes this process: on the SessionDB open/write refuse + path, the in-process writer that still holds the orphan inode is the + one that must not mint a replacement WAL (and must stop committing). + ``_foreign_state_db_holders`` keeps skipping this PID for FTS + maintenance so a process does not block its own optional repair. + """ + if not sys.platform.startswith("linux"): + return [] + + holders: List[Tuple[int, str]] = [] + watched = _watched_sqlite_sidecar_paths(db_path) + try: + for pid_str in os.listdir("/proc"): + if not pid_str.isdigit(): + continue + pid = int(pid_str) + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue + for fd in fds: + try: + target = os.readlink(f"{fd_dir}/{fd}") + except OSError: + continue + if " (deleted)" not in target: + continue + if _canonical_sqlite_path(target) in watched: + holders.append((pid, target)) + except Exception as exc: + logger.debug("deleted-WAL holder scan failed for %s: %s", db_path, exc) + return holders + return holders + + +def refuse_deleted_wal_generation(db_path) -> None: + """Raise if any process holds a deleted WAL/SHM generation for *db_path*. + + Called *before* ``sqlite3.connect`` so a second opener cannot mint a + replacement WAL inode while a live writer still holds the orphan. + """ + from hermes_state import DeletedWalGenerationError, _DELETED_WAL_GENERATION_MSG + holders = iter_deleted_sqlite_sidecar_holders(db_path) + if not holders: + return + logger.error(_DELETED_WAL_GENERATION_MSG) + raise DeletedWalGenerationError(_DELETED_WAL_GENERATION_MSG) + + +def _connect_tracked_db(path, tracking_path=None, **kwargs): + """``sqlite3.connect`` that registers the open fd for lock-safety. + + While a connection is live, byte-level probes of the same file are + refused: an ``open()``/``close()`` cancels every POSIX advisory lock this + process holds on it -- including a running VACUUM's EXCLUSIVE lock. + Released automatically on ``close()``. + + The ONLY tolerated fallback is the helper being absent entirely + (scaffold/embed installs that ship hermes_state without hermes_cli). A + real connection failure must propagate: silently retrying an *untracked* + connect would disable the guard for the lifetime of that connection, + which is precisely the failure mode this module exists to prevent. + """ + try: + from hermes_cli.sqlite_safe_read import connect_tracked + except ImportError: + logger.debug( + "hermes_cli.sqlite_safe_read unavailable; opening %s untracked " + "(byte-probe guard inactive in this install)", + path, + ) + return sqlite3.connect(str(path), **kwargs) + + # Open through THIS module's sqlite3.connect so callers (and tests) that + # patch hermes_state.sqlite3.connect keep control of connection creation; + # the helper still owns tracking. + return connect_tracked( + path, + tracking_path=tracking_path, + connect_fn=sqlite3.connect, + **kwargs, + ) + + +def is_zeroed_state_db( + path: Path, *, probe_bytes: int = 100, force: bool = False +) -> bool: + """Detect the #68474/#97568 zeroed state.db signature (0-byte or NUL header). + + Byte-level probe, so it is only safe BEFORE any connection to *path* + exists in this process: ``close()`` cancels every POSIX advisory lock the + process holds on the file, which can pull the EXCLUSIVE lock out from + under a running VACUUM and corrupt the database. The read is routed + through ``read_header_bytes_preopen``, which refuses (returning False + here) once a connection is live. Pass ``force=True`` only for offline + files -- quarantined copies, snapshots, archives. + + Prefer ``hermes_cli.backup.is_zeroed_sqlite_file`` when available; this + local copy keeps SessionDB openable without importing the CLI package + in constrained embed paths. + """ + try: + from hermes_cli.backup import is_zeroed_sqlite_file + + return is_zeroed_sqlite_file(path, probe_bytes=probe_bytes, force=force) + except Exception: + pass + try: + if not path.is_file(): + # Special files (FIFO, device, socket) are never "zeroed", and + # probing a FIFO would block until a writer appears. + return False + size = path.stat().st_size + except OSError: + return False + if size < 0: + return False + from hermes_cli.sqlite_safe_read import has_live_connection, read_header_bytes_preopen + + if not force and has_live_connection(path): + return False + + head = read_header_bytes_preopen( + path, length=max(16, probe_bytes), force=force + ) + if head is None: + return False + if len(head) == 0: + return True + if head.startswith(b"SQLite format 3"): + return False + return all(byte == 0 for byte in head) + + +@contextlib.contextmanager +def quarantine_cross_process_lock(path: Path, timeout: float = 5.0): + """Acquire the cross-process lock for path.quarantine.lock.""" + import platform + + lock_path = path.with_name(path.name + ".quarantine.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + acquired = False + try: + deadline = time.monotonic() + timeout + if platform.system() == "Windows": + import msvcrt + + while True: + try: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + acquired = True + break + except OSError: + if time.monotonic() >= deadline: + break + time.sleep(0.020) + else: + import fcntl + + while True: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + break + except (BlockingIOError, OSError): + if time.monotonic() >= deadline: + break + time.sleep(0.020) + yield acquired + finally: + try: + if acquired: + if platform.system() == "Windows": + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except (OSError, AttributeError): + pass + finally: + handle.close() + + +def quarantine_zeroed_state_db( + path: Path, *, already_locked: bool = False +) -> Optional[Path]: + """Move a zeroed state.db aside (preserve bytes) and return quarantine path. + + Uses a cross-process lock (``#68805``) so two concurrent startups cannot + race: the first process moves the zeroed file and the second re-checks + under the lock, finding the file already gone (or a fresh DB in its place) + instead of clobbering the quarantine. + """ + def _do_quarantine(): + if not path.exists(): + logger.info( + "quarantine_zeroed_state_db: %s already moved by another process", + path, + ) + return None + if not is_zeroed_state_db(path): + logger.info( + "quarantine_zeroed_state_db: %s is no longer zeroed (another " + "process quarantined it and a fresh DB was created)", + path, + ) + return None + + try: + ts = time.strftime("%Y%m%d-%H%M%S") + except Exception: + ts = "unknown" + dest = path.with_name( + f"{path.name}.zeroed-{ts}-{os.getpid()}.bak" + ) + n = 0 + while dest.exists(): + n += 1 + dest = path.with_name( + f"{path.name}.zeroed-{ts}-{os.getpid()}-{n}.bak" + ) + try: + path.rename(dest) + except OSError as exc: + logger.error("Failed to quarantine zeroed %s: %s", path, exc) + return None + for suffix in ("-wal", "-shm"): + side = Path(str(path) + suffix) + if side.exists(): + try: + side.rename(Path(str(dest) + suffix)) + except OSError: + pass + return dest + + if already_locked: + return _do_quarantine() + + with quarantine_cross_process_lock(path) as acquired: + if not acquired: + logger.error( + "quarantine lock for %s not acquired within 5s — refusing to " + "quarantine without the cross-process lock. The zeroed file " + "is left in place. If sessions fail to load, restore from " + "state-snapshots via `hermes snapshot list` / " + "`hermes snapshot restore `.", + path, + ) + return None + return _do_quarantine() + + +def collect_state_db_stats(db_path: Path) -> Dict[str, Any]: + """Best-effort, strictly read-only stats snapshot of a state.db file. + + Opens the database with ``mode=ro`` (URI) and a short timeout so it can + run against a *live* database held by a gateway without ever taking a + write lock or mutating the file. Every field is collected independently: + a failed pragma/SELECT yields ``None`` for that field, and the helper + itself never raises. + + Deliberately does NOT instantiate :class:`SessionDB` — its constructor + runs schema DDL (migrations, FTS table creation), which is exactly the + kind of write a diagnostics probe must never perform. + + Returned keys (all present, any may be None on failure): + + - ``page_count``, ``page_size``, ``freelist_count`` — PRAGMA values + - ``logical_size_bytes`` — page_count * page_size (post-checkpoint size) + - ``wal_size_bytes`` — stat() of ``-wal`` (0 when absent) + - ``journal_mode`` — PRAGMA journal_mode string + - ``messages`` / ``sessions`` — row counts + - ``fts_tables`` — dict of {table_name: bool} presence for + messages_fts / messages_fts_trigram / messages_fts_cjk + - ``fts_storage_version`` — int from state_meta, None when the marker is + absent (legacy pre-v23 inline layout) + - ``fts_rebuild_pending`` — True when the deferred v23 backfill has not + finished (high_water present and progress < high_water) + - ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` — raw ints + - ``fts_rebuild_deferral`` — durable blocked-repair diagnostic, when present + """ + from hermes_state import _connect_tracked_db + stats: Dict[str, Any] = { + "page_count": None, + "page_size": None, + "freelist_count": None, + "logical_size_bytes": None, + "wal_size_bytes": None, + "journal_mode": None, + "messages": None, + "sessions": None, + "fts_tables": None, + "fts_storage_version": None, + "fts_rebuild_pending": None, + "fts_rebuild_high_water": None, + "fts_rebuild_progress": None, + "fts_rebuild_deferral": None, + } + + # WAL sidecar size needs no connection at all. + try: + wal_path = Path(str(db_path) + "-wal") + stats["wal_size_bytes"] = wal_path.stat().st_size if wal_path.exists() else 0 + except OSError: + pass + + conn = None + try: + # mode=ro refuses to create the file and refuses every write; a + # short timeout keeps doctor snappy when a writer holds the lock. + # Route through the tracked connect so byte-probe helpers + # (read_header_bytes_preopen) see this connection and refuse raw + # opens that could cancel our POSIX locks mid-read. + conn = _connect_tracked_db( + f"file:{Path(db_path)}?mode=ro", + tracking_path=Path(db_path), + uri=True, + timeout=2.0, + ) + except Exception as exc: + logger.debug("collect_state_db_stats: cannot open %s read-only: %s", + db_path, exc) + return stats + + def _scalar(sql: str) -> Any: + try: + row = conn.execute(sql).fetchone() + return row[0] if row else None + except Exception: + return None + + try: + pc = _scalar("PRAGMA page_count") + ps = _scalar("PRAGMA page_size") + stats["page_count"] = int(pc) if pc is not None else None + stats["page_size"] = int(ps) if ps is not None else None + if stats["page_count"] is not None and stats["page_size"] is not None: + stats["logical_size_bytes"] = stats["page_count"] * stats["page_size"] + + fl = _scalar("PRAGMA freelist_count") + stats["freelist_count"] = int(fl) if fl is not None else None + + jm = _scalar("PRAGMA journal_mode") + stats["journal_mode"] = str(jm) if jm is not None else None + + msgs = _scalar("SELECT COUNT(*) FROM messages") + stats["messages"] = int(msgs) if msgs is not None else None + sess = _scalar("SELECT COUNT(*) FROM sessions") + stats["sessions"] = int(sess) if sess is not None else None + + # FTS table presence via sqlite_master (never SELECTs from the + # virtual tables themselves — a corrupt index must not fail stats). + try: + names = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name IN (?, ?, ?)", + ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"), + ).fetchall() + } + stats["fts_tables"] = { + t: (t in names) + for t in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk") + } + except Exception: + pass + + # Raw state_meta reads — cheap, and independent of SessionDB. + def _meta_int(key: str) -> Optional[int]: + try: + row = conn.execute( + "SELECT value FROM state_meta WHERE key = ?", (key,) + ).fetchone() + return int(row[0]) if row and row[0] is not None else None + except Exception: + return None + + stats["fts_storage_version"] = _meta_int("fts_storage_version") + high_water = _meta_int("fts_rebuild_high_water") + progress = _meta_int("fts_rebuild_progress") + stats["fts_rebuild_high_water"] = high_water + stats["fts_rebuild_progress"] = progress + if high_water is None: + stats["fts_rebuild_pending"] = False + else: + stats["fts_rebuild_pending"] = (progress or 0) < high_water + try: + row = conn.execute( + "SELECT value FROM state_meta WHERE key = ? LIMIT 1", + (FTS_REBUILD_DEFERRAL_KEY,), + ).fetchone() + if row: + parsed = json.loads(row[0]) + if isinstance(parsed, dict): + stats["fts_rebuild_deferral"] = parsed + except Exception: + pass + finally: + try: + conn.close() + except Exception: + pass + + return stats + + +def count_db_holders(db_path: Path) -> Optional[int]: + """Best-effort count of processes holding ``db_path`` open (Linux only). + + Scans ``/proc/*/fd`` symlinks for the resolved database path. Returns + the number of distinct PIDs with the file open, or ``None`` on any + error or on non-Linux platforms. Never raises; no lsof dependency. + Unreadable per-process fd dirs (other users' processes without root) + are silently skipped, so the count is a lower bound. + """ + try: + if not sys.platform.startswith("linux"): + return None + target = os.path.realpath(str(db_path)) + holders = 0 + for pid in os.listdir("/proc"): + if not pid.isdigit(): + continue + fd_dir = f"/proc/{pid}/fd" + try: + fds = os.listdir(fd_dir) + except OSError: + continue # process gone or not ours + for fd in fds: + try: + if os.readlink(f"{fd_dir}/{fd}") == target: + holders += 1 + break # one hit per PID + except OSError: + continue + return holders + except Exception: + return None + + +def _is_inactive_orphan_desktop_holder( + *, + ppid: int, + age_seconds: float, + min_age_seconds: float, + ephemeral_backend: bool, + connection_statuses: List[str], +) -> bool: + """Pure safety predicate for the narrow Desktop holder reap.""" + return ( + ppid in (0, 1) + and age_seconds >= min_age_seconds + and ephemeral_backend + and "ESTABLISHED" not in connection_statuses + ) + + +def _concrete_state_db_holder_pids( + db_path: Path, holders: List[Tuple[int, str]] +) -> List[int]: + """Return unique PIDs proven to hold this DB or one of its sidecars.""" + canonical_db = os.path.normcase(os.path.abspath(os.fspath(db_path))) + watched = { + canonical_db, + canonical_db + "-wal", + canonical_db + "-shm", + } + pids: List[int] = [] + seen = set() + for pid, path in holders: + canonical_path = os.path.normcase( + os.path.abspath(path.removesuffix(" (deleted)")) + ) + if pid <= 0 or pid in seen or canonical_path not in watched: + continue + seen.add(pid) + pids.append(pid) + return pids + + +def _read_proc_cmdline(pid: int) -> Optional[str]: + """Read /proc//cmdline, world-readable even when fd table is not. + + Returns the cmdline as a space-joined string, or None when unreadable + (process exited, or hidepid mount). + """ + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + raw = f.read() + if not raw: + return None + return raw.replace(b"\x00", b" ").decode("utf-8", "replace").strip() + except OSError: + return None + + +_HERMES_CMDLINE_MARKERS = ("hermes_cli.main", "hermes_cli/main", "hermes serve", + "hermes-agent", "hermes gateway", "hermes chat") + + +def _looks_like_hermes(cmdline: str) -> bool: + """Heuristic: does this cmdline look like a Hermes process? + + Used to decide whether an uninspectable process (fd table unreadable + due to different user) should be treated as a potential state.db holder. + We only flag processes that look like Hermes, not every system daemon. + """ + lower = cmdline.lower() + return any(marker in lower for marker in _HERMES_CMDLINE_MARKERS) diff --git a/hermes_state_gateway.py b/hermes_state_gateway.py new file mode 100644 index 0000000000..f74b7dcce0 --- /dev/null +++ b/hermes_state_gateway.py @@ -0,0 +1,1005 @@ +"""Gateway-facing SessionDB persistence: routing index, peers, orphans, heartbeats, handoffs. + +Mixin bound onto ``SessionDB`` via the MRO; built on its ``_read_ctx`` / +``_execute_write`` / ``_write_sql`` / ``_read_all`` primitives. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple + +from hermes_state_common import ( + _RECOVERABLE_END_REASONS_SQL, + _RESET_END_REASONS_SQL, + _sql_session_last_active, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionGatewayMixin: + """Routing index, session peers/orphans, hygiene streaks, heartbeats, handoffs.""" + + def _reap_inactive_orphan_desktop_holders( + self, holders: List[Tuple[int, str]], *, min_age_seconds: float + ) -> List[int]: + """Terminate old PPID-1 Desktop ephemeral backends with no client. + + Fails closed: anything whose parent, age, argv, or network connections + cannot be proved safe remains a repair-blocking holder. + """ + from hermes_state import _concrete_state_db_holder_pids, _is_inactive_orphan_desktop_holder, psutil + if not sys.platform.startswith("linux") or psutil is None: + return [] + try: + from hermes_cli.dashboard_procs import _is_ephemeral_port_zero_backend + except Exception: + return [] + + now = time.time() + candidates = [] + for pid in _concrete_state_db_holder_pids(self.db_path, holders): + try: + process = psutil.Process(pid) + statuses = [ + conn.status for conn in process.net_connections(kind="inet") + ] + if not _is_inactive_orphan_desktop_holder( + ppid=process.ppid(), + age_seconds=now - process.create_time(), + min_age_seconds=min_age_seconds, + ephemeral_backend=_is_ephemeral_port_zero_backend(process.cmdline()), + connection_statuses=statuses, + ): + continue + except Exception: + continue + candidates.append(process) + + signalled: List[int] = [] + for process in candidates: + try: + process.terminate() + signalled.append(process.pid) + except (psutil.Error, OSError): + continue + if not signalled: + return [] + + try: + _gone, alive = psutil.wait_procs(candidates, timeout=1.5) + except Exception: + alive = [] + for process in alive: + try: + process.kill() + except (psutil.Error, OSError): + continue + if alive: + try: + psutil.wait_procs(alive, timeout=1.5) + except Exception: + pass + return signalled + + def record_gateway_session_peer( + self, + session_id: str, + *, + source: str, + user_id: str = None, + session_key: str = None, + chat_id: str = None, + chat_type: str = None, + thread_id: str = None, + display_name: str = None, + origin_json: str = None, + include_compression_ancestors: bool = False, + ) -> None: + """Persist the gateway routing peer for an existing session row. + + ``display_name`` / ``origin_json`` let consumers (mcp_serve, mirror, + channel directory) read routing data from state.db instead of + sessions.json; ``None`` leaves the existing value untouched. + ``include_compression_ancestors`` keeps a compression lineage on one + routing peer when an explicit resume moves its tip to another lane; + normal per-turn refreshes update only the supplied row. + + Self-healing: a missing target row (deferred ``create_session`` write, + or crash between routing publication and row creation) is INSERTed + with full identity rather than silently no-opped, so a gateway row can + never be first-created by an identity-less lazy writer + (``update_token_counts``) and stay unroutable forever. + """ + if not session_id or not session_key: + return + + def _do(conn): + lineage_cte = "" + target_clause = "WHERE id = ?" + query_params = [] + if include_compression_ancestors: + lineage_cte = """ + WITH RECURSIVE compression_lineage(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM compression_lineage lineage + JOIN sessions child ON child.id = lineage.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE parent.end_reason = 'compression' + AND json_extract( + COALESCE(child.model_config, '{}'), + '$._branched_from' + ) IS NULL + AND json_extract( + COALESCE(child.model_config, '{}'), + '$._delegate_from' + ) IS NULL + AND COALESCE(child.source, '') != 'tool' + ) + """ + target_clause = "WHERE id IN (SELECT id FROM compression_lineage)" + query_params.append(session_id) + query_params.extend( + ( + session_key, + source, + user_id, + chat_id, + chat_type, + thread_id, + display_name, + origin_json, + ) + ) + if not include_compression_ancestors: + query_params.append(session_id) + conn.execute( + f"""{lineage_cte} + UPDATE sessions + SET session_key = ?, source = ?, user_id = ?, chat_id = ?, + chat_type = ?, thread_id = ?, + display_name = COALESCE(?, display_name), + origin_json = COALESCE(?, origin_json) + {target_clause}""", + query_params, + ) + # Self-heal: the UPDATE silently no-ops on a missing row — insert it + # with full identity so the session is durably routable. + if not include_compression_ancestors: + cur = conn.execute( + "SELECT 1 FROM sessions WHERE id = ? LIMIT 1", (session_id,) + ) + if cur.fetchone() is None: + conn.execute( + """INSERT INTO sessions ( + id, source, user_id, session_key, chat_id, + chat_type, thread_id, display_name, origin_json, + profile_name, started_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + session_key = COALESCE(sessions.session_key, excluded.session_key), + chat_id = COALESCE(sessions.chat_id, excluded.chat_id), + chat_type = COALESCE(sessions.chat_type, excluded.chat_type), + thread_id = COALESCE(sessions.thread_id, excluded.thread_id), + display_name = COALESCE(sessions.display_name, excluded.display_name), + origin_json = COALESCE(sessions.origin_json, excluded.origin_json)""", + ( + session_id, + source, + user_id, + session_key, + chat_id, + chat_type, + thread_id, + display_name, + origin_json, + # Same ownership stamp as _insert_session_row: an + # unowned (NULL) row vanishes from profile-keyed consumers. + self._own_profile_name(), + time.time(), + ), + ) + + self._execute_write(_do) + + def save_gateway_routing_entry( + self, session_key: str, entry_json: str, *, scope: str = "" + ) -> None: + """Upsert one gateway routing entry (session_key -> SessionEntry JSON). + + ``gateway_routing`` durably replaces sessions.json. ``scope`` namespaces + the index per sessions_dir so two stores never share routing state. + """ + if not session_key or not entry_json: + return + + self._write_sql( + """INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(scope, session_key) DO UPDATE SET + entry_json = excluded.entry_json, + updated_at = excluded.updated_at""", + (scope, session_key, entry_json, time.time()), + ) + + def replace_gateway_routing_entries( + self, entries: Dict[str, str], *, scope: str = "" + ) -> None: + """Atomically replace the routing index for *scope* with *entries*. + + Full-rewrite semantics: keys absent from *entries* are removed. One + write transaction; other scopes untouched. + """ + now = time.time() + + def _do(conn): + conn.execute("DELETE FROM gateway_routing WHERE scope = ?", (scope,)) + if entries: + conn.executemany( + "INSERT INTO gateway_routing (scope, session_key, entry_json, updated_at) " + "VALUES (?, ?, ?, ?)", + [(scope, k, v, now) for k, v in entries.items() if k and v], + ) + + self._execute_write(_do) + + def load_gateway_routing_entries(self, *, scope: str = "") -> Dict[str, str]: + """Load routing entries for *scope* as {session_key: entry_json}.""" + rows = self._read_all( + "SELECT session_key, entry_json FROM gateway_routing WHERE scope = ?", + (scope,), + ) + return {r["session_key"]: r["entry_json"] for r in rows} + + def list_never_active_keyed_sessions( + self, *, older_than_days: float + ) -> List[Dict[str, Any]]: + """Keyed gateway rows that were opened and then never used at all. + + Keyed, still-open rows with no evidence of a single turn (no messages, + tokens, tool/API calls, activity, or title): a leaked test fixture or a + chat routed but never answered. Safe to drop — no transcript to lose, + and the gateway mints a fresh session on the next inbound message. + Needs its own selector because ``bulk prune``/``archive`` are pinned to + ``ended_at IS NOT NULL`` (never pick a live session), which excludes + every never-closed row. ``pinned``/``archived`` = explicit keep intent. + """ + cutoff = time.time() - (float(older_than_days) * 86400.0) + rows = self._read_all( + """ + SELECT s.id, s.session_key, s.source, s.chat_id, + s.chat_type, s.user_id, s.started_at + FROM sessions s + WHERE s.session_key IS NOT NULL + AND s.ended_at IS NULL + AND s.title IS NULL + AND s.last_activity_at IS NULL + AND COALESCE(s.message_count, 0) = 0 + AND COALESCE(s.tool_call_count, 0) = 0 + AND COALESCE(s.api_call_count, 0) = 0 + AND COALESCE(s.input_tokens, 0) = 0 + AND COALESCE(s.output_tokens, 0) = 0 + AND COALESCE(s.pinned, 0) = 0 + AND COALESCE(s.archived, 0) = 0 + AND s.started_at IS NOT NULL + AND s.started_at < ? + AND NOT EXISTS ( + SELECT 1 FROM messages m WHERE m.session_id = s.id + ) + ORDER BY s.started_at + """, + (cutoff,), + ) + return [dict(r) for r in rows] + + def _delete_routing_entries_for_sessions(self, session_ids: Set[str]) -> int: + """Drop ``gateway_routing`` rows pointing at any of *session_ids*. + + The target id lives only inside ``entry_json``, so matching is done in + Python over all scopes. + """ + if not session_ids: + return 0 + rows = self._read_all("SELECT scope, session_key, entry_json FROM gateway_routing") + doomed: List[Tuple[str, str]] = [] + for row in rows: + try: + entry = json.loads(row["entry_json"] or "{}") + except Exception: + continue + if isinstance(entry, dict) and entry.get("session_id") in session_ids: + doomed.append((row["scope"], row["session_key"])) + if not doomed: + return 0 + + self._write_sql( + "DELETE FROM gateway_routing WHERE scope = ? AND session_key = ?", + doomed, + many=True, + ) + return len(doomed) + + def prune_never_active_keyed_sessions( + self, + *, + older_than_days: float, + sessions_dir: Optional[Path] = None, + ) -> Tuple[int, int]: + """Delete never-active keyed rows and the routing entries naming them. + + Returns ``(sessions_deleted, routing_entries_deleted)``. Routing + entries go first: a stale entry outliving its target would have the + gateway resume a nonexistent session id. Deletion goes through + :meth:`delete_session` so the delegate cascade, FTS bookkeeping and + transcript cleanup stay owned by one implementation. + """ + candidates = self.list_never_active_keyed_sessions( + older_than_days=older_than_days + ) + if not candidates: + return (0, 0) + ids = {str(row["id"]) for row in candidates} + routing_deleted = self._delete_routing_entries_for_sessions(ids) + deleted = 0 + for session_id in ids: + if self.delete_session(session_id, sessions_dir=sessions_dir): + deleted += 1 + return (deleted, routing_deleted) + + def list_gateway_sessions( + self, + *, + platform: Optional[str] = None, + active_only: bool = True, + ) -> List[Dict[str, Any]]: + """List gateway sessions (rows with a session_key): newest row per key, + one live mapping per routing key. ``platform`` filters on ``source``.""" + # Full rows carry token/cost totals — drain queued async accounting + # deltas so consumers see exact counters. + self.flush_token_counts() + query = f""" + SELECT sessions.*, + COALESCE(sp.prompt, sessions.system_prompt) + AS _system_prompt_resolved, + {_sql_session_last_active("sessions")} AS last_active + FROM sessions + LEFT JOIN system_prompts sp + ON sp.hash = sessions.system_prompt_hash + WHERE session_key IS NOT NULL + AND started_at = ( + SELECT MAX(s2.started_at) FROM sessions s2 + WHERE s2.session_key = sessions.session_key + ) + """ + params: list = [] + if platform: + query += " AND LOWER(source) = LOWER(?)" + params.append(platform) + if active_only: + query += " AND ended_at IS NULL" + query += " ORDER BY last_active DESC" + rows = self._read_all(query, params) + return [self._session_row_dict(r) for r in rows] + + def find_latest_gateway_session_for_peer( + self, + *, + source: str, + user_id: Optional[str] = None, + session_key: Optional[str] = None, + chat_id: Optional[str] = None, + chat_type: Optional[str] = None, + thread_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Find the latest recoverable gateway session for a routing peer. + + ``sessions.json`` is the fast index but can be missing or pruned; the + durable ``session_key`` on the row rebuilds the mapping exactly. Rows + ended only by the old ``agent_close`` bug or a mistaken TUI + ``ws_orphan_reap`` are recoverable; explicit boundaries (/new, /resume + switches, compression splits) are not. + + Ordering: rank by ``last_activity_at`` (falling back to ``started_at``) + — ``started_at`` alone resurrected days-old zombie rows over the live + conversation. Rows with messages win, but an empty keyed row is still + returned rather than ``None``: ``None`` mints a brand-new session id, + worse than resuming an empty-but-correctly-keyed row (whose transcript + may live under a compression child). + + Reset fence: an intentional boundary (``session_reset`` or any + non-recoverable end_reason) must block fallback to an *older* row for + the same peer, or the has-messages ranking could reach behind a /new + and restore the exact context the user reset — so a candidate is + rejected when a peer boundary row ended *after* its last activity. + """ + if not session_key: + return None + with self._read_ctx() as conn: + row = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) AS _has_messages + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.session_key = ? + AND s.source = ? + AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) + AND NOT EXISTS ( + SELECT 1 FROM sessions b + WHERE b.session_key = s.session_key + AND b.source = s.source + AND b.ended_at IS NOT NULL + AND b.end_reason IN ({_RESET_END_REASONS_SQL}) + AND b.ended_at + > COALESCE(s.last_activity_at, s.started_at) + ) + ORDER BY _has_messages DESC, + COALESCE(s.last_activity_at, s.started_at) DESC + LIMIT 1 + """, + (session_key, source), + ).fetchone() + if row is not None: + return self._session_row_dict(row) + + # Conservative fallback for a temporarily-missing exact key: still + # require the complete peer tuple so we never cross chats/threads/users. + if chat_id is None or chat_type is None: + return None + # Profile fence: a Telegram DM's peer tuple is identical for every + # bot (chat_id == user_id, no thread), so a sibling profile's legacy + # row would otherwise be adopted. A row is ours when profile_name is + # the owner or NULL (legacy rows this store minted); stores outside + # the profile tree derive no owner and stay unfenced. + owner = self._own_profile_name() + row = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) AS _has_messages + FROM sessions s + LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash + WHERE s.source = ? + AND COALESCE(s.user_id, '') = COALESCE(?, '') + AND COALESCE(s.chat_id, '') = COALESCE(?, '') + AND COALESCE(s.chat_type, '') = COALESCE(?, '') + AND COALESCE(s.thread_id, '') = COALESCE(?, '') + AND (? IS NULL OR COALESCE(s.profile_name, ?) = ?) + AND (s.ended_at IS NULL OR s.end_reason IN ({_RECOVERABLE_END_REASONS_SQL})) + AND (COALESCE(s.message_count, 0) > 0 OR EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = s.id LIMIT 1 + )) + AND NOT EXISTS ( + SELECT 1 FROM sessions b + WHERE b.source = s.source + AND COALESCE(b.user_id, '') = COALESCE(s.user_id, '') + AND COALESCE(b.chat_id, '') = COALESCE(s.chat_id, '') + AND COALESCE(b.chat_type, '') = COALESCE(s.chat_type, '') + AND COALESCE(b.thread_id, '') = COALESCE(s.thread_id, '') + AND b.ended_at IS NOT NULL + AND b.end_reason IN ({_RESET_END_REASONS_SQL}) + AND b.ended_at + > COALESCE(s.last_activity_at, s.started_at) + ) + ORDER BY COALESCE(s.last_activity_at, s.started_at) DESC + LIMIT 1 + """, + (source, user_id, chat_id, chat_type, thread_id, owner, owner, owner), + ).fetchone() + return self._session_row_dict(row) if row else None + + def find_orphaned_gateway_sessions( + self, *, max_gap_s: Optional[float] = None + ) -> List[Dict[str, Any]]: + """Report message-bearing session rows that lost their routing identity. + + A candidate orphan has messages but no ``session_key``; it is + *adoptable* only when exactly one keyed predecessor can be named: + + * ``lineage`` — ``parent_session_id`` points at a keyed row of the + same source (a recorded fact; no time window). + * ``contiguity`` — exactly one keyed row of the same source (and + compatible ``user_id``) fell quiet within *max_gap_s* of the + orphan's start, and is older than the orphan's own last activity. + + Ambiguity is reported ``adoptable=False`` with a reason, never guessed: + mis-adopting splices one person's conversation into another's chat. + Branch/delegate/tool rows are excluded — unkeyed by design, not damage. + """ + gap = ( + self._ORPHAN_ADOPTION_MAX_GAP_S + if max_gap_s is None + else float(max_gap_s) + ) + orphan_active = _sql_session_last_active("o") + donor_active = _sql_session_last_active("d") + donor_columns = ( + "d.id, d.session_key, d.chat_id, d.chat_type, d.thread_id, " + "d.user_id, d.origin_json, d.display_name, d.end_reason" + ) + records: List[Dict[str, Any]] = [] + + with self._read_ctx() as conn: + orphans = conn.execute( + f""" + SELECT o.id, o.source, o.user_id, o.started_at, + o.parent_session_id, + {orphan_active} AS last_active, + (SELECT COUNT(*) FROM messages m + WHERE m.session_id = o.id) AS message_count + FROM sessions o + WHERE o.session_key IS NULL + AND EXISTS (SELECT 1 FROM messages m + WHERE m.session_id = o.id) + AND COALESCE(o.source, '') != 'tool' + AND json_extract(COALESCE(o.model_config, '{{}}'), + '$._branched_from') IS NULL + AND json_extract(COALESCE(o.model_config, '{{}}'), + '$._delegate_from') IS NULL + ORDER BY o.started_at ASC + """ + ).fetchall() + + for orphan in orphans: + donor = None + evidence = "" + reason = "" + + if orphan["parent_session_id"]: + evidence = "lineage" + donor = conn.execute( + f""" + SELECT {donor_columns} + FROM sessions d + WHERE d.id = ? + AND d.session_key IS NOT NULL + AND COALESCE(d.source, '') = COALESCE(?, '') + """, + (orphan["parent_session_id"], orphan["source"]), + ).fetchone() + if donor is None: + reason = ( + "parent session carries no gateway identity of " + "this source" + ) + else: + evidence = "contiguity" + candidates = conn.execute( + f""" + SELECT {donor_columns}, {donor_active} AS last_active + FROM sessions d + WHERE d.session_key IS NOT NULL + AND d.id != ? + AND COALESCE(d.source, '') = COALESCE(?, '') + AND (COALESCE(d.user_id, '') = '' + OR COALESCE(?, '') = '' + OR d.user_id = ?) + AND {donor_active} BETWEEN ? AND ? + AND {donor_active} < ? + ORDER BY last_active DESC + LIMIT 2 + """, + ( + orphan["id"], + orphan["source"], + orphan["user_id"], + orphan["user_id"], + (orphan["started_at"] or 0) - gap, + (orphan["started_at"] or 0) + gap, + orphan["last_active"], + ), + ).fetchall() + if not candidates: + reason = ( + f"no keyed predecessor fell quiet within {gap:.0f}s " + "of this session's start" + ) + elif len(candidates) > 1: + reason = ( + "ambiguous: more than one keyed predecessor " + "matches this window" + ) + else: + donor = candidates[0] + + records.append( + { + "orphan_id": orphan["id"], + "source": orphan["source"], + "message_count": orphan["message_count"], + "started_at": orphan["started_at"], + "last_active": orphan["last_active"], + "donor_id": donor["id"] if donor else None, + "session_key": donor["session_key"] if donor else None, + "evidence": evidence if donor else "", + "adoptable": donor is not None, + "reason": reason, + } + ) + + # Two unkeyed successors claiming one predecessor: at most one continues + # that chat, and nothing here says which. + contested = { + r["donor_id"] + for r in records + if r["adoptable"] + and sum(1 for x in records if x["donor_id"] == r["donor_id"]) > 1 + } + for record in records: + if record["donor_id"] in contested: + record["adoptable"] = False + record["reason"] = ( + "ambiguous: more than one unkeyed session claims this " + "predecessor" + ) + return records + + def adopt_orphaned_gateway_session( + self, orphan_id: str, donor_id: str + ) -> bool: + """Stamp *orphan_id* with *donor_id*'s routing identity, retire *donor_id*. + + Re-verifies the pair inside the write transaction so a concurrent + gateway that healed either row makes this a no-op, not a conflicting + write. Non-NULL orphan columns are preserved. True when applied. + """ + if not orphan_id or not donor_id or orphan_id == donor_id: + return False + + def _do(conn): + donor = conn.execute( + "SELECT session_key, chat_id, chat_type, thread_id, user_id, " + "origin_json, display_name, source FROM sessions WHERE id = ?", + (donor_id,), + ).fetchone() + orphan = conn.execute( + "SELECT session_key, source FROM sessions WHERE id = ?", + (orphan_id,), + ).fetchone() + if donor is None or orphan is None: + return False + if not donor["session_key"] or orphan["session_key"]: + return False + if (donor["source"] or "") != (orphan["source"] or ""): + return False + + conn.execute( + """UPDATE sessions + SET session_key = ?, + chat_id = COALESCE(chat_id, ?), + chat_type = COALESCE(chat_type, ?), + thread_id = COALESCE(thread_id, ?), + user_id = COALESCE(user_id, ?), + origin_json = COALESCE(origin_json, ?), + display_name = COALESCE(display_name, ?), + parent_session_id = COALESCE(parent_session_id, ?) + WHERE id = ? AND session_key IS NULL""", + ( + donor["session_key"], + donor["chat_id"], + donor["chat_type"], + donor["thread_id"], + donor["user_id"], + donor["origin_json"], + donor["display_name"], + donor_id, + orphan_id, + ), + ) + # Retire the predecessor under a reason recovery does NOT treat as + # resumable — 'agent_close'/'ws_orphan_reap' would keep it in the + # running and the newly keyed orphan could lose the chat again. + conn.execute( + "UPDATE sessions SET ended_at = COALESCE(ended_at, ?), " + "end_reason = 'superseded_by_repair' WHERE id = ?", + (time.time(), donor_id), + ) + return True + + return self._execute_write(_do) + + def increment_hygiene_failure_streak(self, session_key: str) -> int: + """Atomically increment the session-hygiene failure streak for one chat.""" + if not session_key: + return 1 + result = [] + + def _do(conn): + conn.execute( + """INSERT INTO gateway_hygiene_state (session_key, failure_streak) + VALUES (?, 1) + ON CONFLICT(session_key) DO UPDATE SET + failure_streak = gateway_hygiene_state.failure_streak + 1""", + (session_key,), + ) + row = conn.execute( + "SELECT failure_streak FROM gateway_hygiene_state WHERE session_key = ?", + (session_key,), + ).fetchone() + result.append(int(row[0])) + + self._execute_write(_do) + return result[0] + + def reset_hygiene_failure_streak(self, session_key: str) -> None: + """Clear the persisted session-hygiene failure streak for one chat.""" + if not session_key: + return + + self._write_sql( + "DELETE FROM gateway_hygiene_state WHERE session_key = ?", + (session_key,), + ) + + @staticmethod + def session_gateway_runtime(session_meta: Optional[Dict[str, Any]]) -> Dict[str, Any]: + """Read the persisted runtime route off a session row dict. + + Accepts ``get_session``'s dict (``model_config`` as JSON string) or a + parsed dict. Precedence: nested ``gateway_runtime`` (gateway sync / CLI + ``/model``), then top-level ``provider``/``base_url``/``api_mode`` (TUI + ``_runtime_model_config``), then ``billing_provider`` so sessions that + never ran ``/model`` still restore the provider that served them. + Empty dict on parse failure — resume uses ambient config. + """ + from hermes_state import _BARE_BILLING_PROVIDERS + raw = (session_meta or {}).get("model_config") + if isinstance(raw, str): + try: + raw = json.loads(raw) + except Exception: + raw = {} + if not isinstance(raw, dict): + raw = {} + runtime = raw.get("gateway_runtime") + if isinstance(runtime, dict) and runtime.get("provider"): + # Filter None: the persist path writes or-None to trigger deletion + # in the top-level merge, but gateway_runtime is replaced whole + # (not deep-merged), so None values survive here. + return {k: v for k, v in runtime.items() if v is not None} + top_level = { + key: raw.get(key) + for key in ("provider", "base_url", "api_mode") + if raw.get(key) + } + if top_level: + return top_level + # Last resort: billing_provider, COALESCE-written on the first accounted + # API call — the only durable record for sessions that never ran /model. + # Bare buckets ("auto"/"custom") are not routable identities; filter + # them so resume falls back to the ambient config default. + billing_provider = str( + (session_meta or {}).get("billing_provider") or "" + ).strip() + if ( + billing_provider + and billing_provider.lower() not in _BARE_BILLING_PROVIDERS + ): + return {"provider": billing_provider} + return {k: v for k, v in (runtime or {}).items() if v is not None} if isinstance(runtime, dict) else {} + + def register_backend_heartbeat( + self, + *, + backend_id: str, + pid: int, + started_at: float, + last_heartbeat: Optional[float] = None, + profile: str = "", + host: str = "", + ) -> None: + """Upsert this backend's liveness row. + + ``backend_id`` MUST be stable for the process lifetime (e.g. + ``f"{profile}@{host}:{pid}"``) so a respawn cannot inherit a dead + predecessor's heartbeat and protect stale rows. ``started_at`` is when + THIS process started, not first-refresh wall clock, so a backend whose + previous run died is not mistaken for a freshly-spawned sibling. + """ + if not backend_id: + return + ts = time.time() if last_heartbeat is None else float(last_heartbeat) + self._write_sql( + "INSERT INTO gateway_heartbeats" + " (backend_id, pid, started_at, last_heartbeat, profile, host)" + " VALUES (?, ?, ?, ?, ?, ?)" + " ON CONFLICT(backend_id) DO UPDATE SET" + " pid = excluded.pid," + " started_at = excluded.started_at," + " last_heartbeat = excluded.last_heartbeat," + " profile = excluded.profile," + " host = excluded.host", + (str(backend_id), int(pid), float(started_at), ts, + str(profile), str(host)), + ) + + def clear_backend_heartbeat(self, backend_id: str) -> bool: + """Remove this backend's heartbeat row (from ``atexit``); True if removed. + A crashed backend's row is reclaimed later by ``prune_stale_heartbeats``.""" + if not backend_id: + return False + return self._write_rowcount( + "DELETE FROM gateway_heartbeats WHERE backend_id = ?", + (str(backend_id),), + ) > 0 + + def prune_stale_heartbeats(self, *, max_age_seconds: float) -> List[str]: + """Drop heartbeat rows older than the staleness window; return removed + backend ids. Safe from any process — only stale rows are touched.""" + if max_age_seconds <= 0: + return [] + cutoff = time.time() - max_age_seconds + def _do(conn): + cur = conn.execute( + "DELETE FROM gateway_heartbeats WHERE last_heartbeat < ?" + " RETURNING backend_id", + (cutoff,), + ) + return [str(r[0]) for r in cur.fetchall()] + return list(self._execute_write(_do) or []) + + def list_backend_heartbeats(self) -> List[Dict[str, Any]]: + """Snapshot of every backend heartbeat (diagnostics/tests); fields mirror the table.""" + rows = self._read_all( + "SELECT backend_id, pid, started_at, last_heartbeat," + " profile, host FROM gateway_heartbeats" + " ORDER BY last_heartbeat DESC", + ) + out: List[Dict[str, Any]] = [] + for r in rows: + if isinstance(r, sqlite3.Row): + out.append({k: r[k] for k in r.keys()}) + else: + out.append({ + "backend_id": r[0], "pid": r[1], "started_at": r[2], + "last_heartbeat": r[3], "profile": r[4], "host": r[5], + }) + return out + + def request_handoff(self, session_id: str, platform: str) -> bool: + """Mark a session pending handoff to *platform*; False if a handoff is already in flight.""" + return self._write_rowcount( + "UPDATE sessions " + "SET handoff_state = 'pending', " + " handoff_platform = ?, " + " handoff_error = NULL " + "WHERE id = ? AND (handoff_state IS NULL " + " OR handoff_state IN ('completed', 'failed'))", + (platform, session_id), + ) > 0 + + def get_handoff_state(self, session_id: str) -> Optional[Dict[str, Any]]: + """Return ``{"state", "platform", "error"}`` or None if the session has no handoff record.""" + try: + row = self._read_one( + "SELECT handoff_state, handoff_platform, handoff_error " + "FROM sessions WHERE id = ?", + (session_id,), + ) + if not row: + return None + return { + "state": row["handoff_state"], + "platform": row["handoff_platform"], + "error": row["handoff_error"], + } + except Exception: + return None + + def list_pending_handoffs(self) -> List[Dict[str, Any]]: + """All sessions in handoff_state='pending', oldest first (gateway handoff watcher).""" + try: + rows = self._read_all( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.handoff_state = 'pending' " + "ORDER BY s.started_at ASC", + ) + return [self._session_row_dict(r) for r in rows] + except Exception: + return [] + + def claim_handoff(self, session_id: str) -> bool: + """Atomically transition pending → running. Returns True if claimed.""" + return self._write_rowcount( + "UPDATE sessions SET handoff_state = 'running' " + "WHERE id = ? AND handoff_state = 'pending'", + (session_id,), + ) > 0 + + def complete_handoff(self, session_id: str) -> None: + """Mark a handoff as completed.""" + self._write_sql( + "UPDATE sessions SET handoff_state = 'completed', " + "handoff_error = NULL WHERE id = ?", + (session_id,), + ) + + def fail_handoff( + self, + session_id: str, + error: str, + *, + only_states: Optional[Tuple[str, ...]] = None, + ) -> bool: + """Mark a handoff failed and record the reason; True when a row transitioned. + + ``only_states`` makes the write a compare-and-swap on ``handoff_state``. + Waiters that give up (CLI 60s poll, Desktop bounded poll) MUST pass + ``only_states=("pending",)``: once the gateway watcher has claimed the + row (``running``) it owns the terminal state, and an unconditional + waiter-side fail races the dispatch — the gateway later overwrites + ``failed`` → ``completed`` after the user was told the gateway is down + (split-brain: the handoff delivered and ``switch_session`` re-pointed + the session). The watcher fails its OWN claimed row unconditionally. + """ + def _do(conn): + if only_states: + placeholders = ", ".join("?" for _ in only_states) + cur = conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + f"handoff_error = ? WHERE id = ? AND handoff_state IN ({placeholders})", + (error[:500], session_id, *only_states), + ) + else: + cur = conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + "handoff_error = ? WHERE id = ?", + (error[:500], session_id), + ) + return cur.rowcount > 0 + return bool(self._execute_write(_do)) + + def reclaim_stale_running_handoffs(self, error: str) -> List[str]: + """Fail every handoff stuck in ``running``. Returns the ids reclaimed. + + Only the gateway watcher sets ``running``, and only for one in-process + dispatch — so a ``running`` row at watcher startup belongs to a PREVIOUS + gateway that died mid-dispatch. It is poisonous: ``request_handoff`` + only accepts NULL/``completed``/``failed``, so the session could never + hand off again, with no error surfaced. Failing rather than re-queueing + is deliberate: the dead gateway may already have switched the session + key and dispatched the synthetic turn, so a blind retry risks double + delivery; a clean terminal state the user can retry from is right. + """ + def _do(conn): + cur = conn.execute( + "SELECT id FROM sessions WHERE handoff_state = 'running'" + ) + ids = [r[0] for r in cur.fetchall()] + if ids: + conn.execute( + "UPDATE sessions SET handoff_state = 'failed', " + "handoff_error = ? WHERE handoff_state = 'running'", + (error[:500],), + ) + return ids + try: + return self._execute_write(_do) or [] + except Exception: + # Swallow but never silently: a persistently failing reclaim leaves + # poisonous 'running' rows in place, so the operator needs a trace. + logger.warning( + "reclaim_stale_running_handoffs failed; stranded 'running' " + "handoff rows (if any) were left in place", exc_info=True, + ) + return [] diff --git a/hermes_state_maintenance.py b/hermes_state_maintenance.py new file mode 100644 index 0000000000..89829b9191 --- /dev/null +++ b/hermes_state_maintenance.py @@ -0,0 +1,699 @@ +"""Retention pruning, stale-session archiving and VACUUM policy mixin for +SessionDB.""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from hermes_state_common import ( + AUTO_VACUUM_MIN_FREELIST_RATIO, + _sql_session_last_active, + escape_like as _escape_like, +) + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionMaintenanceMixin: + """Retention pruning, stale-session archiving and VACUUM policy for SessionDB.""" + + def prune_empty_ghost_sessions(self, sessions_dir: "Optional[Path]" = None) -> int: + """Remove empty TUI ghost sessions (no messages, no title, >24hr old).""" + cutoff = time.time() - 86400 + + def _do(conn): + rows = conn.execute(""" + SELECT id FROM sessions + WHERE source = 'tui' + AND title IS NULL + AND ended_at IS NOT NULL + AND started_at < ? + AND NOT EXISTS ( + SELECT 1 FROM messages WHERE messages.session_id = sessions.id + ) + """, (cutoff,)).fetchall() + ids = [r[0] for r in rows] + if ids: + placeholders = ",".join("?" * len(ids)) + conn.execute( + f"DELETE FROM sessions WHERE id IN ({placeholders})", ids + ) + self._delete_unreferenced_system_prompts(conn) + return ids + + removed_ids = self._execute_write(_do) or [] + if sessions_dir and removed_ids: + for sid in removed_ids: + self._remove_session_files(sessions_dir, sid) + return len(removed_ids) + + def sweep_orphaned_sessions( + self, + *, + max_idle_seconds: float, + sources: Tuple[str, ...] = ("tui", "desktop", "subagent"), + exclude_ids: Tuple[str, ...] = (), + exclude_pinned: bool = False, + heartbeat_staleness_seconds: Optional[float] = None, + heartbeat_ownership_grace_seconds: Optional[float] = None, + respect_gateway_heartbeats: bool = True, + ) -> List[str]: + """Close session rows orphaned by a dead gateway process. + + The TUI/desktop gateway reaps disconnected sessions with an in-process + grace timer; a restart destroys the timer and leaves ``ended_at IS + NULL`` forever. This closes rows for ``sources`` whose ``started_at`` + AND canonical last activity (newest of ``last_activity_at`` and the + newest message, else ``started_at``) are both older than + ``max_idle_seconds``, with ``end_reason='startup_orphan_reap'``. The + separate ``started_at`` predicate protects fresh compression/branch + children whose copied activity is old. + + Only pass sources whose lifecycle the caller owns — never messaging + platforms like ``telegram`` (ending those triggers a routing loop). + ``exclude_ids`` spares rows this process still holds in memory. + Non-destructive: messages are kept and the row stays resumable; + first-reason-wins via ``ended_at IS NULL``. + + Cross-backend liveness: with ``respect_gateway_heartbeats``, a row is + reaped only when stale AND no live backend (heartbeat within + ``heartbeat_staleness_seconds``, default ``2 * max_idle_seconds``) could + own it, where backend B owns session S if ``B.started_at <= S.started_at + + heartbeat_ownership_grace_seconds`` (default = staleness). The grace + covers a migrating backend whose sessions predate its first heartbeat + but is bounded so a PID-reuse respawn cannot protect rows forever. + Disable the gate only for sources owned by state.db itself. + + SELECT, live-lease validation and UPDATE run in one ``BEGIN IMMEDIATE`` + transaction. Active turn leases / compression locks spare the row; + expired guards are removed so their former owner is fenced. + """ + from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError + srcs = tuple(s for s in sources if s) + if max_idle_seconds <= 0 or not srcs: + return [] + hb_staleness = ( + heartbeat_staleness_seconds + if heartbeat_staleness_seconds and heartbeat_staleness_seconds > 0 + else max_idle_seconds * 2 + ) + hb_grace = ( + heartbeat_ownership_grace_seconds + if heartbeat_ownership_grace_seconds is not None + and heartbeat_ownership_grace_seconds >= 0 + else hb_staleness + ) + now = time.time() + cutoff = now - max_idle_seconds + hb_cutoff = now - hb_staleness + placeholders = ",".join("?" for _ in srcs) + staleness = ( + f"started_at < ? AND {_sql_session_last_active('sessions')} < ?" + ) + pin_scope = " AND COALESCE(pinned, 0) = 0" if exclude_pinned else "" + heartbeat_params: Tuple[float, ...] = () + orphan_predicate = staleness + if respect_gateway_heartbeats: + orphan_predicate += ( + " AND NOT EXISTS (" + "SELECT 1 FROM gateway_heartbeats h" + " WHERE h.last_heartbeat >= ?" + " AND h.started_at <= sessions.started_at + ?" + ")" + ) + heartbeat_params = (hb_cutoff, hb_grace) + + def _do(conn): + rows = conn.execute( + f"SELECT id FROM sessions WHERE ended_at IS NULL" + f" AND source IN ({placeholders}){pin_scope}" + f" AND {orphan_predicate}", + (*srcs, cutoff, cutoff, *heartbeat_params), + ).fetchall() + excluded = {str(x) for x in exclude_ids if x} + victims = [] + for row in rows: + sid = str(row["id"]) + if sid in excluded: + continue + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + continue + victims.append(sid) + if not victims: + return [] + closed_at = time.time() + marks = ",".join("?" for _ in victims) + # Re-apply every predicate under the write lock. + conn.execute( + f"UPDATE sessions SET ended_at = ?, end_reason = 'startup_orphan_reap'" + f" WHERE id IN ({marks}) AND ended_at IS NULL" + f" AND source IN ({placeholders}){pin_scope}" + f" AND {orphan_predicate}", + ( + closed_at, + *victims, + *srcs, + cutoff, + cutoff, + *heartbeat_params, + ), + ) + return victims + + return self._execute_write(_do) or [] + + @staticmethod + def _prune_filter_where( + *, + last_active_before: Optional[float] = None, + last_active_after: Optional[float] = None, + started_before: Optional[float] = None, + started_after: Optional[float] = None, + source: Optional[str] = None, + title_like: Optional[str] = None, + end_reason: Optional[str] = None, + cwd_prefix: Optional[str] = None, + min_messages: Optional[int] = None, + max_messages: Optional[int] = None, + archived: Optional[bool] = None, + model_like: Optional[str] = None, + provider: Optional[str] = None, + user_id: Optional[str] = None, + chat_id: Optional[str] = None, + chat_type: Optional[str] = None, + branch_like: Optional[str] = None, + min_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + min_cost: Optional[float] = None, + max_cost: Optional[float] = None, + min_tool_calls: Optional[int] = None, + max_tool_calls: Optional[int] = None, + include_pinned: bool = False, + ) -> Tuple[str, list]: + """Shared WHERE clause for bulk prune/archive selection (alias ``s``). + + Filters AND together; only ended sessions are ever candidates. + ``archived`` is tri-state (None = both). ``*_like`` filters are + case-insensitive substrings; the rest are exact (provider + case-insensitive). Token bounds use input+output; cost bounds use + ``COALESCE(actual_cost_usd, estimated_cost_usd)``. + """ + from hermes_state import _cwd_prefix_clause + clauses = ["s.ended_at IS NOT NULL"] + params: list = [] + if last_active_before is not None: + clauses.append( + """COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) < ?""" + ) + params.append(last_active_before) + # Orphan-swept rows age from the sweep, not their old activity, or + # the next prune pass deletes them before the user can recover. + clauses.append( + "(COALESCE(s.end_reason, '') != 'startup_orphan_reap' " + "OR s.ended_at < ?)" + ) + params.append(last_active_before) + if last_active_after is not None: + clauses.append( + """COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) >= ?""" + ) + params.append(last_active_after) + if started_before is not None: + clauses.append("s.started_at < ?") + params.append(started_before) + if started_after is not None: + clauses.append("s.started_at >= ?") + params.append(started_after) + if source: + clauses.append("s.source = ?") + params.append(source) + if title_like: + clauses.append("LOWER(COALESCE(s.title, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(title_like.lower())}%") + if end_reason: + clauses.append("s.end_reason = ?") + params.append(end_reason) + if cwd_prefix: + clause, clause_params = _cwd_prefix_clause(cwd_prefix) + clauses.append(clause) + params.extend(clause_params) + if min_messages is not None: + clauses.append("s.message_count >= ?") + params.append(min_messages) + if max_messages is not None: + clauses.append("s.message_count <= ?") + params.append(max_messages) + if model_like: + clauses.append("LOWER(COALESCE(s.model, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(model_like.lower())}%") + if provider: + clauses.append("LOWER(COALESCE(s.billing_provider, '')) = ?") + params.append(provider.lower()) + if user_id: + clauses.append("s.user_id = ?") + params.append(user_id) + if chat_id: + clauses.append("s.chat_id = ?") + params.append(chat_id) + if chat_type: + clauses.append("s.chat_type = ?") + params.append(chat_type) + if branch_like: + clauses.append("LOWER(COALESCE(s.git_branch, '')) LIKE ? ESCAPE '\\'") + params.append(f"%{_escape_like(branch_like.lower())}%") + if min_tokens is not None: + clauses.append( + "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) >= ?" + ) + params.append(min_tokens) + if max_tokens is not None: + clauses.append( + "(COALESCE(s.input_tokens, 0) + COALESCE(s.output_tokens, 0)) <= ?" + ) + params.append(max_tokens) + if min_cost is not None: + clauses.append( + "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) >= ?" + ) + params.append(min_cost) + if max_cost is not None: + clauses.append( + "COALESCE(s.actual_cost_usd, s.estimated_cost_usd, 0) <= ?" + ) + params.append(max_cost) + if min_tool_calls is not None: + clauses.append("COALESCE(s.tool_call_count, 0) >= ?") + params.append(min_tool_calls) + if max_tool_calls is not None: + clauses.append("COALESCE(s.tool_call_count, 0) <= ?") + params.append(max_tool_calls) + if archived is True: + clauses.append("s.archived = 1") + elif archived is False: + clauses.append("s.archived = 0") + # Pinned is a durable "keep" flag: bulk prune/delete/archive exclude + # pinned rows unless the caller explicitly opts in. + if not include_pinned: + clauses.append("COALESCE(s.pinned, 0) = 0") + return " AND ".join(clauses), params + + @staticmethod + def _apply_prune_age_filter( + older_than_days: Optional[float], filters: Dict[str, Any] + ) -> None: + """Translate the legacy age window into the shared activity filter.""" + if ( + filters.get("last_active_before") is None + and filters.get("started_before") is None + and older_than_days is not None + ): + filters["last_active_before"] = time.time() - ( + older_than_days * 86400 + ) + + def list_prune_candidates( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> List[Dict[str, Any]]: + """Sessions a matching prune/archive would touch (dry-run), oldest + first. Same filters as :meth:`_prune_filter_where`; ``older_than_days`` + is an inactivity threshold (latest message, else ``started_at``).""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + with self._read_ctx() as conn: + cursor = conn.execute( + f"""SELECT s.id, s.source, s.title, s.model, s.started_at, + COALESCE( + (SELECT MAX(m.timestamp) FROM messages m + WHERE m.session_id = s.id), + s.started_at + ) AS last_active, + s.ended_at, s.message_count, s.archived + FROM sessions s WHERE {where} + ORDER BY last_active ASC, s.started_at ASC""", + params, + ) + return [dict(row) for row in cursor.fetchall()] + + def count_prune_matches( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> int: + """Count-only variant of :meth:`list_prune_candidates` (the CLI uses it + to report how many pinned sessions are spared).""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + with self._read_ctx() as conn: + cursor = conn.execute( + f"SELECT COUNT(*) FROM sessions s WHERE {where}", params + ) + return int(cursor.fetchone()[0]) + + def count_open_prune_matches( + self, + older_than_days: Optional[float] = None, + source: str = None, + **filters, + ) -> int: + """Count open sessions a matching prune skips: every normal filter with + only the ``ended_at`` guard inverted. Visibility-only; live sessions + never become prune-eligible.""" + self._apply_prune_age_filter(older_than_days, filters) + where, params = self._prune_filter_where(source=source, **filters) + ended_guard = "s.ended_at IS NOT NULL" + if not where.startswith(ended_guard): + raise RuntimeError("prune filter lost its ended-session safety guard") + open_where = f"s.ended_at IS NULL{where[len(ended_guard):]}" + with self._read_ctx() as conn: + cursor = conn.execute( + f"SELECT COUNT(*) FROM sessions s WHERE {open_where}", params + ) + return int(cursor.fetchone()[0]) + + def archive_stale_sessions( + self, idle_days: float, *, exclude_pinned: bool = True + ) -> int: + """Archive every session untouched for ``idle_days`` (real recency: + freshest of ``last_activity_at`` / latest message / ``started_at``). + Unlike :meth:`archive_sessions`, this can archive unended sessions. + + Guards: ``pinned = 0`` when ``exclude_pinned``; ``archived = 0`` so + repeats are no-ops; only lineage tips (``end_reason <> 'compression'``) + are candidates — a stale tip archives its chain via + :meth:`set_session_archived`, so an old compressed-away root with a + recent continuation is never matched. Returns the count archived. + """ + if idle_days is None or idle_days < 0: + return 0 + cutoff = time.time() - float(idle_days) * 86400.0 + pin_clause = "AND s.pinned = 0" if exclude_pinned else "" + rows = self._read_all( + f""" + SELECT s.id FROM sessions s + WHERE s.archived = 0 + AND COALESCE(s.end_reason, '') <> 'compression' + {pin_clause} + AND {_sql_session_last_active("s")} < ? + ORDER BY s.started_at ASC + """, + (cutoff,), + ) + ids = [r[0] for r in rows] + for sid in ids: + self.set_session_archived(sid, True) + return len(ids) + + def prune_sessions( + self, + older_than_days: Optional[float] = 90, + source: str = None, + sessions_dir: Optional[Path] = None, + exclude_active_write_guards: bool = False, + **filters, + ) -> int: + """Delete ended sessions matching the filters; returns the count. + + Default: inactive for ``older_than_days`` (latest message, else + ``started_at``), optionally by ``source``. Extra keyword filters are + those of :meth:`_prune_filter_where`; an explicit ``started_before`` / + ``last_active_before`` overrides the ``older_than_days`` cutoff + (pass ``older_than_days=None`` for no implicit age bound). + + Children outside the window are orphaned (parent NULLed), not cascade- + deleted. With *sessions_dir*, on-disk transcript files are removed + outside the DB transaction. ``exclude_active_write_guards`` (automatic + maintenance) skips rows under a live turn lease or compression lock, + while expired/dead holders are reclaimed and fenced in the same write. + """ + from hermes_state import SessionCompressionInProgressError, SessionTurnLeaseLostError + self._apply_prune_age_filter(older_than_days, filters) + where, where_params = self._prune_filter_where(source=source, **filters) + removed_ids: list[str] = [] + + def _do(conn): + cursor = conn.execute( + f"SELECT s.id FROM sessions s WHERE {where}", where_params + ) + session_ids = {row["id"] for row in cursor.fetchall()} + + if exclude_active_write_guards: + protected = set() + for sid in session_ids: + try: + self._check_transcript_write_guards( + conn, + sid, + compression_lock_holder=None, + turn_lease_holder=None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + allow_closed_compression_parent=True, + ) + except ( + SessionCompressionInProgressError, + SessionTurnLeaseLostError, + ): + protected.add(sid) + session_ids.difference_update(protected) + + if not session_ids: + return 0 + + placeholders = ",".join("?" * len(session_ids)) + conn.execute( + f"UPDATE sessions SET parent_session_id = NULL " + f"WHERE parent_session_id IN ({placeholders})", + list(session_ids), + ) + + for sid in session_ids: + conn.execute("DELETE FROM messages WHERE session_id = ?", (sid,)) + conn.execute("DELETE FROM sessions WHERE id = ?", (sid,)) + removed_ids.append(sid) + self._delete_unreferenced_system_prompts(conn) + return len(session_ids) + + count = self._execute_write(_do) + for sid in removed_ids: + self._remove_session_files(sessions_dir, sid) + return count + + def logical_size_bytes(self) -> Optional[int]: + """``page_count * page_size``: the main-file size once the WAL is + checkpointed back in. Prefer over ``os.path.getsize`` when reporting a + VACUUM: in WAL mode the rewrite lands in ``-wal`` and the checkpoint is + refused while another connection holds a read-mark, so a stat() delta + understates the win and can go negative. None if pragmas fail. + """ + try: + with self._read_ctx() as conn: + if self._conn is None: + return None + page_count = conn.execute("PRAGMA page_count").fetchone()[0] + page_size = conn.execute("PRAGMA page_size").fetchone()[0] + return int(page_count) * int(page_size) + except Exception as exc: + logger.debug("Could not read logical DB size: %s", exc) + return None + + def _freelist_ratio(self) -> Optional[float]: + """Reclaimable fraction (``freelist_count / page_count``) over the + existing connection — never a byte-level probe of the live file. Gates + VACUUM in :meth:`maybe_auto_prune_and_vacuum`. None if pragmas fail + (callers then fall back to the time throttle alone). + """ + try: + with self._read_ctx() as conn: + if self._conn is None: + return None + page_count = int(conn.execute("PRAGMA page_count").fetchone()[0]) + freelist = int(conn.execute("PRAGMA freelist_count").fetchone()[0]) + if page_count <= 0: + return 0.0 + return freelist / page_count + except Exception as exc: + logger.debug("Could not read freelist ratio: %s", exc) + return None + + def vacuum(self) -> int: + """VACUUM to reclaim space after large deletes (SQLite never shrinks + the file on its own). + + Rewrites the whole DB, cannot run inside a transaction, and takes an + exclusive lock — callers must ensure no other writers are active (safe + at startup before serving traffic). FTS5 segments are merged first via + :meth:`optimize_fts` so the VACUUM reclaims those pages too. Returns + the number of FTS indexes optimized (0 on merge failure / no FTS). + """ + # optimize_fts() manages its own lock. + optimized = 0 + try: + optimized = self.optimize_fts() + except Exception as exc: + logger.warning("FTS optimize before VACUUM failed: %s", exc) + # VACUUM cannot be executed inside a transaction. + with self._lock: + # PASSIVE, not TRUNCATE: a manual `hermes sessions vacuum` runs in + # a transient CLI process, and a TRUNCATE reset here would race a + # live gateway writer and tear B-tree pages. + try: + self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") + except Exception as exc: + logger.debug("WAL checkpoint (PASSIVE) before VACUUM failed: %s", exc) + self._conn.execute("VACUUM") + # VACUUM rewrites every page THROUGH the WAL; without this TRUNCATE + # a 3 GB database leaves a 3 GB -wal behind and the command is a + # net loss on disk. + try: + self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except Exception as exc: + logger.debug("WAL checkpoint (TRUNCATE) after VACUUM failed: %s", exc) + # TRUNCATE may replace the WAL inode; adopt the new sidecars so the + # write-path generation guard does not halt this connection. + self._record_db_file_identity() + return optimized + + def maybe_auto_prune_and_vacuum( + self, + retention_days: int = 90, + min_interval_hours: int = 24, + vacuum: bool = True, + sessions_dir: Optional[Path] = None, + min_vacuum_interval_days: int = 30, + min_vacuum_freelist_ratio: float = AUTO_VACUUM_MIN_FREELIST_RATIO, + ) -> Dict[str, Any]: + """Idempotent startup auto-maintenance: prune inactive sessions, + reap stale open state-owned rows, optional VACUUM. Never raises. + + Runs at most once per ``min_interval_hours`` (state_meta). VACUUM has + its own ``min_vacuum_interval_days`` throttle and additionally requires + ``freelist_count / page_count`` > ``min_vacuum_freelist_ratio`` so a + small prune on a dense multi-GB database never triggers a full rewrite. + With *sessions_dir*, pruned transcripts are removed from disk too. + + Stale-open reconciliation: cron/kanban/subagent/one-shot CLI rows never + set ``ended_at`` when their process dies, and prune only deletes ended + rows. After pruning, open rows from :attr:`_AUTO_PRUNE_STALE_OPEN_SOURCES` + older than ``retention_days`` are closed (``startup_orphan_reap``); they + stay resumable and age from their close, so they get one more full + retention window. Messaging and UI sources are never touched. + + Returns ``{"skipped", "pruned", "closed", "vacuumed"}`` plus + ``"freelist_ratio"`` when a VACUUM was considered and ``"error"`` on + failure. + """ + from hermes_state import _release_auto_maintenance_lock, _try_acquire_auto_maintenance_lock + result: Dict[str, Any] = { + "skipped": False, + "pruned": 0, + "closed": 0, + "vacuumed": False, + } + maintenance_lock = _try_acquire_auto_maintenance_lock(self.db_path) + if maintenance_lock is None: + result["skipped"] = True + return result + try: + last_raw = self.get_meta("last_auto_prune") + now = time.time() + if last_raw: + try: + last_ts = float(last_raw) + if now - last_ts < min_interval_hours * 3600: + result["skipped"] = True + return result + except (TypeError, ValueError): + pass # corrupt meta; treat as no prior run + + # Prune first: orphans closed below get a full retention window. + pruned = self.prune_sessions( + older_than_days=retention_days, + sessions_dir=sessions_dir, + exclude_active_write_guards=True, + ) + result["pruned"] = pruned + + closed = self.sweep_orphaned_sessions( + max_idle_seconds=float(retention_days) * 86400.0, + sources=self._AUTO_PRUNE_STALE_OPEN_SOURCES, + exclude_pinned=True, + # State-owned lifecycles, not gateway heartbeats. + respect_gateway_heartbeats=False, + ) + result["closed"] = len(closed) + # VACUUM only if rows were freed, the time throttle passed ("not + # too often") AND the freelist ratio passed ("only when it pays + # off") — it holds an exclusive lock for a full rewrite. + last_vacuum_raw = self.get_meta("last_vacuum") + vacuum_due = True + if last_vacuum_raw: + try: + vacuum_due = (now - float(last_vacuum_raw)) >= min_vacuum_interval_days * 86400 + except (TypeError, ValueError): + vacuum_due = True + if vacuum and pruned > 0 and vacuum_due: + ratio = self._freelist_ratio() + result["freelist_ratio"] = ratio + if ratio is None or ratio > min_vacuum_freelist_ratio: + try: + self.vacuum() + result["vacuumed"] = True + self.set_meta("last_vacuum", str(now)) + except Exception as exc: + logger.warning("state.db VACUUM failed: %s", exc) + else: + logger.debug( + "state.db auto-maintenance: skipping VACUUM, only " + "%.1f%% of pages reclaimable (threshold %.0f%%)", + ratio * 100.0, + min_vacuum_freelist_ratio * 100.0, + ) + + # Record even when pruned == 0 so the throttle holds. + self.set_meta("last_auto_prune", str(now)) + + if closed or pruned > 0: + logger.info( + "state.db auto-maintenance: closed %d stale open session(s), " + "pruned %d session(s) inactive for %d days%s", + len(closed), + pruned, + retention_days, + " + VACUUM" if result["vacuumed"] else "", + ) + except Exception as exc: + # Maintenance must never block startup. + logger.warning("state.db auto-maintenance failed: %s", exc) + result["error"] = str(exc) + finally: + _release_auto_maintenance_lock(maintenance_lock) + + return result diff --git a/hermes_state_messages.py b/hermes_state_messages.py new file mode 100644 index 0000000000..4e5091d46c --- /dev/null +++ b/hermes_state_messages.py @@ -0,0 +1,2642 @@ +"""Transcript persistence for SessionDB. + +Mixin split out of ``hermes_state.py``; bound onto ``SessionDB`` via the MRO +and built on its ``_read_ctx`` / ``_execute_write`` / ``_write_sql`` / +``_read_one`` / ``_read_all`` primitives. Covers message append / replace / +rewind, reactions, resume-conversation assembly and replayed-user-message +duplicate detection. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Dict, List, Optional, Tuple + +from agent.context_compressor import _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY +from agent.memory_manager import sanitize_context +from agent.message_sanitization import _sanitize_surrogates +from hermes_state_common import ( + _RESET_END_REASONS, + _RESET_END_REASONS_SQL, + _legacy_reset_child_sql, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +class SessionMessagesMixin: + """Message append/replace/rewind, reactions, resume conversations, replay dedupe.""" + + def _bump_conversation_generation(self, conn, session_id: str, end_reason: str) -> None: + """Advance this peer's conversation generation past a boundary. + + Called inside the transaction that writes the boundary, so the + generation and the ``end_reason`` that caused it commit together. + + Only ``_RESET_END_REASONS`` count: ``compression`` continues one + conversation, and an accidental close is not a replacement. Rows with + no ``session_key`` have no routing peer to advance. + + The counter deliberately does NOT read the session rows. An aggregate + over them (COUNT/MAX of boundaries) can return a pair it already + emitted once ``delete_session()`` or bulk pruning removes an ended row, + which would hand a new conversation a retired affinity identity. This + value only ever increments, so a generation is never reused for a peer + even if every row behind it is gone. + """ + if end_reason not in _RESET_END_REASONS: + return + row = conn.execute( + "SELECT source, session_key FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if row is None: + return + source = str(row["source"] or "").strip() + session_key = str(row["session_key"] or "").strip() + if not source or not session_key: + return + conn.execute( + """ + INSERT INTO conversation_generations (source, session_key, generation) + VALUES (?, ?, 1) + ON CONFLICT(source, session_key) DO UPDATE + SET generation = conversation_generations.generation + 1 + """, + (source, session_key), + ) + + @classmethod + def _encode_content(cls, content: Any) -> Any: + """Serialize structured (list/dict) message content for sqlite. + + sqlite3 can only bind ``str``, ``bytes``, ``int``, ``float``, and ``None`` + to query parameters. Multimodal messages have ``content`` as a list of + parts (``[{"type": "text", ...}, {"type": "image_url", ...}]``), which + raises ``ProgrammingError: Error binding parameter N: type 'list' is + not supported`` when bound directly. + + Returns the value unchanged when it's already a safe scalar, or a + sentinel-prefixed JSON string for lists/dicts. Paired with + :meth:`_decode_content` on read. + """ + if isinstance(content, str): + # Lone UTF-16 surrogates reach here inside tool results scraped + # from the web/social platforms (the same input that crashed the + # guardrail hasher). The proactive sanitizer upstream only cleans + # the *api_messages* copy, and the recovery sanitizer only runs + # after the API call itself raises — which it no longer does — so + # the canonical history keeps them and this write is where they + # land. Left raw, sqlite3 raises UnicodeEncodeError, the flush is + # abandoned, and the session silently stops persisting for the + # rest of its life. Scrub so persistence never fails. + return _sanitize_surrogates(content) + if content is None or isinstance(content, (bytes, int, float)): + return content + try: + # json.dumps defaults to ensure_ascii=True, which escapes any + # surrogate as \udXXX — already safe to bind. + return cls._CONTENT_JSON_PREFIX + json.dumps(content) + except (TypeError, ValueError): + # Last-resort fallback: stringify so persistence never fails. + return _sanitize_surrogates(str(content)) + + @classmethod + def _decode_content(cls, content: Any) -> Any: + """Reverse :meth:`_encode_content`; returns scalars unchanged.""" + if isinstance(content, str) and content.startswith(cls._CONTENT_JSON_PREFIX): + try: + return json.loads(content[len(cls._CONTENT_JSON_PREFIX):]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Failed to decode JSON-encoded message content; " + "returning raw string" + ) + return content + return content + + @staticmethod + def _encode_display_metadata(display_metadata: Any) -> Optional[str]: + """Serialize ``display_metadata`` for its TEXT column without double-encoding. + + Import/replace paths can hand us an already-serialized JSON string (the + same hazard ``tool_calls`` guards against above). ``json.dumps`` on that + string would store a quoted JSON string, and the single ``json.loads`` + on read then yields a ``str`` instead of a dict. + """ + if not display_metadata: + return None + if isinstance(display_metadata, str): + try: + parsed = json.loads(display_metadata) + except (json.JSONDecodeError, TypeError): + logger.warning("Ignoring non-JSON display metadata on write") + return None + if not isinstance(parsed, dict): + logger.warning("Ignoring non-object display metadata on write") + return None + return json.dumps(parsed) + if isinstance(display_metadata, dict): + return json.dumps(display_metadata) + logger.warning( + "Ignoring unexpected display metadata type on write: %s", + type(display_metadata).__name__, + ) + return None + + def _check_transcript_write_guards( + self, + conn, + session_id: str, + compression_lock_holder: Optional[str], + turn_lease_holder: Optional[str] = None, + turn_lease_ttl_seconds: float = 300.0, + reject_active_turn_lease: bool = False, + reject_active_compression_lock: bool = False, + allow_closed_compression_parent: bool = False, + ) -> None: + """Transcript-write admission checks, run INSIDE the write txn. + + Shared by :meth:`append_message` and :meth:`append_messages_batch` so + the two writers can never diverge on these correctness invariants + (this guard has already needed targeted fixes — see the #74478 patience + note below). User-initiated transcript mutations may opt in to rejecting + an active unowned turn lease in that same transaction. + """ + from hermes_state import CompressionSessionClosedError, SessionCompressionInProgressError, SessionTurnLeaseLostError, _compression_lock_holder_process_is_dead + # NOTE (#75316 redesign): appends do NOT check compression_locks. + # The lock's job is to stop two COMPRESSIONS colliding, not to fence + # ordinary transcript writes. Concurrent appends during a compression + # are safe by construction: archive_and_compact() commits against a + # watermark captured at compression start and clones every row that + # arrived after it back into the live transcript, in the same write + # transaction. Blocking appends here was the root cause of a whole + # symptom family — turns dying as session_persistence_failed while a + # slow provider summary held the lease (#74568, #77386), including + # stale locks from dead PIDs blocking writes for the full TTL. + # Destructive user mutations are different: a compressor that already + # captured its watermark can otherwise publish the pre-rewind snapshot + # after the mutation and resurrect the removed turn. Keep that narrow + # fence opt-in so ordinary appends retain the watermark behavior. + if reject_active_compression_lock: + active_lock = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if active_lock is not None: + current_holder = active_lock["holder"] + if ( + float(active_lock["expires_at"]) <= time.time() + or _compression_lock_holder_process_is_dead(current_holder) + ): + conn.execute( + "DELETE FROM compression_locks " + "WHERE session_id = ? AND holder = ?", + (session_id, current_holder), + ) + elif current_holder != compression_lock_holder: + raise SessionCompressionInProgressError( + f"Session {session_id!r} is being compressed by another writer" + ) + if turn_lease_holder or reject_active_turn_lease: + conversation_id = self._session_turn_lease_key_on_conn(conn, session_id) + lease = conn.execute( + "SELECT holder, expires_at FROM session_turn_leases " + "WHERE conversation_id = ?", + (conversation_id,), + ).fetchone() + now = time.time() + if turn_lease_holder: + if lease is None or lease["holder"] != turn_lease_holder: + raise SessionTurnLeaseLostError( + f"Session turn lease lost; refusing transcript write " + f"for {session_id!r}" + ) + if float(lease["expires_at"]) <= now: + # Expiry makes the row reclaimable; it does not prove that a + # takeover occurred. BEGIN IMMEDIATE serializes this renewal + # with acquisition, so a still-matching owner can recover from + # a starved refresher without weakening the foreign-holder fence. + conn.execute( + "UPDATE session_turn_leases SET expires_at = ? " + "WHERE conversation_id = ? AND holder = ?", + ( + now + max(0.1, float(turn_lease_ttl_seconds)), + conversation_id, + turn_lease_holder, + ), + ) + elif lease is not None: + current_holder = lease["holder"] + if ( + float(lease["expires_at"]) <= now + or _compression_lock_holder_process_is_dead(current_holder) + ): + # Match acquisition semantics: an expired or provably dead + # owner is reclaimable. Deleting it inside this BEGIN IMMEDIATE + # transaction also fences a stale late flush after the mutation. + conn.execute( + "DELETE FROM session_turn_leases " + "WHERE conversation_id = ? AND holder = ?", + (conversation_id, current_holder), + ) + else: + raise SessionTurnLeaseLostError( + f"Session has an active turn lease; refusing transcript " + f"mutation for {session_id!r}" + ) + session = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + session is not None + and session["ended_at"] is not None + and session["end_reason"] == "compression" + and not allow_closed_compression_parent + ): + raise CompressionSessionClosedError(session_id) + + @staticmethod + def _decode_display_metadata(raw: Any) -> Optional[Dict[str, Any]]: + """Decode a ``display_metadata`` column into the dict every reader expects. + + Every message read path must go through this. Returning the raw TEXT + instead reaches the desktop as a string, where ``'task_count' in meta`` + throws and fails the whole resume. Rows written before the encode guard + landed are double-encoded, so unwrap a second layer when we find one. + """ + if raw is None: + return None + try: + meta = json.loads(raw) if isinstance(raw, str) else raw + if isinstance(meta, str): + meta = json.loads(meta) + except (json.JSONDecodeError, TypeError): + logger.warning("Ignoring invalid display metadata on message row") + return None + if not isinstance(meta, dict): + logger.warning("Ignoring non-object display metadata on message row") + return None + return meta + + @staticmethod + def _reasoning_json_text(value: Any) -> Optional[str]: + """Serialize a structured reasoning field for its TEXT column. + + ``reasoning_details`` / ``codex_reasoning_items`` / ``codex_message_items`` + arrive as list/dict structures from the live runtime, but callers that + round-trip stored rows — ``get_messages`` straight into + ``replace_messages``, e.g. the POST /api/sessions/{id}/fork handler — + hand back the raw TEXT these columns already hold, because + ``get_messages`` only deserializes ``content`` and ``tool_calls``. + Re-dumping that TEXT double-encodes it, and the forked session's next + ``get_messages_as_conversation`` json.loads then yields the inner + string instead of the original list, so every reasoning-replay consumer + (all of which check ``isinstance(..., list)``) silently drops it. + Strings are therefore stored as-is; structures are dumped. + """ + if not value: + return None + if isinstance(value, str): + return value + return json.dumps(value) + + def append_message( + self, + session_id: str, + role: str, + content: str = None, + tool_name: str = None, + tool_calls: Any = None, + tool_call_id: str = None, + token_count: int = None, + finish_reason: str = None, + reasoning: str = None, + reasoning_content: str = None, + reasoning_details: Any = None, + codex_reasoning_items: Any = None, + codex_message_items: Any = None, + platform_message_id: str = None, + observed: bool = False, + effect_disposition: Optional[str] = None, + _compressed_summary: bool = False, + timestamp: Any = None, + api_content: Optional[str] = None, + display_kind: Optional[str] = None, + display_metadata: Optional[Dict[str, Any]] = None, + compression_lock_holder: Optional[str] = None, + turn_lease_holder: Optional[str] = None, + turn_lease_ttl_seconds: float = 300.0, + ) -> int: + """ + Append a message to a session. Returns the message row ID. + + Also increments the session's message_count (and tool_call_count + if role is 'tool' or tool_calls is present). + + ``platform_message_id`` is the external messaging platform's own + message ID (e.g. Telegram update_id, Yuanbao msg_id). It is + independent of the SQLite autoincrement primary key and is used by + platform-specific flows like yuanbao's recall guard to redact a + message by its platform-side identifier. + + ``api_content`` is the exact content string sent to the API for this + message when it differs from ``content`` (ephemeral memory/plugin + injections, persist overrides). It is a byte-fidelity sidecar for + prompt-cache-stable replay — stored as sent, except lone surrogates + (which sqlite3 cannot bind and which the conversation loop scrubs + from every outgoing payload anyway, so the scrubbed form IS the + wire bytes). + """ + from hermes_state import _scrub_surrogates + # Display metadata is presentation-only and never changes the model + # context role/content replayed to providers. + display_metadata_json = self._encode_display_metadata(display_metadata) + # Serialize structured fields to JSON before entering the write txn + reasoning_details_json = self._reasoning_json_text(reasoning_details) + codex_items_json = self._reasoning_json_text(codex_reasoning_items) + codex_message_items_json = self._reasoning_json_text(codex_message_items) + # tool_calls may arrive as a Python list (from the live agent) or + # as a JSON string (from import/export). Parse first to avoid + # double-encoding. + if isinstance(tool_calls, str): + try: + tool_calls = json.loads(tool_calls) + except (json.JSONDecodeError, TypeError): + tool_calls = [] + tool_calls_json = json.dumps(tool_calls) if tool_calls else None + # Multimodal content (list of parts) must be JSON-encoded: sqlite3 + # cannot bind list/dict parameters directly. + stored_content = self._encode_content(content) + + message_timestamp = time.time() + if timestamp is not None: + try: + if hasattr(timestamp, "timestamp"): + message_timestamp = float(timestamp.timestamp()) + else: + message_timestamp = float(timestamp) + except (TypeError, ValueError): + logger.debug("Ignoring invalid explicit message timestamp: %r", timestamp) + + # Pre-compute tool call count + num_tool_calls = 0 + if tool_calls is not None: + num_tool_calls = len(tool_calls) if isinstance(tool_calls, list) else 1 + + def _do(conn): + self._check_transcript_write_guards( + conn, + session_id, + compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + cursor = conn.execute( + """INSERT INTO messages (session_id, role, content, tool_call_id, + tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, + reasoning, reasoning_content, reasoning_details, codex_reasoning_items, + codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + session_id, + role, + stored_content, + tool_call_id, + tool_calls_json, + _scrub_surrogates(tool_name), + effect_disposition, + message_timestamp, + token_count, + finish_reason, + _scrub_surrogates(reasoning), + _scrub_surrogates(reasoning_content), + reasoning_details_json, + codex_items_json, + codex_message_items_json, + platform_message_id, + 1 if observed else 0, + 1 if _compressed_summary else 0, + 1, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, + _scrub_surrogates(display_kind) if isinstance(display_kind, str) else None, + display_metadata_json, + ), + ) + msg_id = cursor.lastrowid + + # Update counters + if num_tool_calls > 0: + conn.execute( + """UPDATE sessions SET message_count = message_count + 1, + tool_call_count = tool_call_count + ? WHERE id = ?""", + (num_tool_calls, session_id), + ) + else: + conn.execute( + "UPDATE sessions SET message_count = message_count + 1 WHERE id = ?", + (session_id,), + ) + return msg_id + + # Transcript append is THE critical write: its failure aborts the + # user's turn (session_persistence_failed). Use the long patience so + # a sibling process legitimately holding the write lock for seconds + # (VACUUM, TRUNCATE checkpoint at close, an older pre-bounded-merge + # process's FTS optimize) can't destroy a healthy turn (#74478). + return self._execute_write( + _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S + ) + + def append_messages_batch( + self, + session_id: str, + messages: List[Dict[str, Any]], + compression_lock_holder: Optional[str] = None, + turn_lease_holder: Optional[str] = None, + chunk_rows: Optional[int] = None, + turn_lease_ttl_seconds: float = 300.0, + ) -> int: + """Append multiple messages atomically in ONE write transaction. + + ``messages`` is a list of dicts in the same shape + :meth:`_insert_message_rows` already consumes for replace/compact/ + import (role, content, tool_name, tool_calls, tool_call_id, + finish_reason, reasoning*, codex_*, timestamp, api_content, + display_kind, display_metadata, ...). Reusing that helper keeps ONE + row-serialization path for every multi-row writer. + + A turn-boundary flush writes the whole turn (user + assistant + tool + rows, typically 3-8 messages) as one BEGIN IMMEDIATE / commit pair + instead of one transaction (and, off WAL, one fsync) per row. + + Atomicity contract: all rows land or none do (the caller re-flushes + unstamped messages on the next attempt). The same admission guards + as :meth:`append_message` run once for the batch — same session, + same instant. + + ``chunk_rows`` bounds the transaction size for LARGE copies (branch + seeds can be thousands of rows; measured: 10k rows ≈ 2.4s inside one + BEGIN IMMEDIATE because the FTS triggers run per row, which would + monopolize the write lock and starve concurrent writers). When set, + the batch commits in chunks of at most that many rows — same + recovery semantics as the old per-row loops (a mid-copy failure + leaves a partial seed), just with bounded lock holds. A turn flush + never needs it. Returns the inserted row count. + """ + if not messages: + return 0 + + if chunk_rows is not None and len(messages) > chunk_rows: + inserted_total = 0 + for start in range(0, len(messages), chunk_rows): + inserted_total += self.append_messages_batch( + session_id, + messages[start:start + chunk_rows], + compression_lock_holder=compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + return inserted_total + + def _do(conn): + self._check_transcript_write_guards( + conn, + session_id, + compression_lock_holder, + turn_lease_holder=turn_lease_holder, + turn_lease_ttl_seconds=turn_lease_ttl_seconds, + ) + from agent.transcript_repair import resolve_and_repair_transcript_batch + + inserted_rows = resolve_and_repair_transcript_batch( + conn, + session_id, + messages, + encode_content_fn=self._encode_content, + decode_content_fn=self._decode_content, + ) + inserted = 0 + tool_calls_total = 0 + if inserted_rows: + inserted, tool_calls_total = self._insert_message_rows( + conn, session_id, inserted_rows + ) + + # One aggregated counter update for the newly inserted rows. + if tool_calls_total > 0: + conn.execute( + """UPDATE sessions SET message_count = message_count + ?, + tool_call_count = tool_call_count + ? WHERE id = ?""", + (inserted, tool_calls_total, session_id), + ) + elif inserted > 0: + conn.execute( + "UPDATE sessions SET message_count = message_count + ? WHERE id = ?", + (inserted, session_id), + ) + return inserted + + # Same criticality as append_message: this IS the turn's transcript. + return self._execute_write( + _do, patience_s=self._TRANSCRIPT_WRITE_PATIENCE_S + ) + + def set_latest_matching_message_display_kind( + self, session_id: str, *, role: str, content: str, display_kind: str, + display_metadata: Optional[Dict[str, Any]] = None, + ) -> bool: + """Stamp presentation metadata on this turn's freshly persisted row. + + The model still receives ``role`` and ``content`` unchanged. Gateway and + CLI synthetic inputs call this immediately after their serial turn has + flushed, preserving producer provenance without classifying by content + during transcript rendering. + """ + from hermes_state import _scrub_surrogates + if not session_id or not content or not display_kind: + return False + + def _do(conn): + row = conn.execute( + "SELECT id FROM messages WHERE session_id = ? AND role = ? " + "AND content = ? AND active = 1 ORDER BY id DESC LIMIT 1", + (session_id, role, self._encode_content(content)), + ).fetchone() + if row is None: + return False + conn.execute( + "UPDATE messages SET display_kind = ?, display_metadata = ? WHERE id = ?", + ( + _scrub_surrogates(display_kind), + self._encode_display_metadata(display_metadata), + row[0], + ), + ) + return True + + return bool(self._execute_write(_do)) + + def set_message_reaction( + self, + session_id: str, + message_row_id: int, + emoji: Optional[str], + *, + author: str = "user", + ) -> Optional[List[Dict[str, Any]]]: + """Set (or with ``emoji=None`` clear) *author*'s reaction on one message. + + iOS Tapback semantics: one reaction per author per message. Re-sending + the same emoji clears it, a different emoji replaces it. Returns the + message's full reaction list after the write, or ``None`` when the row + doesn't exist or isn't part of *session_id*. + """ + from hermes_state import _scrub_surrogates + if not session_id or message_row_id is None: + return None + + def _do(conn): + row = conn.execute( + "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", + (message_row_id, session_id), + ).fetchone() + if row is None: + return None + + meta = self._decode_display_metadata(row[0]) or {} + existing = meta.get(self.REACTIONS_METADATA_KEY) + reactions = [ + r + for r in (existing if isinstance(existing, list) else []) + if isinstance(r, dict) and r.get("author") != author + ] + previous = next( + ( + r + for r in (existing if isinstance(existing, list) else []) + if isinstance(r, dict) and r.get("author") == author + ), + None, + ) + # Tapping the live reaction again retracts it. + toggling_off = ( + emoji is not None and previous is not None and previous.get("emoji") == emoji + ) + if emoji and not toggling_off: + reactions.append( + {"emoji": _scrub_surrogates(emoji), "author": author, "at": time.time()} + ) + + if reactions: + meta[self.REACTIONS_METADATA_KEY] = reactions + else: + meta.pop(self.REACTIONS_METADATA_KEY, None) + + conn.execute( + "UPDATE messages SET display_metadata = ? WHERE id = ?", + (self._encode_display_metadata(meta) if meta else None, message_row_id), + ) + return reactions + + return self._execute_write(_do) + + def get_message_reactions( + self, session_id: str, message_row_id: int + ) -> List[Dict[str, Any]]: + """Return the reaction list persisted on one message row (never ``None``).""" + if not session_id or message_row_id is None: + return [] + + row = self._read_one( + "SELECT display_metadata FROM messages WHERE id = ? AND session_id = ?", + (message_row_id, session_id), + ) + + if row is None: + return [] + + meta = self._decode_display_metadata(row[0]) or {} + reactions = meta.get(self.REACTIONS_METADATA_KEY) + + return [r for r in reactions if isinstance(r, dict)] if isinstance(reactions, list) else [] + + def take_unseen_reactions( + self, session_id: str, *, author: str = "user" + ) -> List[Dict[str, Any]]: + """Return *author*'s not-yet-surfaced reactions and mark them seen. + + Powers the cache-safe model-context path: reactions are announced on the + NEXT user turn (never by rewriting the message that was reacted to), and + the ``seen`` stamp guarantees each one is announced exactly once. + """ + if not session_id: + return [] + + def _do(conn): + rows = conn.execute( + "SELECT id, role, content, display_metadata FROM messages " + "WHERE session_id = ? AND active = 1 AND display_metadata IS NOT NULL " + "ORDER BY id", + (session_id,), + ).fetchall() + + pending = [] + for row in rows: + meta = self._decode_display_metadata(row["display_metadata"]) + if not meta: + continue + reactions = meta.get(self.REACTIONS_METADATA_KEY) + if not isinstance(reactions, list): + continue + + changed = False + for reaction in reactions: + if ( + not isinstance(reaction, dict) + or reaction.get("author") != author + or reaction.get("seen") + ): + continue + reaction["seen"] = True + changed = True + content = self._decode_content(row["content"]) + pending.append( + { + "row_id": row["id"], + "role": row["role"], + "emoji": reaction.get("emoji") or "", + "text": content if isinstance(content, str) else "", + } + ) + + if changed: + conn.execute( + "UPDATE messages SET display_metadata = ? WHERE id = ?", + (self._encode_display_metadata(meta), row["id"]), + ) + + return pending + + return self._execute_write(_do) or [] + + def latest_message_row_id( + self, session_id: str, *, role: str = "user", offset: int = 0, require_text: bool = True + ) -> Optional[int]: + """Row id of the most recent active message with *role*, or ``None``. + + Two callers, same need — "the message I mean, without an id": the agent + defaulting to the turn that triggered it, and the desktop reacting to a + live message that hasn't round-tripped through a resume yet. + ``offset`` steps to earlier turns (1 = the one before the latest) so a + reaction can land retroactively — "two messages ago" is how the caller + thinks about it. + + ``require_text`` (default) skips rows with no plain-text content — + tool-call-only assistant turns and attachment stubs don't render as + bubbles, so "the latest message" as a HUMAN means it must never + resolve to one (a reaction landing on an invisible row looks dropped, + and its annotation quotes an empty string). + """ + if not session_id or role not in {"user", "assistant"} or offset < 0: + return None + + text_filter = ( + "AND content IS NOT NULL AND TRIM(content) != '' " if require_text else "" + ) + + row = self._read_one( + "SELECT id FROM messages WHERE session_id = ? AND role = ? " + f"AND active = 1 {text_filter}ORDER BY id DESC LIMIT 1 OFFSET ?", + (session_id, role, int(offset)), + ) + + return row[0] if row else None + + def latest_user_message_row_id(self, session_id: str) -> Optional[int]: + """Row id of the most recent active user message, or ``None``. + + The agent's default reaction target: "the message that triggered me", + so the model never has to thread row ids through a tool call (mirrors + the photon adapter's ``_record_last_inbound``). + """ + return self.latest_message_row_id(session_id, role="user") + + def get_message_role(self, session_id: str, row_id: int) -> Optional[str]: + """Role of the active message at *row_id* in *session_id*, or ``None``. + + Lets a reaction event carry the target's role so a renderer can match + a live message that doesn't know its durable row id yet. + """ + if not session_id: + return None + + row = self._read_one( + "SELECT role FROM messages WHERE id = ? AND session_id = ? AND active = 1", + (int(row_id), session_id), + ) + + return row[0] if row else None + + def _insert_message_rows(self, conn, session_id: str, messages: List[Dict[str, Any]]) -> tuple[int, int]: + """Insert *messages* as fresh active rows for *session_id*. + + Shared by :meth:`replace_messages` (delete-then-insert) and + :meth:`archive_and_compact` (soft-archive-then-insert). Runs inside the + caller's write transaction (takes the live ``conn``). Returns + ``(inserted_count, tool_call_count)``. Does NOT touch sessions.* counters + — the caller owns that, since the two flows reconcile counts differently. + """ + from hermes_state import _scrub_surrogates + now_ts = time.time() + inserted = 0 + tool_calls_total = 0 + for msg in messages: + role = msg.get("role", "unknown") + tool_calls = msg.get("tool_calls") + message_timestamp = now_ts + if msg.get("timestamp") is not None: + try: + ts_value = msg.get("timestamp") + if hasattr(ts_value, "timestamp"): + message_timestamp = float(ts_value.timestamp()) + else: + message_timestamp = float(ts_value) + except (TypeError, ValueError): + logger.debug("Ignoring invalid explicit message timestamp: %r", msg.get("timestamp")) + reasoning_details = msg.get("reasoning_details") if role == "assistant" else None + codex_reasoning_items = ( + msg.get("codex_reasoning_items") if role == "assistant" else None + ) + codex_message_items = ( + msg.get("codex_message_items") if role == "assistant" else None + ) + reasoning_details_json = self._reasoning_json_text(reasoning_details) + codex_items_json = self._reasoning_json_text(codex_reasoning_items) + codex_message_items_json = self._reasoning_json_text(codex_message_items) + # tool_calls may arrive as a Python list (from the live agent) + # or as a JSON string (from import_sessions / export_session, + # which store it as TEXT). json.dumps on an already-serialized + # string double-encodes it, so parse first. + if isinstance(tool_calls, str): + try: + tool_calls = json.loads(tool_calls) + except (json.JSONDecodeError, TypeError): + tool_calls = [] + tool_calls_json = json.dumps(tool_calls) if tool_calls else None + # Accept either `platform_message_id` (new explicit name) or + # `message_id` (yuanbao's existing convention on message dicts). + platform_msg_id = ( + msg.get("platform_message_id") or msg.get("message_id") + ) + + api_content = msg.get("api_content") + + cur = conn.execute( + """INSERT INTO messages (session_id, role, content, tool_call_id, + tool_calls, tool_name, effect_disposition, timestamp, token_count, finish_reason, + reasoning, reasoning_content, reasoning_details, codex_reasoning_items, + codex_message_items, platform_message_id, observed, _compressed_summary, active, api_content, display_kind, display_metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + session_id, + role, + self._encode_content(msg.get("content")), + msg.get("tool_call_id"), + tool_calls_json, + _scrub_surrogates(msg.get("tool_name")), + msg.get("effect_disposition"), + message_timestamp, + msg.get("token_count"), + msg.get("finish_reason"), + _scrub_surrogates(msg.get("reasoning")) if role == "assistant" else None, + _scrub_surrogates(msg.get("reasoning_content")) if role == "assistant" else None, + reasoning_details_json, + codex_items_json, + codex_message_items_json, + platform_msg_id, + 1 if msg.get("observed") else 0, + 1 if msg.get("_compressed_summary") else 0, + 1, + _scrub_surrogates(api_content) if isinstance(api_content, str) else None, + _scrub_surrogates(msg.get("display_kind")) if isinstance(msg.get("display_kind"), str) else None, + self._encode_display_metadata(msg.get("display_metadata")), + ), + ) + if isinstance(msg, dict) and cur.lastrowid is not None: + msg["_row_id"] = cur.lastrowid + inserted += 1 + if tool_calls is not None: + tool_calls_total += ( + len(tool_calls) if isinstance(tool_calls, list) else 1 + ) + now_ts = max(now_ts + 1e-6, message_timestamp + 1e-6) + return inserted, tool_calls_total + + def replace_messages( + self, + session_id: str, + messages: List[Dict[str, Any]], + active_only: bool = False, + archive_dropped: bool = False, + reject_active_turn_lease: bool = False, + ) -> None: + """Atomically replace the stored messages for a session. + + Used by transcript-rewrite flows such as /retry, /undo, and /compress. + The delete + reinsert sequence must commit as one transaction so a + mid-rewrite failure does not leave SQLite with a partial transcript. + + DESTRUCTIVE by default: every row for the session is DELETEd (and drops + out of the FTS index). For compaction that must preserve the + pre-compaction transcript under the same id, use + :meth:`archive_and_compact` instead. + + Pass ``active_only=True`` to replace ONLY the live (``active = 1``) rows, + leaving soft-archived rows (``active = 0`` — e.g. the ``compacted = 1`` + turns that :meth:`archive_and_compact` keeps on disk for #38763 + durability, or rewind/undo rows) untouched. Callers that share a session + id with an agent already running in-place compaction must use this so a + full-history rewrite doesn't wipe the rows the agent deliberately + archived. ``message_count``/``tool_call_count`` then track the live set, + matching :meth:`archive_and_compact`. + + Pass ``archive_dropped=True`` to SOFT-archive the live rows instead of + DELETEing them: the replaced turns stay on disk with ``active = 0``, + ``compacted = 0`` — the same "the user took it back" marking + :meth:`rewind_to_message` applies — and stay readable via + :meth:`get_messages` with ``include_inactive=True``. This is the mode a + rewind/edit/regenerate must use: those flows overwrite a transcript the + user may not have meant to drop, and a plain DELETE also evicts the rows + from the FTS index, leaving nothing to recover from (#82756). It implies + active-only handling — already-archived rows are never touched — so + ``active_only`` is redundant with it. The rewritten set is inserted as + fresh active rows exactly as in the destructive path, so the live view + is identical either way; only the durability of the dropped turns + differs. + + Pass ``reject_active_turn_lease=True`` for user-initiated rewrites that + do not already own the cross-process turn lease. The lease check and + transcript mutation then share one write transaction, so a second + process cannot archive or replace a turn that is still being produced. + """ + from hermes_state import CompressionSessionClosedError + + active_clause = " AND active = 1" if active_only else "" + + def _do(conn): + if reject_active_turn_lease: + self._check_transcript_write_guards( + conn, + session_id, + None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + else: + session = conn.execute( + "SELECT ended_at, end_reason FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if ( + session is not None + and session["ended_at"] is not None + and session["end_reason"] == "compression" + ): + raise CompressionSessionClosedError(session_id) + if archive_dropped: + # Content-preserving UPDATE: the rows keep their FTS entries + # (the messages_fts triggers fire on INSERT / DELETE / UPDATE + # of content columns, not on `active`), so the replaced turns + # stay readable via get_messages(include_inactive=True) and + # searchable with include_inactive=True after the rewrite. + conn.execute( + "UPDATE messages SET active = 0 " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + else: + conn.execute( + f"DELETE FROM messages WHERE session_id = ?{active_clause}", + (session_id,), + ) + conn.execute( + "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", + (session_id,), + ) + total_messages, total_tool_calls = self._insert_message_rows( + conn, session_id, messages + ) + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (total_messages, total_tool_calls, session_id), + ) + + self._execute_write(_do) + + def has_archived_messages(self, session_id: str) -> bool: + """Return True if the session has any soft-archived (``active = 0``) rows. + + Cheap existence probe — does not load rows. NOTE: production rewrite + paths no longer branch on this (they pass ``active_only=True`` + unconditionally — a probe can fail open or race a concurrent + ``archive_and_compact``, #80216); kept for tests and diagnostics. + """ + return self._read_one( + "SELECT 1 FROM messages WHERE session_id = ? AND active = 0 LIMIT 1", + (session_id,), + ) is not None + + def get_active_message_watermark(self, session_id: str) -> int: + """MAX(id) of the session's active rows — the compression watermark. + + Captured at compression START (before the slow provider summary call). + Every active row with ``id > watermark`` at commit time arrived + concurrently and must survive the compaction verbatim. Returns 0 for + an empty/unknown session. + """ + if not session_id: + return 0 + row = self._read_one( + "SELECT COALESCE(MAX(id), 0) FROM messages " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + return int(row[0]) if row else 0 + + def archive_and_compact( + self, + session_id: str, + compacted_messages: List[Dict[str, Any]], + model_config_patch: Optional[Dict[str, Any]] = None, + watermark: Optional[int] = None, + lock_holder: Optional[str] = None, + tail_count: int = 0, + ) -> int: + """Non-destructive in-place compaction for a single durable session id. + + Soft-archives the active messages (``active = 0``) and inserts + *compacted_messages* as fresh active rows — atomically, in one write + transaction. The conversation keeps ONE session id for life (#38763) + WITHOUT destroying history: + + - The live-context load (:meth:`get_messages_as_conversation`, + :meth:`get_messages`) filters ``active = 1`` by default, so the model + reloads ONLY the compacted set. + - The archived pre-compaction turns stay on disk (active=0) and stay + DISCOVERABLE: they are marked compacted=1, and search_messages() + includes compacted=1 rows by default — so session_search still finds + them, unlike rewind/undo rows (active=0, compacted=0) which stay + hidden. They remain in the FTS index (the messages_fts* triggers + index on INSERT / drop on DELETE and don't key on active/compacted; + flipping to active=0 is a content-preserving UPDATE) and are + recoverable via get_messages(..., include_inactive=True). + + Concurrent-append safety (#75316): when *watermark* is provided (the + value of :meth:`get_active_message_watermark` captured at compression + START), rows that arrived during the slow provider summary call + (``id > watermark``) are NOT summarized away. They are re-sequenced + after the compacted set by a pure-SQL column clone (every column + except ``id`` — content, api_content, platform_message_id, token + counts, reasoning sidecars all survive byte-exact, and the FTS + triggers index the clones naturally), and the originals are archived. + NOTE: re-sequencing assigns the tail rows fresh ids; consumers that + reference durable row ids re-resolve by content (see 3e8ab0610). + ``watermark=None`` preserves the historical archive-everything + behavior. + + Commit-fence safety: when *lock_holder* is provided, the commit + verifies INSIDE the transaction that the compression lock is still + held by that holder and unexpired — a compression whose lease was + reclaimed (crash cleanup, TTL expiry, competing writer) fails the + commit instead of clobbering the winner's transcript. + + *tail_count* (default 0) names how many of the LAST rows of + *compacted_messages* are the verbatim carried-forward tail the + compressor protected rather than summarized (#86366). Those rows' + ORIGINALS — which this call archives as a side effect of the blanket + soft-archive — are superseded byte-identical duplicates, not + "summarized away" content, so they are stamped rewind-style + (``active=0, compacted=0``, hidden from search_messages) instead of + ``compacted=1``. Without this the tail originals satisfy the recall + filter alongside their live clones and session_search returns every + carried-forward message once per compaction. Callers that cannot know + their tail shape keep the historical archive-everything behavior. + + ``message_count`` is set to the ACTIVE count after commit, matching + what the live load returns. ``model_config_patch`` is merged into the + session's JSON config in the same transaction; a ``None`` value + removes that key. Returns the new active count. + """ + from hermes_state import SessionCompressionInProgressError + + def _do(conn): + if lock_holder is not None: + lock_row = conn.execute( + "SELECT holder, expires_at FROM compression_locks " + "WHERE session_id = ?", + (session_id,), + ).fetchone() + if ( + lock_row is None + or lock_row["holder"] != lock_holder + or float(lock_row["expires_at"]) <= time.time() + ): + raise SessionCompressionInProgressError( + f"Compression lease for {session_id!r} lost before " + "commit; refusing to publish a stale compaction" + ) + + patched_model_config = None + if model_config_patch is not None: + # on_missing="raise": a prune/compaction must not commit + # against a vanished session row (the compressor's caller + # converts the raised error into a safe keep-the-original + # no-op), unlike the flag setters which tolerate missing rows. + patched_model_config = self._merge_model_config_json( + conn, session_id, model_config_patch, on_missing="raise" + ) + + # Concurrent tail: active rows that arrived after the watermark. + # Snapshot their ids and tool_calls now — the clone below needs a + # stable id list, and the tool-call count keeps sessions.* honest. + tail_ids: list[int] = [] + tail_tool_calls = 0 + if watermark is not None: + for row in conn.execute( + "SELECT id, tool_calls FROM messages " + "WHERE session_id = ? AND active = 1 AND id > ? " + "ORDER BY id", + (session_id, int(watermark)), + ).fetchall(): + tail_ids.append(int(row["id"])) + raw = row["tool_calls"] + if raw: + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + tail_tool_calls += len(parsed) if isinstance(parsed, list) else 0 + except (TypeError, ValueError): + pass + + # Soft-archive the live turns: active=0 hides them from the live + # context load, compacted=1 marks them as "summarized away" (vs + # rewind/undo's active=0+compacted=0, which means "user took it + # back"). search_messages includes compacted=1 rows by default so + # the pre-compaction transcript stays discoverable; live-context + # loads (active=1 only) still exclude them. Tail originals whose + # verbatim clones ride inside *compacted_messages* (tail_count) + # are superseded duplicates instead (#86366): they get the + # rewind-style flags so they stop matching the recall filter. + # Rewind-target ids: the originals of the carried-forward tail + # rows (tail_count), captured BEFORE any flag flips. Named apart + # from the watermark `tail_ids` below on purpose — the two are + # different sets (#86366): rewind targets sit AT/BELOW the + # watermark (the compressor only saw rows up to it), while + # `tail_ids` are concurrent appends ABOVE it. Without the bound, + # a concurrent append would steal a LIMIT slot and leave a real + # carried-forward original stamped compacted=1. + rewind_tail_ids: Optional[list[int]] = None + if tail_count > 0: + if watermark is not None: + tail_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 AND id <= ? " + "ORDER BY id DESC LIMIT ?", + (session_id, int(watermark), int(tail_count)), + ).fetchall() + else: + tail_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id DESC LIMIT ?", + (session_id, int(tail_count)), + ).fetchall() + rewind_tail_ids = [int(row["id"]) for row in tail_rows] + + # The watermark clone below re-inserts `tail_ids` rows byte-exact + # as live rows — their originals are the SAME superseded-duplicate + # class as the carried-forward tail (#86366), so they take the + # rewind flags too instead of double-matching the recall filter. + rewind_ids = [*(rewind_tail_ids or []), *tail_ids] + + if rewind_ids: + placeholders = ",".join("?" for _ in rewind_ids) + conn.execute( + "UPDATE messages SET active = 0, compacted = 0 " + f"WHERE session_id = ? AND id IN ({placeholders})", + [session_id, *rewind_ids], + ) + conn.execute( + "UPDATE messages SET active = 0, compacted = 1 " + "WHERE session_id = ? AND active = 1 " + f"AND id NOT IN ({placeholders})", + [session_id, *rewind_ids], + ) + else: + conn.execute( + "UPDATE messages SET active = 0, compacted = 1 " + "WHERE session_id = ? AND active = 1", + (session_id,), + ) + inserted, tool_calls_total = self._insert_message_rows( + conn, session_id, compacted_messages + ) + + if tail_ids: + # Re-sequence the concurrent tail after the compacted set via + # a pure-SQL column clone: no decode/re-encode round trip, no + # field drift — new id, active=1, compacted=0, all else exact. + placeholders = ",".join("?" for _ in tail_ids) + clone_cols = [ + c for c in self._message_column_names(conn) + if c not in ("id", "active", "compacted") + ] + col_list = ", ".join(clone_cols) + conn.execute( + f"INSERT INTO messages ({col_list}, active, compacted) " + f"SELECT {col_list}, 1, 0 FROM messages " + f"WHERE id IN ({placeholders}) ORDER BY id", + tail_ids, + ) + inserted += len(tail_ids) + tool_calls_total += tail_tool_calls + + # message_count / tool_call_count reflect the LIVE (active) set — + # the archived rows are still on disk but not part of the live count. + if model_config_patch is None: + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", + (inserted, tool_calls_total, session_id), + ) + else: + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ?, " + "model_config = ? WHERE id = ?", + (inserted, tool_calls_total, patched_model_config, session_id), + ) + return inserted + + return self._execute_write(_do) + + def _message_column_names(self, conn) -> List[str]: + """Column names of the messages table, cached per-connection era.""" + cached = getattr(self, "_message_columns_cache", None) + if cached: + return cached + cols = [r[1] for r in conn.execute("PRAGMA table_info(messages)").fetchall()] + self._message_columns_cache = cols + return cols + + def set_latest_user_api_content( + self, session_id: str, content: Any, api_content: str + ) -> int: + """Backfill the ``api_content`` sidecar onto the newest ACTIVE user row. + + In-place preflight compaction (:meth:`archive_and_compact`) inserts the + current turn's user row BEFORE the turn prologue composes the + prefetch/plugin sidecar, and the subsequent crash persist identity-skips + every compacted dict — without this backfill the stamped sidecar would + never land in the DB and any reload would replay clean content, + re-introducing the prompt-cache divergence the sidecar exists to close. + + The ``content`` match is a defensive guard: if the newest active user + row is not the message the caller stamped (racing rewrite, unexpected + tail shape), nothing is written. Returns the number of rows updated + (0 or 1). + """ + from hermes_state import _scrub_surrogates + encoded = self._encode_content(content) + + return self._write_rowcount( + "UPDATE messages SET api_content = ? WHERE id = (" + "SELECT id FROM messages " + "WHERE session_id = ? AND role = 'user' AND active = 1 " + "ORDER BY id DESC LIMIT 1" + ") AND content IS ?", + (_scrub_surrogates(api_content), session_id, encoded), + ) + + def _dedupe_display_generations(self, rows): + """Collapse compaction generations so each message appears once. + + Compaction epochs copy the protected tail into each new generation, so + one logical message can exist as several rows (identical + role/content/timestamp) with different ``active`` flags and ids. A + display read must surface each exactly once: prefer the live row, then + the newest generation. + + This is the ONE definition shared by every display projection — + :meth:`get_messages` (REST), :meth:`get_resume_conversations` and + :meth:`get_ancestor_display_prefix` (gateway resume), and + :meth:`get_messages_as_conversation` (warm-session payload) — so the + surfaces cannot disagree about the same transcript. *rows* must already + be ordered by ``id``; the returned list keeps that order. + """ + seen: Dict[Tuple[Any, ...], Any] = {} + for row in rows: + dedupe_content = row["content"] + if row["role"] == "user": + from agent.context_compressor import split_user_originated_turn + + candidate = { + "role": "user", + "content": self._decode_content(row["content"]), + "display_kind": row["display_kind"], + "display_metadata": self._decode_display_metadata( + row["display_metadata"] + ), + } + handoff, live_view = split_user_originated_turn(candidate) + if handoff is not None and live_view is not None: + dedupe_content = self._encode_content(live_view.get("content")) + # Tool fields participate in the dedupe key: compaction copies them + # verbatim, so identical tool messages across generations still + # collapse, while distinct tool calls that happen to share + # role/content/timestamp are never merged. + key = ( + row["role"], + dedupe_content, + row["timestamp"], + row["tool_call_id"], + row["tool_calls"], + row["tool_name"], + ) + cur = seen.get(key) + if cur is None or (row["active"], row["id"]) > (cur["active"], cur["id"]): + seen[key] = row + return sorted(seen.values(), key=lambda r: r["id"]) + + def get_messages( + self, + session_id: str, + include_inactive: bool = False, + include_compacted: bool = False, + limit: Optional[int] = None, + offset: int = 0, + latest: bool = False, + after_id: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """Load messages for a session in insertion order. + + By default only active messages are returned. Pass + ``include_inactive=True`` to load soft-deleted rows (e.g. for + audit / debug views of rewound history). See + :meth:`rewind_to_message` for the soft-delete mechanic. + + Pass ``include_compacted=True`` to additionally load rows preserved + by in-place context compaction (``active=0, compacted=1``). Those are + durable display history, not soft-deleted rows — a user-visible + transcript read must not drop them, or earlier turns silently become + unreachable once the UI exhausts its active-only window. Soft-deleted + Undo/Rewind rows (``active=0, compacted=0``) stay excluded; use + ``include_inactive`` for those. + + Ordered by AUTOINCREMENT id (true insertion order) rather than + timestamp — see c03acca50 for the WSL2 clock-regression rationale. + + When ``limit`` is provided, returns at most ``limit`` messages + starting from ``offset`` (0-based, in insertion order). Enables + pagination for the API endpoint to avoid loading entire transcripts. + With ``latest=True``, the offset is measured back from the newest + message and the selected page is still returned in chronological + order. ``offset`` alone (without ``limit``) also pages — SQLite + requires a LIMIT clause for OFFSET, so it's emitted as ``LIMIT -1`` + (unbounded). + + ``after_id`` enables keyset pagination (``id > after_id``): O(1) + page seeks on huge transcripts where OFFSET degrades to O(n) per + page. Ascending order only (incompatible with ``latest``/``offset``). + """ + if after_id is not None and (latest or offset): + raise ValueError("after_id is incompatible with latest/offset paging") + if after_id is not None and include_compacted: + raise ValueError("after_id is incompatible with include_compacted (deduped display reads use offset paging)") + if include_inactive: + # Audit / debug reads: every row, including soft-deleted. + active_clause = "" + elif include_compacted: + # Display history: active rows plus rows preserved by in-place + # compaction (active=0, compacted=1), but never soft-deleted + # Undo/Rewind rows (active=0, compacted=0). + active_clause = " AND (active = 1 OR compacted = 1)" + else: + active_clause = " AND active = 1" + keyset_clause = " AND id > ?" if after_id is not None else "" + sql = ( + "SELECT * FROM messages WHERE session_id = ?" + f"{active_clause}{keyset_clause} ORDER BY id {'DESC' if latest else 'ASC'}" + ) + params: list = [session_id] + if after_id is not None: + params.append(after_id) + if include_compacted: + # Read the full display set (a session's rows are bounded; the + # UI-level 500-row cap lives in the endpoint, not here), dedupe + # generations, then apply paging. + all_rows = self._read_all( + "SELECT * FROM messages WHERE session_id = ?" + active_clause + + " ORDER BY id ASC", + [session_id], + ) + rows = self._dedupe_display_generations(all_rows) + if latest: + rows = rows[::-1] + rows = rows[offset:] + if limit is not None: + rows = rows[:limit] + if latest: + rows = rows[::-1] + else: + if limit is not None or offset: + # SQLite's OFFSET requires LIMIT; -1 means "no limit". + sql += " LIMIT ? OFFSET ?" + params.extend([-1 if limit is None else limit, offset]) + rows = self._read_all(sql, params) + if latest: + rows.reverse() + result = [] + for row in rows: + msg = dict(row) + if msg.pop("_compressed_summary", 0): + msg["_compressed_summary"] = True + if "content" in msg: + msg["content"] = self._decode_content(msg["content"]) + if msg.get("tool_calls"): + try: + msg["tool_calls"] = json.loads(msg["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize tool_calls in get_messages, falling back to []") + msg["tool_calls"] = [] + if msg.get("display_metadata") is not None: + msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) + result.append(msg) + return result + + def find_pr_url_messages(self, session_ids: List[str]) -> List[Dict[str, Any]]: + """Tool results in these sessions that mention a GitHub PR url. + + A candidate scan, deliberately loose: it hands back every tool result + containing ``/pull/`` and leaves the caller to decide which ones make a + claim (see the desktop's PR recovery, which only accepts an output that + is a bare PR url — the signature of ``gh pr create``). Ordered + oldest-first per session so the caller can take the last match. + """ + found: List[Dict[str, Any]] = [] + ids = [s for s in session_ids if s] + for start in range(0, len(ids), 900): # SQLite's bound-variable ceiling. + chunk = ids[start : start + 900] + placeholders = ",".join("?" * len(chunk)) + rows = self._read_all( + f"""SELECT session_id, content FROM messages + WHERE session_id IN ({placeholders}) + AND role = 'tool' AND content LIKE '%/pull/%' + ORDER BY id ASC""", + chunk, + ) + found.extend({"session_id": row[0], "content": row[1]} for row in rows) + return found + + def get_messages_around( + self, + session_id: str, + around_message_id: int, + window: int = 5, + ) -> Dict[str, Any]: + """Load a window of messages anchored on a specific message id. + + Returns a dict with: + - ``window``: up to ``window`` messages before the anchor, the anchor + itself, and up to ``window`` messages after, ordered by id ascending. + - ``messages_before``: count of messages strictly before the anchor + still in the session (== window unless we hit the start). + - ``messages_after``: count of messages strictly after the anchor + still in the session (== window unless we hit the end). + + Used by ``session_search`` for both the discovery shape (anchored on the + FTS5 match) and the scroll shape (anchored on any message id). The + ``messages_before`` / ``messages_after`` counts let the caller detect + session boundaries: when either is less than ``window``, the agent has + reached one end of the session. + + Returns an empty window when ``around_message_id`` is not a real id in + ``session_id`` — callers decide how to surface that. + """ + if window < 0: + window = 0 + with self._read_ctx() as conn: + # Confirm the anchor exists in this session. + anchor_exists = conn.execute( + "SELECT 1 FROM messages WHERE id = ? AND session_id = ? LIMIT 1", + (around_message_id, session_id), + ).fetchone() + if not anchor_exists: + return {"window": [], "messages_before": 0, "messages_after": 0} + + # Two queries: anchor + before (DESC, take window+1), and after + # (ASC, take window). Final order is id ASC. + before_rows = conn.execute( + "SELECT * FROM messages " + "WHERE session_id = ? AND id <= ? " + "ORDER BY id DESC LIMIT ?", + (session_id, around_message_id, window + 1), + ).fetchall() + after_rows = conn.execute( + "SELECT * FROM messages " + "WHERE session_id = ? AND id > ? " + "ORDER BY id ASC LIMIT ?", + (session_id, around_message_id, window), + ).fetchall() + + # before_rows is DESC; reverse so it's ASC, then concatenate after_rows. + rows = list(reversed(before_rows)) + list(after_rows) + result = [] + for row in rows: + msg = dict(row) + if "content" in msg: + msg["content"] = self._decode_content(msg["content"]) + if msg.get("tool_calls"): + try: + msg["tool_calls"] = json.loads(msg["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Failed to deserialize tool_calls in get_messages_around, falling back to []" + ) + msg["tool_calls"] = [] + if msg.get("display_metadata") is not None: + msg["display_metadata"] = self._decode_display_metadata(msg["display_metadata"]) + result.append(msg) + + # before_rows includes the anchor itself; subtract 1 for the count of + # messages strictly before the anchor in the returned slice. + messages_before = max(0, len(before_rows) - 1) + messages_after = len(after_rows) + return { + "window": result, + "messages_before": messages_before, + "messages_after": messages_after, + } + + def resolve_resume_session_id(self, session_id: str) -> str: + """Redirect a resume target to the descendant session that holds the messages. + + Context compression ends the current session and forks a new child session + (linked via ``parent_session_id``). The flush cursor is reset, so the + child is where new messages actually land — the parent ends up with + ``message_count = 0`` rows unless messages had already been flushed to + it before compression. See #15000. + + This helper walks ``parent_session_id`` forward from ``session_id`` and + returns the descendant in the chain that has the **most recent** messages. + Unlike the original logic, it does NOT short-circuit when the starting + session already has messages — a descendant that was created by + compression may hold the continuation content and should be preferred + by the WebUI and gateway for ``--resume`` and session loading. + + If no descendant (including the starting session) has any messages, + the original ``session_id`` is returned unchanged. + + The chain is always walked via the child whose ``started_at`` is + latest; that matches the single-chain shape that compression creates. + A depth cap (32) guards against accidental loops in malformed data. + """ + if not session_id: + return session_id + + # Follow the compression-continuation chain forward to the live tip + # FIRST. Auto-compression ends the current session and forks a + # continuation child, but a long-lived parent keeps its own flushed + # message rows — so the empty-head walk below never redirects it, and + # resuming the parent id reloads the pre-compression transcript while + # the turns generated *after* compression (and their responses) sit in + # the continuation. ``get_compression_tip`` is lineage-aware: it only + # follows children whose parent ended with ``end_reason='compression'`` + # (created after the parent was ended), so delegation / branch children + # never hijack the resume. This is the fix for the desktop "I came back + # and the reply isn't there" report on large sessions. + try: + tip = self.get_compression_tip(session_id) + except Exception: + tip = session_id + if tip and tip != session_id: + session_id = tip + + with self._read_ctx() as conn: + current = session_id + seen = {current} + best = None # tracks the last (deepest) node with messages + + for _ in range(32): + # Check if the current node has messages. + try: + row = conn.execute( + "SELECT 1 FROM messages WHERE session_id = ? LIMIT 1", + (current,), + ).fetchone() + except Exception: + return session_id + if row is not None: + best = current + + # Walk to the most-recently-started child — but skip explicit + # branch (`_branched_from`), delegate/subagent (`_delegate_from`), + # reset-continuation (`_reset_from` or the legacy same-key + # heuristic — a post-reset conversation must never be reached + # by resuming the parent the user reset away), and tool + # children. They also carry a ``parent_session_id`` yet + # are NOT compression continuations; following them would hijack + # the resume target to an unrelated session (e.g. a subagent + # run). This mirrors the child-exclusion in ``get_compression_tip``. + try: + child_row = conn.execute( + "SELECT id FROM sessions AS child " + "WHERE child.parent_session_id = ? " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._branched_from') IS NULL " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._delegate_from') IS NULL " + " AND json_extract(COALESCE(child.model_config, '{}'), '$._reset_from') IS NULL " + f" AND NOT {_legacy_reset_child_sql('child', _RESET_END_REASONS_SQL)} " + " AND COALESCE(child.source, '') != 'tool' " + "ORDER BY child.started_at DESC, child.id DESC LIMIT 1", + (current,), + ).fetchone() + except Exception: + return session_id + if child_row is None: + break + child_id = child_row["id"] if hasattr(child_row, "keys") else child_row[0] + if not child_id or child_id in seen: + break + seen.add(child_id) + current = child_id + + return best if best is not None else session_id + + def get_messages_as_conversation( + self, + session_id: str, + include_ancestors: bool = False, + include_inactive: bool = False, + repair_alternation: bool = False, + include_row_ids: bool = False, + include_compacted: bool = False, + ) -> List[Dict[str, Any]]: + """ + Load messages in the OpenAI conversation format (role + content dicts). + Used by the gateway to restore conversation history. + + By default only active messages are returned. Pass + ``include_inactive=True`` to load soft-deleted (rewound) rows + as well. See :meth:`rewind_to_message`. + + ``include_compacted=True`` additionally loads rows preserved by + in-place compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. DISPLAY reads want this; the + model-fed restore must NOT pass it, or a resumed session regrows the + very history compaction just summarized away. + + ``repair_alternation=True`` runs ``repair_message_sequence`` over the + loaded list before returning it. Callers that restore a session for + LIVE REPLAY should pass it: a durable alternation violation (e.g. a + ``user;user`` pair left by a turn that persisted no assistant row) + otherwise re-triggers the pre-request defensive repair on every + single request for the rest of the session's life — the repair + mutates only the per-request list, never the stored transcript. + Inspection/export consumers keep the default and see the transcript + verbatim. + """ + session_ids = [session_id] + if include_ancestors and not self._is_explicit_branch_session(session_id): + session_ids = self._session_lineage_root_to_tip(session_id) + + if include_inactive: + active_clause = "" + elif include_compacted: + active_clause = " AND (active = 1 OR compacted = 1)" + else: + active_clause = " AND active = 1" + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders})" + # Order by AUTOINCREMENT id (true insertion order), NOT timestamp: + # append_message stamps rows with time.time(), which is not + # monotonic (WSL2, NTP steps, VM/laptop sleep resume). A later + # row can carry an earlier timestamp than its predecessor, and + # ORDER BY timestamp would then sort an assistant tool_calls row + # after its tool response, breaking tool-call/response adjacency + # and triggering an HTTP 400 on replay. This matches get_messages + # — see c03acca50 for the original fix. + f"{active_clause} ORDER BY id", + tuple(session_ids), + ).fetchall() + + if include_compacted: + rows = self._dedupe_display_generations(rows) + + return self._rows_to_conversation( + rows, + session_id=session_id, + include_ancestors=include_ancestors, + repair_alternation=repair_alternation, + include_row_ids=include_row_ids, + ) + + def _rows_to_conversation( + self, + rows, + *, + session_id: str, + include_ancestors: bool, + repair_alternation: bool, + include_row_ids: bool = False, + include_summary_markers: bool = False, + ) -> List[Dict[str, Any]]: + """Decode fetched message rows into the OpenAI conversation format. + + Extracted from get_messages_as_conversation so get_resume_conversations + can build the model-fed and display views from one SELECT. ``rows`` must + already be ordered by ``id`` (insertion order) and filtered to the + desired session set / active state by the caller. + """ + from hermes_state import _strip_background_review_harness, _strip_stale_tool_call_markers + messages = [] + # Watermark rotation column-clones concurrent tail rows into the child + # after the new summary, so the copies need not be adjacent. Index the + # exact durable clone identity while decoding instead of rescanning the + # whole accumulated lineage for every user row. + exact_user_clones: Dict[Tuple[Any, str], Dict[str, Any]] = {} + for row in rows: + content = self._decode_content(row["content"]) + if row["role"] in {"user", "assistant"} and isinstance(content, str): + content = sanitize_context(content).strip() + msg = {"role": row["role"], "content": content} + # Born durable (#92231): this dict is materialized FROM a durable + # row, so stamp the persistence marker at the source instead of + # relying on every restore caller to thread the loaded list back + # through a flush as ``conversation_history=`` — any + # identity-losing handoff (compression's durable-snapshot + # adoption, incremental persists with no history arg) would + # otherwise re-append the ENTIRE transcript on flush. + # Underscore-prefixed like ``_row_id``: every transport strips it + # before the wire, and compression's assembly copies deliberately + # strip it so rotated child handoffs still flush (see + # _fresh_compaction_message_copy). + msg[_DB_PERSISTED_MARKER_KEY] = True + # Durable per-message identity for surfaces that need to address a + # specific row later (desktop reactions). OPT-IN: only the gateway + # asks for it — every other consumer (ACP restore, export, + # inspection) gets the transcript in its historical shape. + # Underscore-prefixed so every transport's convert_messages() + # strips it before the wire. + if include_row_ids and row["id"] is not None: + msg["_row_id"] = row["id"] + # api_content is the byte-fidelity sidecar: the exact string sent + # to the API when it differed from the clean content. Returned + # VERBATIM — no sanitize_context, no strip — because the replay + # path substitutes it for content to keep the provider prompt + # cache prefix byte-stable across turns. Cleaning it here would + # re-introduce the divergence it exists to remove. + if row["api_content"]: + msg["api_content"] = row["api_content"] + if row["display_kind"]: + msg["display_kind"] = row["display_kind"] + if row["display_metadata"]: + decoded = self._decode_display_metadata(row["display_metadata"]) + if decoded is not None: + msg["display_metadata"] = decoded + if include_summary_markers and row["_compressed_summary"]: + msg["_compressed_summary"] = True + if row["timestamp"]: + msg["timestamp"] = row["timestamp"] + if row["tool_call_id"]: + msg["tool_call_id"] = row["tool_call_id"] + if row["tool_name"]: + msg["tool_name"] = row["tool_name"] + if row["effect_disposition"]: + msg["effect_disposition"] = row["effect_disposition"] + if row["tool_calls"]: + try: + msg["tool_calls"] = json.loads(row["tool_calls"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize tool_calls in conversation replay, falling back to []") + msg["tool_calls"] = [] + # Surface the platform-side message id (e.g. yuanbao msg_id, + # telegram update_id) so platform-specific flows like recall + # can match by external identifier instead of having to fall + # back to content-match heuristics. Exposed as ``message_id`` + # for backward compatibility with the JSONL transcript shape. + if row["platform_message_id"]: + msg["message_id"] = row["platform_message_id"] + if row["observed"]: + msg["observed"] = True + # Restore reasoning fields on assistant messages so providers + # that replay reasoning (OpenRouter, OpenAI, Nous) receive + # coherent multi-turn reasoning context. + if row["role"] == "assistant": + if row["finish_reason"]: + msg["finish_reason"] = row["finish_reason"] + if row["reasoning"]: + msg["reasoning"] = row["reasoning"] + if row["reasoning_content"] is not None: + msg["reasoning_content"] = row["reasoning_content"] + if row["reasoning_details"]: + try: + msg["reasoning_details"] = json.loads(row["reasoning_details"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize reasoning_details, falling back to None") + msg["reasoning_details"] = None + if row["codex_reasoning_items"]: + try: + msg["codex_reasoning_items"] = json.loads(row["codex_reasoning_items"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize codex_reasoning_items, falling back to None") + msg["codex_reasoning_items"] = None + if row["codex_message_items"]: + try: + msg["codex_message_items"] = json.loads(row["codex_message_items"]) + except (json.JSONDecodeError, TypeError): + logger.warning("Failed to deserialize codex_message_items, falling back to None") + msg["codex_message_items"] = None + if include_ancestors: + canonical_content, _is_composite = ( + self._canonical_replayed_user_content(msg) + ) + exact_clone_key = self._exact_replayed_user_clone_key( + msg.get("timestamp"), canonical_content + ) + previous_exact = ( + exact_user_clones.get(exact_clone_key) + if exact_clone_key is not None + else None + ) + duplicate = None + if previous_exact is not None: + previous_index = next( + ( + index + for index, candidate in enumerate(messages) + if candidate is previous_exact + ), + None, + ) + if previous_index is not None: + duplicate = (previous_index, True) + if duplicate is None: + duplicate = self._find_duplicate_replayed_user_message( + messages, msg + ) + if duplicate is not None: + duplicate_index, prefer_current = duplicate + if prefer_current: + # A rotated compression child can carry the same live + # ask as the parent row plus the only surviving summary + # scaffold. Keep the child carrier (and its durable row + # id), not the simpler ancestor copy. + messages.pop(duplicate_index) + else: + continue + messages.append(msg) + if include_ancestors and exact_clone_key is not None: + exact_user_clones[exact_clone_key] = msg + # DEFENSE-IN-DEPTH against background-review session pollution: a forked + # skill/memory review that (in older builds, before the _persist_disabled + # fix) shared the parent's session_id wrote its harness turn into this + # real session. The harness is a user/system message instructing the + # agent to "Review the conversation above and update the skill library / + # save to memory" under a hard tool restriction; re-loading it as live + # history makes the agent adopt the curator role and refuse the user's + # actual task. Strip any such harness message AND the curator-mode + # assistant reply immediately following it, so a polluted session + # resumes clean even if stray rows exist. + messages = _strip_background_review_harness(messages) + # DEFENSE-IN-DEPTH against #78148: before that fix, a bare tool-call + # marker (e.g. "[memory]") could get cached as a fallback and + # persisted as if it were the model's real answer. Sessions written + # before the fix can still carry those rows — clear the stray + # content on load so replaying history doesn't re-teach the model + # to keep emitting the marker. No-op for unaffected sessions. + messages = _strip_stale_tool_call_markers(messages) + if repair_alternation and messages: + # Lazy import: hermes_state already depends on agent.* (see + # sanitize_context above), but keep this optional path from + # widening the import surface at module load. + from agent.agent_runtime_helpers import repair_message_sequence + + repaired = repair_message_sequence(None, messages) + if repaired: + logger.info( + "Repaired %d message-alternation violation(s) while " + "restoring session %s — durable transcript kept them, " + "see repair_message_sequence", + repaired, + session_id, + ) + return messages + + def get_resume_conversations( + self, session_id: str + ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Return ``(model_history, display_history)`` for a session resume in ONE SELECT. + + ``session.resume`` needs two projections of the same lineage: + + - ``model_history`` — the tip session's active rows, alternation-repaired + (the live-replay working conversation). Equivalent to + ``get_messages_as_conversation(session_id, repair_alternation=True)``. + - ``display_history`` — the full compression lineage (ancestors → tip), + verbatim, with replayed-user dedup. Explicit ``/branch`` sessions are + excluded from this lineage because their own rows already contain the + copied transcript; including the live parent's rows would let messages + written to the original after the fork leak into the branch. + + The display projection also includes rows preserved by IN-PLACE + compaction (``active=0, compacted=1``), deduped by + :meth:`_dedupe_display_generations`. Without them a compacted + conversation resumes showing only its summary plus the carried-forward + tail — the user's own turns read as deleted even though every row is + still on disk, and the REST transcript read (which has always included + them) disagreed with this one about the same session (#92080). + + The display fetch already reads a superset of the model fetch (the tip + rows are part of the lineage), so serving both from one lineage SELECT + halves the resume's DB work versus two separate calls, with byte-identical + output (see test_get_resume_conversations_matches_separate_reads). + """ + session_ids = self._resume_lineage_ids(session_id) + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Compaction-archived rows (active=0, compacted=1) are display + # history; Undo/Rewind rows (active=0, compacted=0) are not. + "AND (active = 1 OR compacted = 1) " + # ORDER BY id (insertion order) — see get_messages_as_conversation + # for why timestamp ordering is unsafe. + "ORDER BY id", + tuple(session_ids), + ).fetchall() + + # Tip rows are exactly the model-fed set (get_messages_as_conversation + # with session_ids=[session_id]); filtering the lineage fetch preserves + # their relative id order. The model projection stays active-only — it + # is the compressed working context and must not regrow the history + # compaction just summarized away. + tip_rows = [r for r in rows if r["session_id"] == session_id and r["active"]] + model_history = self._rows_to_conversation( + tip_rows, + session_id=session_id, + include_ancestors=False, + repair_alternation=True, + include_row_ids=True, + # Pre-compress checkpointing: the resumed model history must keep + # the summary marker so checkpoint providers can exclude derivative + # summaries after a process restart (marker survives restart). + include_summary_markers=True, + ) + display_history = self._rows_to_conversation( + self._dedupe_display_generations(rows), + session_id=session_id, + include_ancestors=True, + repair_alternation=False, + include_row_ids=True, + ) + return model_history, display_history + + def _resume_lineage_ids(self, session_id: str) -> List[str]: + """Session ids a full (display) resume materializes for *session_id*. + + Compression continuations need their ended ancestors' rows for the + display transcript; an explicit ``/branch`` copy already owns its + transcript, so its lineage is itself alone. This is the ONE definition + shared by the resume readers (``get_resume_conversations``, + ``get_ancestor_display_prefix``) and the resume guard + (``assert_resume_safe`` / ``get_resume_message_count``) — the guard must + count exactly the rows a resume would load, never a superset. + """ + if self._is_explicit_branch_session(session_id): + return [session_id] + return self._session_lineage_root_to_tip(session_id) + + def get_resume_message_count( + self, session_id: str, *, tip_only: bool = False + ) -> int: + """Count the rows that a resume would materialize. + + ``tip_only=True`` counts the tip segment's ACTIVE rows — the set a + model-history restore loads (``get_messages_as_conversation`` without + ancestors, or the deferred Desktop resume that pages the display + transcript over REST and never materializes the ancestor prefix in + memory). + + Otherwise this counts the full-lineage DISPLAY set — active rows plus + the compaction-archived rows ``get_resume_conversations`` now loads + for the transcript. Counting only active rows here would let a + heavily-compacted conversation pass a limit sized for a handful of + live rows and then materialize tens of thousands. + """ + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" + placeholders = ",".join("?" for _ in session_ids) + row = self._read_one( + f"SELECT COUNT(*) FROM messages " + f"WHERE session_id IN ({placeholders}) AND {active_clause}", + tuple(session_ids), + ) + return int(row[0] if row else 0) + + def assert_resume_safe( + self, + session_id: str, + max_messages: Optional[int] = None, + *, + tip_only: bool = False, + ) -> int: + """Return resume row count or reject a transcript too large to load. + + ``max_messages=None`` resolves the limit from config + (``sessions.max_resume_messages``); 0 disables the guard and returns + the (bounded) count without raising. + + ``tip_only=True`` bounds only the tip segment's ACTIVE rows, for + callers that never materialize the ancestor lineage or the + compaction archive in memory (tip-only model restore, deferred + Desktop resume whose display history is REST-paginated). A + heavily-compressed conversation — 85 compaction segments and ~29k + lineage rows behind a ~700-row tip — is exactly the shape compression + is supposed to produce; counting its whole lineage against a limit + sized for in-memory materialization rejected the healthiest sessions + (Desktop Bot Chat stuck on "Waking up…" with code 4130) while the + process would only ever have held the tip. + + The full (non-``tip_only``) bound counts the DISPLAY set — active plus + compaction-archived rows — because that is what + ``get_resume_conversations`` materializes for the transcript. + """ + from hermes_state import SessionResumeTooLargeError, resolved_max_resume_messages + if max_messages is None: + max_messages = resolved_max_resume_messages() + if max_messages < 0: + raise ValueError("max_messages must be non-negative") + if max_messages == 0: + # Guard disabled by config — skip counting entirely. Every live + # caller invokes this for its raise side effect and ignores the + # return value, and an unbounded lineage COUNT here would do the + # exact pathological work the disable exists to avoid. + return 0 + session_ids = [session_id] if tip_only else self._resume_lineage_ids(session_id) + active_clause = "active = 1" if tip_only else "(active = 1 OR compacted = 1)" + placeholders = ",".join("?" for _ in session_ids) + row = self._read_one( + "SELECT COUNT(*) FROM (" + f"SELECT 1 FROM messages WHERE session_id IN ({placeholders}) " + f"AND {active_clause} LIMIT ?" + ")", + (*session_ids, max_messages + 1), + ) + message_count = int(row[0] if row else 0) + if message_count > max_messages: + raise SessionResumeTooLargeError( + message_count, + max_messages, + scope="in its tip segment" if tip_only else "across its lineage", + ) + return message_count + + def get_ancestor_display_prefix(self, session_id: str) -> List[Dict[str, Any]]: + """Return the ancestor-only display messages for a session lineage. + + These are messages from parent/grandparent sessions (compression + ancestors) that appear in the display transcript but NOT in the + tip session's model-fed history. Used by ``session.resume`` to + build the ``display_history_prefix`` that ``_live_session_payload`` + prepends to the live model history. + + Previously the prefix was calculated as + ``display_history[:len(display) - len(raw)]``, but that overcounts + when ``repair_message_sequence`` removes messages from the MIDDLE + of the tip history (e.g. verification candidates collapsed by the + consecutive-assistant merge) — the length difference includes both + ancestor messages AND repair-removed tip messages, but the slice + only captures the first N display messages (which are tip messages + when there are no ancestors), causing duplication. This method + returns ONLY the genuine ancestor messages, identified by + ``session_id != tip_session_id``. (#65919) + """ + session_ids = self._resume_lineage_ids(session_id) + if len(session_ids) <= 1: + return [] + with self._read_ctx() as conn: + placeholders = ",".join("?" for _ in session_ids) + rows = conn.execute( + f"SELECT session_id, {self._CONVERSATION_ROW_COLUMNS} " + f"FROM messages WHERE session_id IN ({placeholders}) " + # Display read: compaction-archived rows included, Undo/Rewind + # rows excluded (see get_resume_conversations). + "AND (active = 1 OR compacted = 1) " + "ORDER BY id", + tuple(session_ids), + ).fetchall() + rows = self._dedupe_display_generations(rows) + ancestor_ids = { + int(row["id"]) + for row in rows + if row["session_id"] != session_id and row["id"] is not None + } + if not ancestor_ids: + return [] + lineage = self._rows_to_conversation( + rows, + session_id=session_id, + include_ancestors=True, + repair_alternation=False, + include_row_ids=True, + ) + prefix: List[Dict[str, Any]] = [] + for message in lineage: + if message.get("_row_id") not in ancestor_ids: + continue + projected = message.copy() + projected.pop("_row_id", None) + prefix.append(projected) + return prefix + + def get_conversation_root(self, session_id: str) -> str: + """Return the ROOT id of *session_id*'s lineage chain. + + The root is the stable "conversation id": context compression + rotates ``session_id`` to a new segment linked via + ``parent_session_id``, and delegate subagents hang off their + parent the same way. Walking to the root gives every segment of + one user-facing conversation (and its delegation tree) a single + identifier — used for Nous Portal ``conversation=`` usage tagging. + Returns *session_id* unchanged when it has no recorded parent. + """ + chain = self._session_lineage_root_to_tip(session_id) + return (chain[0] if chain and chain[0] else session_id) + + @staticmethod + def _canonical_replayed_user_content( + msg: Dict[str, Any], + ) -> Tuple[Any, bool]: + """Return canonical live content and whether *msg* is composite.""" + if msg.get("role") != "user": + return None, False + + from agent.context_compressor import split_user_originated_turn + + handoff, live_view = split_user_originated_turn(msg) + is_composite = handoff is not None and live_view is not None + return ( + live_view.get("content") + if is_composite and live_view is not None + else msg.get("content"), + is_composite, + ) + + @staticmethod + def _exact_replayed_user_clone_key( + timestamp: Any, content: Any + ) -> Optional[Tuple[Any, str]]: + """Return a hashable key for a column-exact rotation clone.""" + if timestamp is None or content in (None, "", []): + return None + try: + encoded = json.dumps( + content, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError): + return None + return timestamp, encoded + + @staticmethod + def _find_duplicate_replayed_user_message( + messages: List[Dict[str, Any]], msg: Dict[str, Any] + ) -> Optional[Tuple[int, bool]]: + """Return an adjacent replay duplicate and whether *msg* must win. + + Compression rotation may persist the current ask once in the parent + and again inside a composite child carrier. Compare the canonical live + payload for that carrier, while retaining the historical exact-string + dedupe for ordinary replayed users. The child carrier wins because it + owns both the current durable row identity and the retained scaffold. + """ + from hermes_state import SessionDB + if msg.get("role") != "user": + return None + + content, prefer_current = SessionDB._canonical_replayed_user_content(msg) + if content in (None, "", []): + return None + + for index in range(len(messages) - 1, -1, -1): + prev = messages[index] + if prev.get("role") == "user": + prev_content, prev_is_composite = ( + SessionDB._canonical_replayed_user_content(prev) + ) + if prev_content == content and ( + prefer_current + or prev_is_composite + or isinstance(content, str) + ): + return index, prefer_current + if prev.get("role") == "assistant" and (prev.get("content") or prev.get("tool_calls")): + return None + return None + + def get_active_message_ids(self, session_id: str) -> List[int]: + """Return the ordered physical ids pinned by rewind CAS checks. + + Conversation projections intentionally omit legacy background-review + harness rows. Destructive rewinds must nevertheless pin every active + physical row so the caller snapshot matches the transaction-local + comparison in :meth:`rewind_to_message`. + """ + rows = self._read_all( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id", + (session_id,), + ) + return [int(row[0]) for row in rows] + + @staticmethod + def _active_transcript_counts(conn, session_id: str) -> tuple[int, int]: + """Return active message/tool-call counts inside the caller's txn.""" + rows = conn.execute( + "SELECT tool_calls FROM messages " + "WHERE session_id = ? AND active = 1", + (session_id,), + ).fetchall() + tool_call_count = 0 + for row in rows: + raw = row[0] + if not raw: + continue + try: + decoded = json.loads(raw) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError): + continue + if isinstance(decoded, list): + tool_call_count += len(decoded) + elif decoded: + tool_call_count += 1 + return len(rows), tool_call_count + + def rewind_to_message( + self, + session_id: str, + target_message_id: int, + *, + preserve_compaction_handoff: bool = False, + expected_active_ids: Optional[List[int]] = None, + expected_target_content: Any = None, + ) -> Dict[str, Any]: + """Soft-delete all messages with id >= ``target_message_id`` in *session_id*. + + The target message itself becomes inactive as well so the caller + can pre-fill it as the next user prompt without it appearing + twice in the replayed transcript. Rewound rows are kept on + disk with ``active=0`` for audit / forensic inspection — use + :meth:`get_messages` with ``include_inactive=True`` to see them. + + Returns a dict:: + + { + "rewound_count": int, # number of rows newly flipped to active=0 + "target_message": dict, # full row dict of the target + "new_head_id": int|None # id of the last still-active row, or None + } + + Raises ``ValueError`` if the target message does not exist in + *session_id* or if its role is not ``"user"``. With + ``preserve_compaction_handoff=True``, a composite summary carrier is + split inside the same write transaction: its original row is archived + and its canonical hidden handoff scaffold is inserted as the new head. + That opt-in result also contains ``replacement_message_id``. + + ``expected_active_ids`` optionally pins the ordered active row set. + ``expected_target_content`` additionally pins the selected canonical + live-user payload. Both checks run inside the write transaction before + any row or counter mutation. Presentation-only metadata changes (for + example Desktop reactions) deliberately do not invalidate a rewind. + A live cross-process turn lease always refuses the rewind; expired or + provably dead holders are reclaimed inside the mutation transaction. + + Always increments ``sessions.rewind_count`` — even when the + target is already inactive — so the counter accurately reflects + the number of rewind operations performed against the session. + Idempotent on the ``active`` flag: re-rewinding past the same + target is a no-op on row state but still bumps the counter. + """ + + def _do(conn): + # Rewind changes the active transcript and must honor the same + # compression/closed-parent and cross-process turn guards as + # append writers. + self._check_transcript_write_guards( + conn, + session_id, + None, + reject_active_turn_lease=True, + reject_active_compression_lock=True, + ) + + if expected_active_ids is not None: + active_rows = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND active = 1 ORDER BY id", + (session_id,), + ).fetchall() + active_ids = [int(active_row[0]) for active_row in active_rows] + if active_ids != expected_active_ids: + raise RuntimeError( + "active transcript changed before the rewind could be persisted" + ) + + row = conn.execute( + "SELECT * FROM messages WHERE id = ? AND session_id = ?", + (target_message_id, session_id), + ).fetchone() + if row is None: + raise ValueError( + f"message {target_message_id} not found in session {session_id}" + ) + target_row = dict(row) + if target_row.get("role") != "user": + raise ValueError( + f"rewind target must be a 'user' message (got role=" + f"{target_row.get('role')!r}, id={target_message_id})" + ) + + replacement_message_id: Optional[int] = None + replacement: Optional[Dict[str, Any]] = None + if preserve_compaction_handoff or expected_target_content is not None: + if not target_row.get("active"): + raise ValueError("rewind target is not active") + from agent.context_compressor import split_user_originated_turn + + split_target = target_row.copy() + split_target["content"] = self._decode_content( + split_target.get("content") + ) + split_target["display_metadata"] = self._decode_display_metadata( + split_target.get("display_metadata") + ) + handoff, live_view = split_user_originated_turn(split_target) + if live_view is None: + raise ValueError("rewind target is not a user-originated turn") + live_content = live_view.get("content") + if isinstance(live_content, str): + live_content = sanitize_context(live_content).strip() + if ( + expected_target_content is not None + and live_content != expected_target_content + ): + raise RuntimeError( + "rewind target changed before it could be persisted" + ) + if preserve_compaction_handoff and handoff is None: + raise ValueError( + "preserve_compaction_handoff requires an active composite carrier" + ) + replacement = handoff if preserve_compaction_handoff else None + + cursor = conn.execute( + "SELECT id FROM messages " + "WHERE session_id = ? AND id >= ? AND active = 1", + (session_id, target_message_id), + ) + ids = [r[0] for r in cursor.fetchall()] + if ids: + placeholders = ",".join("?" for _ in ids) + conn.execute( + f"UPDATE messages SET active = 0 WHERE id IN ({placeholders})", + ids, + ) + if replacement is not None: + self._insert_message_rows(conn, session_id, [replacement]) + inserted = conn.execute("SELECT last_insert_rowid()").fetchone() + replacement_message_id = int(inserted[0]) + conn.execute( + "UPDATE sessions SET rewind_count = COALESCE(rewind_count, 0) + 1 " + "WHERE id = ?", + (session_id,), + ) + message_count, tool_call_count = self._active_transcript_counts( + conn, session_id + ) + conn.execute( + "UPDATE sessions SET message_count = ?, tool_call_count = ? " + "WHERE id = ?", + (message_count, tool_call_count, session_id), + ) + head_row = conn.execute( + "SELECT MAX(id) FROM messages WHERE session_id = ? AND active = 1", + (session_id,), + ).fetchone() + new_head_id = ( + head_row[0] if head_row and head_row[0] is not None else None + ) + return target_row, ids, new_head_id, replacement_message_id + + target_row, rewound, new_head_id, replacement_message_id = ( + self._execute_write(_do) + ) + + # Decode content for callers (prefill the prompt buffer) without a + # second fallible database operation after the transaction commits. + target_row["content"] = self._decode_content(target_row.get("content")) + + result = { + "rewound_count": len(rewound), + "target_message": target_row, + "new_head_id": new_head_id, + } + if preserve_compaction_handoff: + result["replacement_message_id"] = replacement_message_id + return result + + def message_count(self, session_id: str = None) -> int: + """Count messages, optionally for a specific session.""" + with self._read_ctx() as conn: + if session_id: + cursor = conn.execute( + "SELECT COUNT(*) FROM messages WHERE session_id = ?", (session_id,) + ) + else: + cursor = conn.execute("SELECT COUNT(*) FROM messages") + return cursor.fetchone()[0] + + def has_platform_message_id( + self, session_id: str, platform_message_id: str + ) -> bool: + """Check if a message with the given platform_message_id exists. + + Uses the idx_messages_platform_msg_id partial index for efficient + lookup. Used by the gateway's transient-failure dedupe guard (#47237) + to skip re-persisting a user message that was already saved on a + prior retry of the same inbound platform message. + """ + return self._read_one( + "SELECT 1 FROM messages " + "WHERE session_id = ? AND platform_message_id = ? LIMIT 1", + (session_id, platform_message_id), + ) is not None + + def _is_explicit_fork_child_row(self, session: Dict[str, Any]) -> bool: + """True when ``session`` is a branch, delegate, or tool child of its parent. + + Markers only count as a fork when they point at ``parent_session_id``. + Compression copies ``model_config`` onto the continuation + (``publish_compression_child`` callers pass + ``agent._session_init_model_config``), so a delegate's continuation + carries ``_delegate_from=``. Presence-only + matching would treat that real continuation as a fork — the same + misclassification ``_NON_CONTINUATION_CHILD_FILTER_SQL`` already + avoids by binding both markers to the queried parent. + """ + if session.get("source") == "tool": + return True + raw = session.get("model_config") + if not raw: + return False + try: + cfg = json.loads(raw) if isinstance(raw, str) else raw + except (TypeError, json.JSONDecodeError): + return False + if not isinstance(cfg, dict): + return False + parent_id = session.get("parent_session_id") + branched = cfg.get("_branched_from") + delegated = cfg.get("_delegate_from") + if parent_id: + return branched == parent_id or delegated == parent_id + return branched is not None or delegated is not None + + def is_explicit_fork_child(self, session_id: str) -> bool: + """True when ``session_id`` is a /branch, delegate, or tool child row. + + Read-only public view of :meth:`_is_explicit_fork_child_row` for + callers that must respect the fork boundary without re-implementing + its marker rules (``agent/prompt_cache_scope.py`` keeps a declared + conversation key from crossing it). A missing row is not a fork. + """ + session = self.get_session(session_id) + return bool(session and self._is_explicit_fork_child_row(session)) + + def latest_conversation_boundary( + self, session_key: str, source: str + ) -> Optional[int]: + """How many conversation boundaries this routing peer has crossed. + + A boundary is a row this peer ended at an intentional conversation + break — the ``_RESET_END_REASONS`` set (``/new``, ``/switch``, idle, + daily, suspended, resume_pending_expired). That is the same fence + :meth:`find_latest_gateway_session_for_peer` refuses to reach behind, + so the two agree on where one conversation stops and the next begins + and cannot drift. + + The peer is ``(session_key, source)``, the SAME identity tuple recovery + uses — never the key alone. ``X-Hermes-Session-Key`` accepts any + authenticated caller-supplied string, so an API conversation may + legally carry the same key as a Telegram row in one database; keying + on the string alone would let a ``/new`` on that unrelated row rotate + this conversation's affinity identity while recovery correctly refuses + to cross the same line. + + Returns the count, or ``None`` when this peer has never been reset. + + The value comes from ``conversation_generations``, which + :meth:`_bump_conversation_generation` advances inside the transaction + that writes each boundary — NOT from an aggregate over the session + rows. An aggregate cannot prove non-reuse: ``delete_session()`` + orphans children and deletes the row, and bulk prune selects ended + rows, so ``COUNT``/``MAX`` over boundaries can return a pair it already + emitted and hand a new conversation a retired affinity identity. It is + also wall-clock-free, so a backwards NTP correction cannot reorder it. + + Databases upgraded mid-conversation start at no generation and take + their first one from the next boundary written; a conversation that + reset before the upgrade shares its predecessor's scope once, which + costs a warm prompt-cache bucket and never crosses an identity. + + These rows are never garbage-collected, by design: dropping one resets + the peer to "no generation", so its next boundary writes ``1`` again + and re-issues a scope a retired conversation already used — the ABA + this counter exists to prevent. See the schema comment in + ``hermes_state_common.py``. + """ + if not session_key or not source: + return None + row = self._read_one( + "SELECT generation FROM conversation_generations " + "WHERE source = ? AND session_key = ?", + (source, session_key), + ) + if row is None or row["generation"] is None: + return None + generation = int(row["generation"]) + return generation if generation > 0 else None + + def clear_messages(self, session_id: str) -> None: + """Delete all messages for a session and reset its counters.""" + def _do(conn): + conn.execute( + "DELETE FROM messages WHERE session_id = ?", (session_id,) + ) + conn.execute( + "UPDATE sessions SET message_count = 0, tool_call_count = 0 WHERE id = ?", + (session_id,), + ) + self._execute_write(_do) + + def purge_stale_tool_call_markers( + self, *, dry_run: bool = False, backup: bool = True + ) -> Dict[str, Any]: + """Permanently clear bare tool-call marker content (e.g. "[memory]") + left in the ``messages`` table by sessions persisted before the + #78148 fix in ``agent.conversation_loop``. + + ``_strip_stale_tool_call_markers`` already repairs this in memory on + every session load (see ``_rows_to_conversation``), so running this + is optional — but for long-lived sessions the same rows get + re-scanned and re-repaired on every resume, which is wasted work + and keeps the contaminated bytes sitting in the DB (and in any + downstream cache/backup snapshot of it) indefinitely. This rewrites + the affected rows once, in place. + + Only the ``content`` column is touched — ``role``, ``tool_calls``, + and every other column on the row are left exactly as they are, so + provider tool_call/tool_result pairing is unaffected. + + Unlike the in-memory repair, this UPDATE is permanent and can't be + undone from within the DB. Since ``backup`` defaults to True, a + timestamped full snapshot is taken via ``VACUUM INTO`` (safe against + a live connection, unlike the raw-copy ``_backup_db_file`` used for + malformed-schema repair) before any row is touched — mirroring + ``repair_state_db_schema``'s backup-by-default convention for + destructive state.db operations. No snapshot is taken when there is + nothing to change. + + With ``dry_run=True``, reports the affected row count/ids without + writing or backing up (read-only, no write lock taken). + + Returns ``{"dry_run": bool, "rows_affected": int, "row_ids": [...], + "backup_path": str|None}``. + """ + from hermes_state import _STALE_TOOL_CALL_MARKER_RE + + def _find_affected(conn) -> List[int]: + cursor = conn.execute( + "SELECT id, content FROM messages " + "WHERE role = 'assistant' AND tool_calls IS NOT NULL AND tool_calls != ''" + ) + affected: List[int] = [] + for row in cursor.fetchall(): + content = row["content"] + if isinstance(content, str) and _STALE_TOOL_CALL_MARKER_RE.fullmatch(content.strip()): + affected.append(row["id"]) + return affected + + with self._read_ctx() as conn: + affected_ids = _find_affected(conn) + + if dry_run: + return { + "dry_run": True, + "rows_affected": len(affected_ids), + "row_ids": affected_ids, + "backup_path": None, + } + + if not affected_ids: + return { + "dry_run": False, + "rows_affected": 0, + "row_ids": [], + "backup_path": None, + } + + backup_path: Optional[str] = None + if backup: + import datetime + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + dest = self.db_path.with_name( + f"{self.db_path.name}.pre-clean-markers-backup-{stamp}" + ) + with self._lock: + self._conn.execute("VACUUM INTO ?", (str(dest),)) + backup_path = str(dest) + logger.info("Backed up state.db to %s before clean-markers write", backup_path) + + def _do(conn): + ids = _find_affected(conn) + if ids: + placeholders = ",".join("?" * len(ids)) + conn.execute( + f"UPDATE messages SET content = '' WHERE id IN ({placeholders})", + ids, + ) + return ids + + affected_ids = self._execute_write(_do) + if affected_ids: + logger.info( + "Permanently cleared %d stale tool-call marker row(s) in state.db (#78148)", + len(affected_ids), + ) + return { + "dry_run": False, + "rows_affected": len(affected_ids), + "row_ids": affected_ids, + "backup_path": backup_path, + } diff --git a/hermes_state_portability.py b/hermes_state_portability.py index ca8df96809..2f740786ff 100644 --- a/hermes_state_portability.py +++ b/hermes_state_portability.py @@ -1,11 +1,9 @@ """Session listing/rich rows, export, and import (portability) for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import logging @@ -16,14 +14,12 @@ from typing import Any, Dict, List, Optional from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE from hermes_state_common import ( SCHEMA_SQL, - _PREVIEW_ELIGIBLE_SQL, - _PREVIEW_RAW_SELECT, + _PREVIEW_RAW_SUBQUERY_SQL, _shape_preview, _sql_session_last_active, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") @@ -32,9 +28,8 @@ class SessionPortabilityMixin: @classmethod def _compact_session_cols(cls) -> str: - """SELECT list for compact_rows: every ``sessions`` column declared in - SCHEMA_SQL except prompt storage internals, aliased with the ``s`` - prefix used by list_sessions_rich/_get_session_rich_row queries.""" + """``s.``-prefixed SELECT list of every SCHEMA_SQL ``sessions`` column + except prompt storage internals (the compact_rows projection).""" if cls._session_compact_cols_sql is None: declared = cls._parse_schema_columns(SCHEMA_SQL)["sessions"] cls._session_compact_cols_sql = ", ".join( @@ -43,13 +38,19 @@ class SessionPortabilityMixin: ) return cls._session_compact_cols_sql + @classmethod + def _rich_row(cls, row) -> Dict[str, Any]: + """Session row dict with ``_preview_raw`` shaped into ``preview``.""" + s = cls._session_row_dict(row) + s["preview"] = _shape_preview(s.pop("_preview_raw", "")) + return s + def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]: """Distinct non-empty session cwds with usage stats, for repo discovery. - Aggregates across ALL session history (not a single page), so the desktop - can surface every git repo the user has worked in — not just the repos - that happen to be in the currently-loaded recents. Children/branches - count: a worktree session is still a real workspace signal. + Aggregates across ALL history (not one page) so every repo the user + worked in surfaces. Children/branches count: a worktree session is a + real workspace signal. """ where = "cwd IS NOT NULL AND TRIM(cwd) != ''" if not include_archived: @@ -77,41 +78,24 @@ class SessionPortabilityMixin: ) -> List[Dict[str, Any]]: """List the run sessions produced by a single cron job, newest first. - Cron runs are flat, independent sessions whose id is - ``cron_{job_id}_{timestamp}`` (see ``cron/scheduler.run_job``). They are - never compression roots and never branch, so this deliberately skips the - ``list_sessions_rich`` recursive compression-chain CTE / leading-wildcard - ``id_query`` path — that path seeds from *every* ``source='cron'`` row in - the DB and only filters to one job's runs after the scan, so it scales - with the whole cron pile (a heavy history makes the desktop run-history - endpoint time out before it eventually populates). + Cron runs are flat sessions with id ``cron_{job_id}_{timestamp}``; they + never compress or branch, so this skips ``list_sessions_rich``'s + compression-chain CTE / leading-wildcard ``id_query`` path, which seeds + from EVERY ``source='cron'`` row and scales with the whole cron pile. + Instead: a ``[prefix, prefix_hi)`` index range scan on id, filtered to + ``source='cron'``, so work scales with the requested window. - Instead this binds to one job with a ``[prefix, prefix_hi)`` range over - the id (an index range scan, not a ``%...%`` substring), filters - ``source='cron'``, and orders by ``started_at DESC``. Work scales with - the requested window, not the total cron history. - - Returns the same enriched row shape as ``list_sessions_rich`` (adds - ``preview`` + ``last_active``) so callers can reuse it. + Returns the ``list_sessions_rich`` row shape (``preview`` + ``last_active``). """ prefix = f"cron_{job_id}_" - # Half-open upper bound for an index range scan: increment the final - # byte of the prefix so the range covers exactly the ids that start - # with ``prefix`` and nothing else. ``prefix`` always ends in '_', but - # compute it generically rather than hardcoding the successor char. + # Half-open upper bound: bump the final byte so the range covers exactly + # the ids starting with ``prefix``. prefix_hi = prefix[:-1] + chr(ord(prefix[-1]) + 1) query = f""" SELECT s.*, COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, + {_PREVIEW_RAW_SUBQUERY_SQL}, {_sql_session_last_active("s")} AS last_active FROM sessions s LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash @@ -120,27 +104,12 @@ class SessionPortabilityMixin: LIMIT ? OFFSET ? """ with self._lock: - cursor = self._conn.execute(query, (prefix, prefix_hi, limit, offset)) - rows = cursor.fetchall() - - runs: List[Dict[str, Any]] = [] - for row in rows: - s = self._session_row_dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - runs.append(s) - return runs + rows = self._conn.execute(query, (prefix, prefix_hi, limit, offset)).fetchall() + return [self._rich_row(row) for row in rows] def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Fetch a single session with the same enriched columns as - ``list_sessions_rich`` (preview + last_active). Returns None if the - session doesn't exist. - - Pass ``compact_rows=True`` to omit the ``system_prompt`` blob (see - ``list_sessions_rich`` for details). - - Thin wrapper over ``_get_session_rich_rows_batch`` so the enriched - SELECT lives in exactly one place. - """ + """One session with the ``list_sessions_rich`` enriched columns, or + None. ``compact_rows=True`` omits the ``system_prompt`` blob.""" return self._get_session_rich_rows_batch( [session_id], compact_rows=compact_rows ).get(session_id) @@ -148,23 +117,15 @@ class SessionPortabilityMixin: def _get_session_rich_rows_batch( self, session_ids, compact_rows: bool = False ) -> Dict[str, Dict[str, Any]]: - """Fetch multiple sessions with the same enriched columns as - ``_get_session_rich_row``, in a single query. - - Used by ``list_sessions_rich``'s compression-tip projection to resolve - every tip row for a page in one round trip instead of one query per - compression-root row. Returns a dict keyed by session id; ids that - don't exist are simply absent from the result (same as - ``_get_session_rich_row`` returning ``None`` for them). + """Enriched rows for many sessions in one query, keyed by id; missing + ids are simply absent. Resolves a page of compression tips in one + round trip instead of one query per root row. """ ids = [sid for sid in session_ids if sid] if not ids: return {} - # Old SQLite builds cap bound variables at 999 - # (SQLITE_MAX_VARIABLE_NUMBER); large pages (limit=10000 callers - # exist) could exceed it. Chunk the IN list so the helper is safe at - # any page size — this is the single choke point for the enriched - # multi-row fetch, so the bound lives here, not at call sites. + # Old SQLite caps bound variables at 999 (SQLITE_MAX_VARIABLE_NUMBER); + # limit=10000 callers exist. Chunk here — the single choke point. _CHUNK = 900 if len(ids) > _CHUNK: result: Dict[str, Dict[str, Any]] = {} @@ -189,45 +150,26 @@ class SessionPortabilityMixin: ) query = f""" SELECT {_sel}{prompt_select}, - COALESCE( - (SELECT {_PREVIEW_RAW_SELECT} - FROM messages m - WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL - AND {_PREVIEW_ELIGIBLE_SQL} - ORDER BY m.timestamp, m.id LIMIT 1), - '' - ) AS _preview_raw, + {_PREVIEW_RAW_SUBQUERY_SQL}, {_sql_session_last_active("s")} AS last_active FROM sessions s {prompt_join} WHERE s.id IN ({placeholders}) """ with self._lock: - cursor = self._conn.execute(query, ids) - rows = cursor.fetchall() - result: Dict[str, Dict[str, Any]] = {} - for row in rows: - s = self._session_row_dict(row) - s["preview"] = _shape_preview(s.pop("_preview_raw", "")) - result[s["id"]] = s - return result + rows = self._conn.execute(query, ids).fetchall() + return {s["id"]: s for s in map(self._rich_row, rows)} def get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]: - """Public wrapper for :meth:`_get_session_rich_row`. - - Exposes the single-session enriched row (same columns as - ``list_sessions_rich``: preview + last_active) for callers outside - this module, e.g. the web server's session-search hydration. - """ + """Public wrapper for :meth:`_get_session_rich_row` (web server hydration).""" return self._get_session_rich_row(session_id, compact_rows=compact_rows) def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]: """Titled sessions whose first user turn was a ``/skill`` invocation. - Those titles were generated from the expanded message, which embeds the - whole skill body — so they describe the skill rather than the request. - Returns ``id``, ``title``, and the full first-turn ``content`` so a - caller can re-derive what the user typed. Newest first. + Their titles were generated from the expanded skill body, so they + describe the skill, not the request. Returns ``id``, ``title`` and the + first-turn ``content`` so callers can re-derive what was typed. Newest first. """ with self._lock: rows = self._conn.execute( @@ -248,63 +190,36 @@ class SessionPortabilityMixin: ).fetchall() return [dict(row) for row in rows] - def get_first_assistant_text(self, session_id: str) -> str: - """The session's first assistant reply as plain text ('' when none). - - Pairs with :meth:`list_skill_scaffolded_sessions` so a re-title can feed - the titler the same (request, reply) shape the live path uses. - """ - with self._lock: - row = self._conn.execute( - "SELECT content FROM messages " - "WHERE session_id = ? AND role = 'assistant' AND content IS NOT NULL " - "ORDER BY timestamp, id LIMIT 1", - (session_id,), - ).fetchone() - if not row: - return "" - decoded = self._decode_content(row["content"]) - return decoded if isinstance(decoded, str) else "" - def export_session(self, session_id: str) -> Optional[Dict[str, Any]]: """Export a single session with all its messages as a dict.""" session = self.get_session(session_id) if not session: return None - messages = self.get_messages(session_id) - return {**session, "messages": messages} + return {**session, "messages": self.get_messages(session_id)} def export_session_lineage(self, session_id: str) -> Optional[Dict[str, Any]]: """Export a compression lineage as one logical session dict.""" lineage_ids = self.get_compression_lineage(session_id) if not lineage_ids: return None - segments = [] - for sid in lineage_ids: - segment = self.export_session(sid) - if segment: - segments.append(segment) + segments = [seg for seg in map(self.export_session, lineage_ids) if seg] if not segments: return None - base = dict(segments[-1]) - total_messages = sum(len(seg.get("messages") or []) for seg in segments) - base["segments"] = segments - base["lineage_session_ids"] = [seg["id"] for seg in segments] - base["message_count"] = total_messages - base["messages"] = [msg for seg in segments for msg in (seg.get("messages") or [])] - return base + messages = [msg for seg in segments for msg in (seg.get("messages") or [])] + return { + **segments[-1], + "segments": segments, + "lineage_session_ids": [seg["id"] for seg in segments], + "message_count": len(messages), + "messages": messages, + } def export_all(self, source: str = None) -> List[Dict[str, Any]]: - """ - Export all sessions (with messages) as a list of dicts. - Suitable for writing to a JSONL file for backup/analysis. - """ - sessions = self.search_sessions(source=source, limit=100000) - results = [] - for session in sessions: - messages = self.get_messages(session["id"]) - results.append({**session, "messages": messages}) - return results + """Export all sessions (with messages) as dicts, e.g. for JSONL backup.""" + return [ + {**session, "messages": self.get_messages(session["id"])} + for session in self.search_sessions(source=source, limit=100000) + ] def adopt_session_lineage_from( self, @@ -313,52 +228,36 @@ class SessionPortabilityMixin: *, retire_donor: bool = True, ) -> Dict[str, Any]: - """Adopt *session_id*'s full compression lineage from *donor_db* into - this store. + """Adopt *session_id*'s full compression lineage from *donor_db*. - The stranded-bot-session heal (#93091 follow-up to #93296): before the - desktop routed session RPCs by their target session, a profile bot's - turns executed on whichever backend held window focus — usually the - default one — so the bot's canonical session rows and messages - accumulated in the DEFAULT profile's state.db. Once routing was fixed, - the profile backend correctly received the RPCs but had no such - session, so the same chat 4001'd for the opposite reason. This method - moves the conversation to where routing now looks for it. + Stranded-bot-session heal: before the desktop routed session RPCs by + target session, a profile bot's rows accumulated in the DEFAULT + profile's state.db; this moves the conversation to where routing now + looks. Pure composition: ``donor_db.export_session_lineage()`` -> + ``self.import_sessions()`` — routing/handoff/activity fields reset, + already-present ids skipped (idempotent re-adoption). - Composition of existing primitives (no new import/export machinery): - ``donor_db.export_session_lineage()`` -> ``self.import_sessions()``. - Import semantics apply unchanged: gateway routing, handoff, and live - activity fields are reset; already-present ids are skipped - (idempotent re-adoption after a partial run). + With ``retire_donor`` and a complete adoption, donor rows are ARCHIVED + (never deleted) with ``end_reason='adopted_by_profile'``. That + end_reason is deliberately NOT in the recoverable set + (agent_close/ws_orphan_reap): resurrection must not undo an adoption. - When ``retire_donor`` is True and at least one segment was imported - (or every segment already exists here), the donor rows are ARCHIVED — - never deleted — with ``end_reason='adopted_by_profile'`` so the - default profile's list stops advertising a conversation that now - lives elsewhere, while the bytes stay recoverable. The archive is - deliberately NOT in the recoverable set (agent_close/ws_orphan_reap): - canonical-lookup resurrection must not undo an adoption. - - Returns the ``import_sessions`` result dict, plus ``adopted`` (bool) - and ``donor_retired`` (bool — True only when EVERY segment's - retirement actually applied). + Returns the ``import_sessions`` dict plus ``adopted`` and + ``donor_retired`` (True only when EVERY segment's retirement applied). """ payload = donor_db.export_session_lineage(session_id) if not payload: return { - "ok": False, - "adopted": False, - "donor_retired": False, + "ok": False, "adopted": False, "donor_retired": False, "error": f"session {session_id!r} not found in donor store", } segments = payload.get("segments") or [payload] - # Divergence guard: a segment we are about to SKIP (already present - # here) may have kept accumulating messages in the donor store after - # a partial earlier adoption. Retiring it would strand those newer - # messages behind a non-recoverable archive. Compare counts up front - # and refuse to retire (still adopt/import) when the donor is ahead. + # Divergence guard: a segment we will SKIP (already here) may have kept + # growing in the donor after a partial adoption; retiring it would strand + # those messages behind a non-recoverable archive. Still import, but + # refuse to retire when the donor is ahead. donor_ahead = False for seg in segments: seg_id = seg.get("id") @@ -394,15 +293,11 @@ class SessionPortabilityMixin: if not seg_id: continue try: - # TOCTOU close-out: the guard above compared EXPORT-TIME - # counts, but another backend can append donor messages - # between export and this loop. Re-read both stores right - # before stamping; a donor-ahead signal here skips the - # stamp so growth never lands behind a non-recoverable - # archive. (Count comparison cannot see equal-count - # CONTENT divergence — e.g. a donor rewind+rewrite; that - # residual case is accepted: bytes stay in the donor - # store either way, only reachability differs.) + # TOCTOU close-out: the guard above used EXPORT-TIME counts; + # re-read both stores right before stamping so donor growth + # never lands behind a non-recoverable archive. (Equal-count + # CONTENT divergence is accepted: bytes stay in the donor + # either way, only reachability differs.) donor_now = len(donor_db.get_messages(seg_id)) local_now = len(self.get_messages(seg_id)) if donor_now > local_now: @@ -414,17 +309,15 @@ class SessionPortabilityMixin: seg_id, donor_now, local_now, ) continue - # First end_reason wins in end_session(); reopen first so - # the adoption boundary is stamped even on ended segments - # (e.g. 'compression' parents). + # First end_reason wins in end_session(); reopen so the + # adoption boundary is stamped even on ended segments. donor_db.reopen_session(seg_id) donor_db.end_session(seg_id, "adopted_by_profile") donor_db.set_session_archived(seg_id, True) except Exception: - # Best-effort by design: a retirement failure must not - # fail the adoption (the profile copy is already whole; - # a later resume retries retirement idempotently). But - # never claim success we didn't have. + # Best-effort: a retirement failure must not fail the adoption + # (a later resume retries idempotently) — but never claim + # success we didn't have. retire_ok = False logger.warning( "failed to retire donor segment %s after adoption", @@ -507,21 +400,16 @@ class SessionPortabilityMixin: def import_sessions(self, sessions: List[Dict[str, Any]]) -> Dict[str, Any]: """Import sessions exported by :meth:`export_session` or ``export_all``. - Existing session IDs are skipped. Imported child sessions keep their - parent only when that parent already exists or is included in the same - import payload; otherwise the child is detached so partial imports don't - fail foreign-key validation. Gateway routing, handoff, rewind, and other - live runtime state are intentionally reset: this restores conversation - history, not ownership of a live channel or process. + Existing ids are skipped. A child keeps its parent only when the parent + exists or is in the same payload; otherwise it is detached so partial + imports pass FK validation. Gateway routing, handoff, rewind and other + live runtime state are reset: this restores history, not ownership of + a live channel or process. - Activity contract (#76354 review S4): export INCLUDES the live - activity fields (``last_activity_at`` / ``last_activity_description`` - / ``last_activity_provenance``) because they are part of the durable - row, but import deliberately RESETS them to NULL. Resurrecting a - stale "working ..." label on a machine where no agent is running - would fabricate activity the watchdog and session listings act on. - This asymmetry is intentional and covered by regression - (tests/gateway/test_watchdog_review_76354.py::test_s4_export_includes_activity_import_resets_it). + Activity contract: export INCLUDES ``last_activity_*`` (durable row + fields) but import RESETS them to NULL — resurrecting a stale + "working ..." label would fabricate activity the watchdog and listings + act on. Intentional asymmetry, pinned by regression test. """ if not isinstance(sessions, list): raise ValueError("sessions must be a list") @@ -536,32 +424,14 @@ class SessionPortabilityMixin: total_messages = 0 total_bytes = 0 session_text_fields = ( - "source", - "user_id", - "model", - "system_prompt", - "end_reason", - "cwd", - "git_branch", - "git_repo_root", - "billing_provider", - "billing_base_url", - "billing_mode", - "cost_status", - "cost_source", - "pricing_version", - "title", + "source", "user_id", "model", "system_prompt", "end_reason", "cwd", + "git_branch", "git_repo_root", "billing_provider", "billing_base_url", + "billing_mode", "cost_status", "cost_source", "pricing_version", "title", ) + # ``role`` is validated separately below (non-empty string). message_text_fields = ( - "role", - "tool_call_id", - "tool_name", - "effect_disposition", - "finish_reason", - "reasoning", - "reasoning_content", - "platform_message_id", - "message_id", + "tool_call_id", "tool_name", "effect_disposition", "finish_reason", + "reasoning", "reasoning_content", "platform_message_id", "message_id", ) for index, raw in enumerate(sessions): @@ -580,43 +450,23 @@ class SessionPortabilityMixin: errors.append(self._import_error(index, session_id, "messages must be a list")) continue if len(messages) > self._IMPORT_MAX_MESSAGES_PER_SESSION: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the per-session import limit", - ) - ) + errors.append(self._import_error(index, session_id, "messages exceeds the per-session import limit")) continue if any(not isinstance(msg, dict) for msg in messages): - errors.append( - self._import_error( - index, - session_id, - "messages must contain only objects", - ) - ) + errors.append(self._import_error(index, session_id, "messages must contain only objects")) continue try: - session_bytes = len( - json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - ) + session_bytes = len(json.dumps(raw, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) except (TypeError, ValueError): - errors.append( - self._import_error(index, session_id, "session must be JSON serializable") - ) + errors.append(self._import_error(index, session_id, "session must be JSON serializable")) continue if session_bytes > self._IMPORT_MAX_SESSION_BYTES: - errors.append( - self._import_error(index, session_id, "session exceeds the import size limit") - ) + errors.append(self._import_error(index, session_id, "session exceeds the import size limit")) continue total_bytes += session_bytes if total_bytes > self._IMPORT_MAX_TOTAL_BYTES: - errors.append( - self._import_error(index, session_id, "import exceeds the total size limit") - ) + errors.append(self._import_error(index, session_id, "import exceeds the total size limit")) continue try: @@ -640,8 +490,6 @@ class SessionPortabilityMixin: if not isinstance(role, str) or not role: raise ValueError(f"messages[{message_index}].role must be a non-empty string") for field in message_text_fields: - if field == "role": - continue clean_message[field] = self._import_text_or_none( clean_message.get(field), field ) @@ -655,13 +503,7 @@ class SessionPortabilityMixin: total_messages += len(clean_messages) if total_messages > self._IMPORT_MAX_TOTAL_MESSAGES: - errors.append( - self._import_error( - index, - session_id, - "messages exceeds the total import limit", - ) - ) + errors.append(self._import_error(index, session_id, "messages exceeds the total import limit")) continue seen_ids.add(session_id) normalized.append( @@ -669,13 +511,7 @@ class SessionPortabilityMixin: ) if errors: - return { - "ok": False, - "imported": 0, - "skipped": 0, - "detached": 0, - "errors": errors, - } + return {"ok": False, "imported": 0, "skipped": 0, "detached": 0, "errors": errors} def _do(conn): imported_ids: List[str] = [] @@ -738,27 +574,17 @@ class SessionPortabilityMixin: "end_reason": raw.get("end_reason"), "input_tokens": self._int_or_default(raw.get("input_tokens")), "output_tokens": self._int_or_default(raw.get("output_tokens")), - "cache_read_tokens": self._int_or_default( - raw.get("cache_read_tokens") - ), - "cache_write_tokens": self._int_or_default( - raw.get("cache_write_tokens") - ), - "reasoning_tokens": self._int_or_default( - raw.get("reasoning_tokens") - ), + "cache_read_tokens": self._int_or_default(raw.get("cache_read_tokens")), + "cache_write_tokens": self._int_or_default(raw.get("cache_write_tokens")), + "reasoning_tokens": self._int_or_default(raw.get("reasoning_tokens")), "cwd": raw.get("cwd"), "git_branch": raw.get("git_branch"), "git_repo_root": raw.get("git_repo_root"), "billing_provider": raw.get("billing_provider"), "billing_base_url": raw.get("billing_base_url"), "billing_mode": raw.get("billing_mode"), - "estimated_cost_usd": self._float_or_none( - raw.get("estimated_cost_usd") - ), - "actual_cost_usd": self._float_or_none( - raw.get("actual_cost_usd") - ), + "estimated_cost_usd": self._float_or_none(raw.get("estimated_cost_usd")), + "actual_cost_usd": self._float_or_none(raw.get("actual_cost_usd")), "cost_status": raw.get("cost_status"), "cost_source": raw.get("cost_source"), "pricing_version": raw.get("pricing_version"), @@ -771,18 +597,12 @@ class SessionPortabilityMixin: sanitized_messages: List[Dict[str, Any]] = [] for msg in messages: clean = dict(msg) - for key in ( - "reasoning_details", - "codex_reasoning_items", - "codex_message_items", - ): + for key in ("reasoning_details", "codex_reasoning_items", "codex_message_items"): clean[key] = self._reasoning_json_value(clean.get(key)) sanitized_messages.append(clean) total_messages, total_tool_calls = self._insert_message_rows( - conn, - session_id, - sanitized_messages, + conn, session_id, sanitized_messages ) conn.execute( "UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?", @@ -826,9 +646,8 @@ class SessionPortabilityMixin: (parent_id, session_id), ) else: - # Drop only the closing edge. Later entries can still attach - # to this now-root session, preserving the acyclic portion - # of a malformed imported lineage. + # Drop only the closing edge; later entries can still attach + # to this now-root session. parent_by_child.pop(session_id, None) detached += 1 diff --git a/hermes_state_registry.py b/hermes_state_registry.py index 0f3bcedc20..70d83cd34e 100644 --- a/hermes_state_registry.py +++ b/hermes_state_registry.py @@ -1,40 +1,31 @@ -"""Process-wide shared SessionDB registry (#90837). +"""Process-wide shared SessionDB registry. -A gateway process opens state.db from many call sites — the runner's -``AsyncSessionDB``, the ``SessionStore`` per-path cache, per-agent lazy -recall (``run_agent._get_session_db_for_recall``), per-job cron opens, -and per-message opens in mirror / channel_directory / slash_commands / -shutdown_flush / session_search / react_to_message. Each bare -``SessionDB()`` mints its own writer connection, ``self._lock``, -close-time WAL checkpoint, and async token-writer thread. With N -independent writer connections on one WAL file, mutual exclusion relies -only on SQLite's WAL write lock plus each instance's busy_timeout retry -ladder — and one connection's close-time checkpoint can race another's -growth, producing the lost/reordered-page-write signature reported -across 11+ incidents (#90837). - -This module owns that boundary: one shared ``SessionDB`` per resolved -path per process, refcounted, with generation-aware retirement when the -underlying file is replaced (snapshot restore, recovery swap). +A gateway process opens state.db from many call sites (runner, SessionStore, +per-agent recall, cron, per-message helpers). Each bare ``SessionDB()`` mints +its own writer connection, lock, close-time WAL checkpoint and token-writer +thread; N independent writers on one WAL file rely only on SQLite's write lock +plus busy_timeout, and one connection's close-time checkpoint can race another's +growth (lost/reordered-page-write corruption). This module owns that boundary: +one shared ``SessionDB`` per resolved path per process, refcounted, with +generation-aware retirement when the file is replaced (snapshot restore, +recovery swap). Lifecycle rules: -- ``acquire(path)`` returns the current generation for *path*, - incrementing its refcount. Same path ⇒ same instance ⇒ same writer - connection. -- ``close()`` on a shared instance is a NO-OP. The registry — not any - individual caller — owns the connection lifecycle, so one caller's - ``close()`` can never tear down a writer other callers still hold. -- ``release(db)`` decrements the generation *db was acquired from* - (object-keyed, not pathname-keyed, so an inode replacement cannot - strand a still-owned generation). The final release of a retired - generation tears it down. -- On inode change, the old generation is RETIRED — never lent again — - but stays alive until its existing holders release. If a replacement - open fails, the registry is left WITHOUT a path entry (never a closed - stale object), so the next acquire retries fresh. -- All teardown happens OUTSIDE the registry lock: a final release's - WAL checkpoint must never stall acquisition for every state.db. +- ``acquire(path)`` returns the current generation for *path* and bumps its + refcount. Same path ⇒ same instance ⇒ same writer connection. +- ``close()`` on a shared instance is a NO-OP: the registry, not any caller, + owns the connection lifecycle, so one caller can never tear down a writer + others still hold. +- ``release(db)`` decrements the generation *db was acquired from* (object- + keyed, not pathname-keyed, so an inode replacement cannot strand a + still-owned generation). The final release of a retired generation tears + it down. +- On inode change the old generation is RETIRED (never lent again) but stays + alive until its holders release. If the replacement open fails the registry + keeps NO path entry (never a closed stale object) so the next acquire retries. +- All teardown happens OUTSIDE the registry lock: a final release's WAL + checkpoint must never stall acquisition for every state.db. """ from __future__ import annotations @@ -44,34 +35,14 @@ import threading from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from hermes_state_common import stat_db_file_identity as _stat_db_file_identity + if TYPE_CHECKING: # pragma: no cover - import cycle guard, typed only from hermes_state import SessionDB logger = logging.getLogger(__name__) -def _stat_db_file_identity(path: Path) -> Optional[Tuple[int, int]]: - """Return ``(st_dev, st_ino)`` for *path*, or None when unavailable. - - Mirrors the hermes_state helper of the same name; kept local so this - module has no import-time dependency on hermes_state (which imports - this module — the cycle is resolved by deferring SessionDB lookup - to call time). - """ - import os - - try: - st = os.stat(path) - except OSError: - return None - # Windows volumes (and some network FS) report st_ino=0; a (0, 0) - # identity would false-positive every check. Skip the inode half of - # the guard there. - if not st.st_dev or not st.st_ino: - return None - return (st.st_dev, st.st_ino) - - class _Generation: """One shared SessionDB generation: instance, refcount, file identity.""" @@ -85,16 +56,13 @@ class _Generation: _lock = threading.Lock() -# path → live generation (never retired). A retired generation leaves -# this table immediately on retirement and lives on in _retired until -# its last holder releases. +# path → live generation. Retired generations move to _retired (keyed by +# id(db)) until their last holder releases. _generations: Dict[Path, _Generation] = {} -# Object-keyed retired generations still draining holders. -_retired: Dict[int, _Generation] = {} # id(db) → generation -# Paths whose next generation is currently being constructed. Construction -# stays outside _lock because schema reconciliation can take seconds, but peers -# for the SAME file must wait: otherwise every cold caller opens a writable -# SQLite connection before the registry chooses one winner. +_retired: Dict[int, _Generation] = {} +# Paths whose next generation is being constructed. Construction runs outside +# _lock (schema reconciliation can take seconds), but peers for the SAME file +# must wait or every cold caller opens its own writer before a winner is chosen. _opening: Dict[Path, threading.Event] = {} @@ -120,20 +88,13 @@ def _teardown(db: "SessionDB") -> None: def acquire(db_path: Optional[Path] = None) -> "SessionDB": """Return the shared SessionDB for *db_path*, incrementing its refcount. - The same resolved path always returns the same ``SessionDB`` instance - within one process, so all long-lived in-process callers share one - writer connection, one ``self._lock``, and one token-writer thread. + If the file was replaced (different inode) since the generation opened — + ``hermes sessions recover``, snapshot restore — that generation is RETIRED + but stays alive for its holders, and a fresh one is opened in its place. - If the underlying file was replaced (different inode) since the - shared generation was opened — e.g. by ``hermes sessions recover`` or - a snapshot restore — the current generation is RETIRED (never lent - again) but stays alive for its existing holders, and a fresh - generation is opened in its place. - - Raises whatever ``SessionDB.__init__`` raises (malformed, locked, - etc.). On a replacement-open failure the registry holds NO entry for - the path, so the next acquire retries fresh rather than handing out - a closed stale object. + Raises whatever ``SessionDB.__init__`` raises. On a replacement-open + failure the registry holds NO entry for the path, so the next acquire + retries fresh instead of receiving a closed stale object. """ from hermes_state import _default_db_path @@ -153,9 +114,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": and generation.identity is not None and current != generation.identity ): - # File replaced: retire the live generation (its - # holders keep it until they release) and elect one - # caller to construct the replacement below. + # File replaced: retire, then elect one caller to open + # the replacement below. _retire_generation_locked(path, generation) else: generation.refcount += 1 @@ -167,13 +127,12 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": _opening[path] = opening break - # Another caller is constructing this path. Do not hold the global - # registry lock while waiting: unrelated databases continue opening. - # A failed opener signals too, so one waiter can retry as the successor. + # Another caller is constructing this path; wait without holding the + # global lock. A failed opener signals too, so a waiter can retry. opening.wait() - # Open a fresh generation OUTSIDE the lock. The per-path opening marker - # prevents redundant writer connections without serialising other files. + # Open OUTSIDE the lock; the per-path marker prevents redundant writers + # without serialising other files. try: db = _open_session_db(path) db._shared_registry_owned = True @@ -188,8 +147,7 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": with _lock: existing = _generations.get(path) if existing is not None: - # Defensive: a generation may have been installed by explicit - # registry manipulation while this open was in flight. + # Defensive: installed by explicit registry manipulation mid-open. existing.refcount += 1 winner = existing.db else: @@ -206,9 +164,8 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB": def _retire_generation_locked(path: Path, generation: _Generation) -> None: """Retire *generation* so it is never lent again (caller holds _lock). - The instance stays alive — its holders still own references — and is - tracked in ``_retired`` keyed by ``id(db)`` so their releases find - the right generation even after the path maps to a new one. + It stays alive for its holders, tracked in ``_retired`` by ``id(db)`` so + their releases find it even after the path maps to a new generation. """ generation.retired = True if _generations.get(path) is generation: @@ -219,15 +176,11 @@ def _retire_generation_locked(path: Path, generation: _Generation) -> None: def release(db: "SessionDB") -> bool: """Decrement the refcount of a shared SessionDB. - Returns ``True`` if *db* was a shared instance and its refcount was - decremented; ``False`` if *db* is not registry-managed (caller owns - its own close()). The final release of a generation tears it down — - OUTSIDE the registry lock, so a close-time WAL checkpoint never - stalls acquisition for every state.db in the process. - - Object-keyed lookup means an inode replacement cannot strand a - still-owned generation: holders of the old generation release into - the retired record, not into whatever the path currently names. + Returns ``True`` if *db* was shared; ``False`` if it is not registry-managed + (caller owns close()). The final release tears the generation down OUTSIDE + the registry lock so a close-time WAL checkpoint never stalls acquisition. + Lookup is object-keyed, so holders of an old generation release into its + retired record, not into whatever the path currently names. """ if db is None: return False @@ -244,8 +197,7 @@ def release(db: "SessionDB") -> bool: return False generation = _generations.get(path) if generation is None or generation.db is not db: - # Not a shared instance (caller used SessionDB() - # directly) — nothing to do; the caller owns close(). + # Not shared (bare SessionDB()); the caller owns close(). return False generation.refcount -= 1 needs_teardown = generation.refcount <= 0 @@ -259,21 +211,17 @@ def release(db: "SessionDB") -> bool: _generations.pop(Path(path), None) except (TypeError, ValueError): pass - # Teardown OUTSIDE the lock: it stops the token writer, checkpoints - # the WAL, and drains the read pool — none of which may hold up - # acquisition for every other state.db in the process. + # Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and + # read-pool drain must not block acquisition for every other state.db. if needs_teardown: _teardown(db) return True def close_all() -> int: - """Close every shared SessionDB in this process, regardless of refcount. + """Close every shared SessionDB regardless of refcount; returns the count. - Called at gateway shutdown (after all agents and cron jobs have - finished) to release every WAL write lock and drain every - token-writer thread cleanly. Returns the number of instances - closed. Idempotent. + For gateway shutdown, after all agents and cron jobs finished. Idempotent. """ closed = 0 with _lock: @@ -282,7 +230,6 @@ def close_all() -> int: _retired.clear() for generation in generations: generation.retired = True - # Teardown outside the lock, one generation at a time. for generation in generations: _teardown(generation.db) closed += 1 @@ -290,12 +237,11 @@ def close_all() -> int: def live_shared_session_dbs() -> List["SessionDB"]: - """Snapshot of every live (non-retired) shared SessionDB in this process. + """Snapshot of every live (non-retired) shared SessionDB. - For periodic in-process maintenance (the gateway housekeeping tick's - deferred-FTS retry). Refcounts are NOT touched: the caller only invokes - a method on an instance that some holder already keeps alive; a - concurrent final release closes it and the callee sees ``_conn is None``. + For in-process maintenance (housekeeping deferred-FTS retry). Refcounts + are NOT touched: a concurrent final release may close an instance, in + which case the callee sees ``_conn is None``. """ with _lock: return [g.db for g in _generations.values() if not g.retired] @@ -314,9 +260,7 @@ def stats() -> Dict[str, int]: } -# ── Backwards-compatible aliases (hermes_state re-exports) ── -# Kept so call sites and tests can import either from hermes_state -# (the historical path) or from this module directly. +# ── Backwards-compatible aliases (hermes_state re-exports them) ── def get_shared_session_db(db_path: Optional[Path] = None) -> "SessionDB": return acquire(db_path) @@ -333,10 +277,8 @@ def close_shared_session_dbs() -> int: def release_or_close(db: "SessionDB") -> None: """Release a shared instance, or close it when it is not registry-managed. - The one-line cleanup for call sites that previously did a plain - ``db.close()``: shared instances return their refcount to the - registry (the registry owns the lifecycle), anything else — read-only - opens, CLI one-shots, test fakes — falls back to a direct close. + Drop-in for a plain ``db.close()``: read-only opens, CLI one-shots and + test fakes fall back to a direct close. """ if not release(db): try: diff --git a/hermes_state_repair.py b/hermes_state_repair.py new file mode 100644 index 0000000000..ba10d93376 --- /dev/null +++ b/hermes_state_repair.py @@ -0,0 +1,1639 @@ +"""state.db repair, backup and writability preflight (split from hermes_state). + +Every name is re-imported into ``hermes_state``; intra-module calls to +patchable helpers go through a lazy ``from hermes_state import ...`` at call +time so monkeypatches there still intercept. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import logging +import os +import sqlite3 +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from hermes_constants import get_hermes_home +from hermes_startup_watchdog import report_startup_progress +from hermes_state_common import ( + _acquire_db_flock, + _clear_lock_holder_record, + _describe_lock_holder, + _read_lock_holder_record, + is_advisory_lock_contention, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +def _claim_repair_attempt(db_path: Path) -> bool: + """Claim the one-shot per-process repair attempt for *db_path*. + + True for the first caller, False afterwards: bounds the repair/reopen loop + and stops concurrent callers racing surgery on one file. + """ + from hermes_state import _repair_attempt_lock, _repair_attempted_paths + key = str(db_path) + with _repair_attempt_lock: + if key in _repair_attempted_paths: + return False + _repair_attempted_paths.add(key) + return True + + +_REPAIR_LOCK_POLL_SECONDS = 0.1 + + +# Snapshot copies are data transfer, not inter-process locking: bound them +# separately at 10 MiB/s, with the historical two-minute floor. +_REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND = 10 * 1024 * 1024 + + +@contextlib.contextmanager +def _cross_process_repair_lock(db_path: Path): + """Serialize state.db schema surgery across processes. + + Yields True when this process holds the repair lock for *db_path*, False + when the bounded acquire timed out or the lock file could not be opened. + Unlike the kanban init lock (idempotent critical section), running surgery + unlocked IS the unsafe interleaving this prevents: a caller that gets + False must NOT do surgery. + + ``flock`` because the kernel drops it when the holder dies (a pidfile + would wedge every future repair); a forked child that inherited the fd is + the exception, so the acquire records the holder's pid + start time and + breaks the lock when that holder is provably dead (``_acquire_db_flock``). + The acquire is bounded because a *live* repairer can sit in ``VACUUM`` + for minutes, and an unbounded wait would hang the caller's open silently. + """ + from hermes_state import _IS_WINDOWS, _REPAIR_LOCK_TIMEOUT_SECONDS + lock_path = db_path.with_name(db_path.name + ".repair.lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + # Fail closed, like a timed-out acquire. An unopenable lock file means + # out of space/inodes/descriptors — and a sibling that opened ITS + # handle before the disk filled may still be inside surgery; yielding + # True here once let two processes run surgery on the same live + # state.db. Callers handle False by re-probing. + logger.warning( + "Could not open state.db repair lock %s (%s) — skipping schema " + "surgery rather than running it without cross-process authority.", + lock_path, exc, + ) + yield False + return + + acquired = False + try: + if _IS_WINDOWS: + deadline = time.monotonic() + _REPAIR_LOCK_TIMEOUT_SECONDS + while True: + try: + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + acquired = True + break + except (BlockingIOError, OSError) as exc: + if not is_advisory_lock_contention(exc): + logger.warning( + "Could not acquire state.db repair lock %s (%s) — " + "skipping schema surgery on a non-contention error.", + lock_path, exc, + ) + acquired = None + break + if time.monotonic() >= deadline: + break + time.sleep(_REPAIR_LOCK_POLL_SECONDS) + else: + acquired, handle = _acquire_db_flock( + str(lock_path), + handle, + _REPAIR_LOCK_TIMEOUT_SECONDS, + _REPAIR_LOCK_POLL_SECONDS, + "state.db repair lock", + ) + if acquired is None: + # Non-contention failure already logged with its errno. + acquired = False + elif not acquired: + record = None if _IS_WINDOWS else _read_lock_holder_record(handle) + logger.warning( + "state.db repair lock %s held by another process for more " + "than %.0fs — skipping schema surgery in this process to " + "avoid racing the repairer. Recorded holder: %s.", + lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS, + _describe_lock_holder(record), + ) + yield acquired + finally: + try: + if acquired: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + _clear_lock_holder_record(handle) + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: # pragma: no cover - best effort release + pass + finally: + handle.close() + + +def _try_acquire_auto_maintenance_lock(db_path: Path) -> Optional[Any]: + """Non-blocking cross-process lock for one auto-maintenance pass. + + Advisory lock the kernel releases if the holder exits. A caller that cannot + acquire it must skip the pass: otherwise two startups both pass the interval + check and the second prunes a row the first has only just closed recoverably. + """ + from hermes_state import _IS_WINDOWS + lock_path = db_path.with_name(db_path.name + ".auto-maintenance.lock") + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + handle = lock_path.open("a+b") + except OSError as exc: + logger.warning( + "Could not open state.db auto-maintenance lock %s (%s) — skipping " + "automatic maintenance.", + lock_path, + exc, + ) + return None + + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_NBLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except (BlockingIOError, OSError): + handle.close() + return None + return handle + + +def _release_auto_maintenance_lock(handle: Any) -> None: + """Release a handle returned by :func:`_try_acquire_auto_maintenance_lock`.""" + from hermes_state import _IS_WINDOWS + try: + if _IS_WINDOWS: + import msvcrt + + handle.seek(0) + msvcrt.locking( # type: ignore[attr-defined] + handle.fileno(), msvcrt.LK_UNLCK, 1 # type: ignore[attr-defined] + ) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except OSError: # pragma: no cover - best effort release + pass + finally: + handle.close() + + +def _bump_schema_cookie(conn: sqlite3.Connection) -> None: + """Increment the schema cookie after direct ``sqlite_master`` surgery. + + Ordinary DDL bumps this counter and other connections compare it before + running a prepared statement — that is how they discard a cached schema. + Editing ``sqlite_master`` under ``writable_schema=ON`` does NOT bump it, + so live connections elsewhere keep compiling against the old schema (e.g. + firing triggers into ``messages_fts*`` shadow tables that no longer + exist). Best-effort, never raises: a failed bump leaves the status quo. + """ + try: + current = conn.execute("PRAGMA schema_version").fetchone()[0] + # Wrap within SQLite's 32-bit signed range; peers compare for equality. + conn.execute(f"PRAGMA schema_version={(int(current) + 1) & 0x7FFFFFFF}") + except (sqlite3.DatabaseError, TypeError, IndexError) as exc: + logger.warning("Could not bump state.db schema cookie: %s", exc) + + +_MAX_PERSISTENT_REPAIR_ATTEMPTS = 3 + + +_MAX_MALFORMED_BACKUPS = 3 + + +# Sidecars copied alongside a damaged DB and pruned with it. ``-journal`` +# matters because rollback-journal (DELETE) mode — Hermes's fallback on +# NFS/SMB/FUSE/ZFS and WAL-reset-vulnerable SQLite builds — leaves a hot +# journal whenever a transaction was open; without it the forensic copy +# cannot be rolled back to a consistent state by hand. +_DB_SIDECAR_SUFFIXES = ("-wal", "-shm", "-journal") + + +# Head/tail bytes sampled by ``_db_fingerprint``: changes on any genuine +# repair/truncation/restore while staying O(1) on a multi-GB file. +_FINGERPRINT_SAMPLE_BYTES = 65536 + + +# Header ranges that move on ordinary commits rather than on repair, masked +# out of the content sample: file change counter (24-27) and version-valid-for +# (92-95). In DELETE mode a commit writes the main file directly and a +# malformed-SCHEMA DB still accepts writes, so without the mask any live write +# re-keys the ledger and the repair budget resets to 1 forever. (WAL mode +# routes commits to the -wal sidecar; masking is harmless there.) The page-1 +# sqlite_master b-tree — what repair identity depends on — sits after byte 100 +# and stays in the sample. +_FINGERPRINT_VOLATILE_HEADER_RANGES = ((24, 28), (92, 96)) + + +def _mask_volatile_header(head: bytes) -> bytes: + """Zero the commit-counter fields so ordinary writes don't re-key the ledger.""" + if len(head) < 96: + return head + buf = bytearray(head) + for start, end in _FINGERPRINT_VOLATILE_HEADER_RANGES: + buf[start:end] = b"\x00" * (end - start) + return bytes(buf) + + +# Free-space headroom for the pre-repair forensic backup: a full raw copy of +# the damaged DB plus sidecars, so a repair loop on a large state.db is a disk +# amplifier (one incident wrote ~98MB every ~10s until the volume was nearly +# full). Proportional, not a flat floor: an absolute multi-GB reserve would +# refuse backups that fit on small container/VM volumes, and since a refused +# backup is a HARD STOP that would turn "repair loops" into "repair never +# runs" there. Require the copy plus a small slice of the volume, with a +# modest floor. +_REPAIR_BACKUP_MIN_FREE_BYTES = 256 * 1024 * 1024 # 256 MiB absolute floor + + +_REPAIR_BACKUP_FREE_FRACTION = 0.02 # plus 2% of the volume + + +def _repair_backup_headroom_bytes(total_bytes: int) -> int: + """Free space required *beyond* the copy itself, for a volume of *total_bytes*.""" + return max( + _REPAIR_BACKUP_MIN_FREE_BYTES, + int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION), + ) + + +def _repair_scratch_space_error(db_path: Path) -> Optional[str]: + """Return an error unless snapshot, VACUUM and promotion can fit safely.""" + import shutil + + try: + main_bytes = db_path.stat().st_size + snapshot_bytes = main_bytes + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + snapshot_bytes += sidecar.stat().st_size + usage = shutil.disk_usage(db_path.parent) + headroom = _repair_backup_headroom_bytes(usage.total) + # Strategy 2 runs VACUUM on the staged DB, which SQLite documents may + # need up to 2x the database size in extra space; the same reserve then + # covers transactional promotion into the live DB. + required = snapshot_bytes + (2 * snapshot_bytes) + headroom + if usage.free >= required: + return None + return ( + f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; the " + f"repair snapshot needs up to {snapshot_bytes / 1e9:.2f}GB, " + f"VACUUM may need another {(2 * snapshot_bytes) / 1e9:.2f}GB, and " + f"{headroom / 1e9:.2f}GB must remain as headroom. Free disk space, " + "then retry." + ) + except OSError as exc: + return ( + f"could not determine free space on {db_path.parent} ({exc}); " + "refusing the repair snapshot rather than risk filling the volume" + ) + + +def _repair_snapshot_timeout_seconds(source_path: Path) -> float: + """Bound one SQLite snapshot by source size, including live sidecars. + + A WAL can hold committed rows not yet in the main file; count it so a + healthy large-database copy is not cut off by the repair-lock timeout. + """ + from hermes_state import _REPAIR_LOCK_TIMEOUT_SECONDS, _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND + source_bytes = 0 + for suffix in ("", *_DB_SIDECAR_SUFFIXES): + candidate = ( + source_path + if not suffix + else source_path.with_name(source_path.name + suffix) + ) + try: + source_bytes += candidate.stat().st_size + except FileNotFoundError: + continue + return max( + _REPAIR_LOCK_TIMEOUT_SECONDS, + source_bytes / _REPAIR_SNAPSHOT_MIN_THROUGHPUT_BYTES_PER_SECOND, + ) + + +def _repair_failure_consumes_attempt(exc: BaseException) -> bool: + """Whether a pre-strategy SQLite failure proves deterministic corruption. + + Lock contention, timeouts, disk-full, I/O and filesystem failures are + environmental — a retry may succeed, so they must not burn the repair + ledger. Only SQLite's corruption/image result codes prove deterministic + damage, even when SQLite cannot stage a snapshot far enough to run a + named strategy. + """ + if not isinstance(exc, sqlite3.DatabaseError): + return False + error_code = getattr(exc, "sqlite_errorcode", None) + if isinstance(error_code, int): + # Extended result codes keep the primary code in the low byte. + primary_code = error_code & 0xFF + return primary_code in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB) + + # Older sqlite3 without result-code attributes: narrow message match only, + # never turning generic "disk is full"/"readonly" into permanent failures. + message = str(exc).lower() + return ( + "file is not a database" in message + or "database disk image is malformed" in message + ) + + +def _repair_ledger_path(db_path: Path) -> Path: + return db_path.with_name(db_path.name + ".repair-attempts.json") + + +def _db_fingerprint(db_path: Path) -> "Optional[str]": + """Cheap identity for a damaged DB file: size + a bounded content sample. + + Deliberately EXCLUDES mtime: the malformed-schema class still accepts + writes, so live writers, WAL checkpoints and the strategies themselves move + mtime between passes; keyed on mtime, every pass looked like a NEW file, + the attempt counter reset to 1 forever and each pass wrote another + full-size forensic copy. Hashing a multi-GB file on every open is the cost + this ledger exists to avoid, so sample the head/tail slices any real + repair, truncation or restore necessarily changes. + + Runs under ``offline_file_access``: ``close()`` on ANY raw descriptor + cancels every POSIX advisory lock this process holds on the file, including + a peer connection's RESERVED lock (``hermes_cli.sqlite_safe_read`` rule 1), + and a live peer is the expected case here (this runs BEFORE + ``_backup_db_file``'s ``has_live_connection`` guard). Returns ``None`` + ("identity unavailable") when that makes the read unsafe. Callers MUST NOT + substitute a differently-shaped key: the ledger compares keys for equality, + so alternating shapes never matches and the unbounded loop returns; the + ledger helpers keep the recorded key instead. + """ + try: + st = db_path.stat() + try: + from hermes_cli.sqlite_safe_read import ( + LiveConnectionError, + offline_file_access, + ) + except ImportError: + # Scaffold/embed installs ship hermes_state without hermes_cli; no + # tracked connections exist there, so the raw read is safe. + @contextmanager + def offline_file_access(_path, **_kw): + yield + + class LiveConnectionError(Exception): + pass + + try: + with offline_file_access(db_path, what="fingerprint"): + with open(db_path, "rb") as fh: + head = fh.read(_FINGERPRINT_SAMPLE_BYTES) + if st.st_size > _FINGERPRINT_SAMPLE_BYTES: + fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES)) + tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) + else: + tail = b"" + except LiveConnectionError: + return None + digest = hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32] + return f"{st.st_size}:{digest}" + except OSError: + return None + + +def _backup_content_identity(db_path: Path) -> "Optional[str]": + """Recovery-image identity for forensic-backup dedupe: whole file + sidecars. + + A DIFFERENT equivalence relation from :func:`_db_fingerprint`; never + conflate them. The fingerprint answers "same repair epoch?" and masks + commit counters / samples only head+tail so an ordinary write does not mint + a fresh repair budget. A live writer can commit rows into an *interior* + page while preserving size and the first/last 64 KiB, so two materially + different recovery images share one fingerprint; reusing a backup on that + basis hands the operator a snapshot predating real user data. A forensic + copy must claim byte identity, so this digests the ENTIRE main file plus + every present sidecar (the WAL can hold uncheckpointed committed + frames). The O(n) read is cheaper than the O(n) write it avoids. Runs under + ``offline_file_access`` (same POSIX-lock reason as ``_db_fingerprint``); + ``None`` when a live connection makes the read unsafe — the caller then + takes a fresh backup, never a false reuse. + """ + try: + from hermes_cli.sqlite_safe_read import ( + LiveConnectionError, + offline_file_access, + ) + except ImportError: + @contextmanager + def offline_file_access(_path, **_kw): + yield + + class LiveConnectionError(Exception): + pass + + def _hash_whole(path: Path, hasher: "Any") -> None: + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + hasher.update(chunk) + + try: + hasher = hashlib.sha256() + with offline_file_access(db_path, what="backup-identity"): + # Length-delimit every member (main file included) so the + # concatenation is prefix-free; otherwise a main-file tail could + # coincide with a main+sidecar split and dedupe two images together. + hasher.update(f"\0main:{db_path.stat().st_size}\0".encode()) + _hash_whole(db_path, hasher) + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + hasher.update(f"\0{suffix}:{sidecar.stat().st_size}\0".encode()) + _hash_whole(sidecar, hasher) + return hasher.hexdigest() + except LiveConnectionError: + return None + except OSError: + return None + + +def _read_repair_ledger(db_path: Path) -> "Dict[str, Any]": + try: + raw = json.loads(_repair_ledger_path(db_path).read_text(encoding="utf-8")) + if isinstance(raw, dict): + return raw + except (OSError, ValueError): + pass + return {} + + +def _persistent_repair_attempts_exhausted(db_path: Path) -> bool: + """Whether *db_path* has already burned its cross-restart repair budget. + + True only when the ledger records ``_MAX_PERSISTENT_REPAIR_ATTEMPTS`` + failures against the CURRENT fingerprint. Never raises; a missing/corrupt + ledger or unstatable DB reads as "not exhausted" (the in-process claim and + cross-process lock still bound one run). When a live connection makes the + fingerprint unavailable, fall back to the SIZE the ledger recorded — + otherwise a peer connection hides an exhausted budget on every pass. + """ + ledger = _read_repair_ledger(db_path) + recorded = ledger.get("fingerprint") + fp = _db_fingerprint(db_path) + if fp is None: + # Size is the one key component that needs no raw read. + try: + size_prefix = f"{db_path.stat().st_size}:" + except OSError: + return False + if not isinstance(recorded, str) or not recorded.startswith(size_prefix): + return False + elif recorded != fp: + return False + return int(ledger.get("failed_attempts", 0)) >= _MAX_PERSISTENT_REPAIR_ATTEMPTS + + +def _persistent_repair_exhausted_error(db_path: Path) -> str: + """The stable operator-facing diagnostic for an exhausted repair budget.""" + return ( + f"automatic repair has already failed " + f"{_MAX_PERSISTENT_REPAIR_ATTEMPTS} times on this exact file — " + "the corruption is beyond the schema/FTS repair strategies " + "(likely b-tree page damage). Manual recovery required: restore " + f"a backup, or salvage with `sqlite3 {db_path} \".recover\"`. " + f"Delete {_repair_ledger_path(db_path).name} to force another " + "automatic attempt." + ) + + +def _record_repair_outcome( + db_path: Path, *, repaired: bool, fingerprint: "Optional[str]" = None +) -> None: + """Update the persistent attempt ledger after a repair pass. Never raises. + + Defaults to the post-attempt fingerprint (what the NEXT exhaustion probe + observes). When a live connection makes it unavailable, keep the recorded + key and still increment — dropping the pass would let a peer connection + reset the budget every time. Never write a differently shaped key. + """ + ledger_path = _repair_ledger_path(db_path) + try: + if repaired: + ledger_path.unlink(missing_ok=True) + return + ledger = _read_repair_ledger(db_path) + recorded = ledger.get("fingerprint") + fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path) + if fp is None: + if not isinstance(recorded, str): + # No prior key to extend and no safe way to mint one; the + # in-process claim and cross-process lock still bound this run. + return + fp = recorded + attempts = ( + int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1 + ) + import datetime + + ledger_path.write_text( + json.dumps( + { + "fingerprint": fp, + "failed_attempts": attempts, + "last_attempt": datetime.datetime.now().isoformat( + timespec="seconds" + ), + } + ), + encoding="utf-8", + ) + except Exception as exc: # pragma: no cover - best effort + logger.warning("Could not update state.db repair ledger: %s", exc) + + +def _existing_malformed_backups(db_path: Path) -> "List[Path]": + """Timestamped forensic backups of *db_path*, newest first.""" + prefix = f"{db_path.name}.malformed-backup-" + try: + found = [ + p + for p in db_path.parent.iterdir() + if p.name.startswith(prefix) + and not p.name.endswith(_DB_SIDECAR_SUFFIXES) + ] + except OSError: + return [] + return sorted(found, key=lambda p: p.name, reverse=True) + + +def _prune_malformed_backups(db_path: Path, keep: int = _MAX_MALFORMED_BACKUPS) -> None: + """Delete all but the *keep* newest forensic backups (and sidecars).""" + for stale in _existing_malformed_backups(db_path)[keep:]: + for victim in ( + stale, + *(stale.with_name(stale.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES), + ): + try: + victim.unlink(missing_ok=True) + except OSError as exc: # pragma: no cover - best effort + logger.warning("Could not prune stale DB backup %s: %s", victim, exc) + + +def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]": + """Raw-copy a (possibly malformed) DB plus sidecars to a timestamped backup. + + Raw bytes on purpose: the DB won't open cleanly, so preserve them exactly + for forensics / manual restore. Returns ``(backup_path, None)`` or + ``(None, reason)``; the repair path treats a refused backup as a HARD STOP + because the forensic bundle is the recovery path when every strategy fails. + Refuses while a connection to this DB is live in the process: reading the + file would ``close()`` a descriptor and cancel that connection's POSIX + advisory locks (see ``hermes_cli.sqlite_safe_read``) — a real case, since + one SessionDB can enter repair while the gateway holds others. + """ + import datetime + import shutil + + try: + from hermes_cli.sqlite_safe_read import has_live_connection + except ImportError: + has_live_connection = None # type: ignore[assignment] + + if has_live_connection is not None and has_live_connection(db_path): + reason = ( + f"a connection to {db_path} is still open in this process; " + "raw-copying it would cancel that connection's POSIX advisory " + "locks. Close all SessionDB handles first." + ) + logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason) + return None, reason + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}") + # Same-second collision (two damaged states within one second) must not + # overwrite the earlier forensic copy. + seq = 1 + while backup_path.exists(): + backup_path = db_path.with_name( + f"{db_path.name}.malformed-backup-{stamp}_{seq}" + ) + seq += 1 + try: + # Sweep staging debris from an earlier interrupted pass BEFORE the + # dedupe: leftover staging is byte-identical to the damaged DB, so + # dedupe would hand it back as a legitimate backup. Also sweeps the + # pre-merge ``.incomplete`` spelling, which prefix-matches as a backup, + # sorts NEWEST and would otherwise survive prune forever. + for pattern in ( + f"{db_path.name}.backup-staging-*", + f"{db_path.name}.malformed-backup-*.incomplete*", + ): + for old in db_path.parent.glob(pattern): + try: + old.unlink(missing_ok=True) + except OSError: # pragma: no cover - best effort + pass + # Dedupe: a repair loop used to copy the SAME damaged bytes on every + # restart (~900MB a pass, 89GB over 11 days). If the newest existing + # backup is byte-identical to the current recovery image, reuse it. + # Match on ``_backup_content_identity`` — NOT mtime (the malformed- + # SCHEMA class still accepts writes, so mtime missed every pass) and + # NOT ``_db_fingerprint`` (see its docstring: an interior-page write + # changes the recovery image without changing the fingerprint). + try: + # Only hash the source when there is a candidate to compare + # against; hashing a multi-GB source right before copying it is + # pure waste on the common first-corruption pass. + existing_backups = _existing_malformed_backups(db_path)[:1] + if existing_backups: + src_id = _backup_content_identity(db_path) + for existing in existing_backups: + if src_id is not None and _backup_content_identity(existing) == src_id: + logger.info( + "Reusing existing forensic backup %s (identical to the " + "damaged DB).", existing, + ) + return existing, None + except OSError: + pass + # Disk guard: a full raw copy on a nearly-full volume (which a + # preceding repair loop may itself have caused) can finish off the + # disk and every process on the machine. Refuse while there is room to. + try: + need = db_path.stat().st_size + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + need += sidecar.stat().st_size + usage = shutil.disk_usage(db_path.parent) + headroom = _repair_backup_headroom_bytes(usage.total) + if usage.free - need < headroom: + reason = ( + f"only {usage.free / 1e9:.2f}GB free on {db_path.parent}; " + f"copying the damaged DB needs {need / 1e9:.2f}GB and must " + f"leave {headroom / 1e9:.2f}GB headroom. Free disk space, " + f"then retry (or recover manually with `sqlite3 {db_path} " + '".recover"`).' + ) + logger.error("Refusing forensic backup of %s: %s", db_path, reason) + return None, reason + except OSError as exc: + # Fail CLOSED: the nearly-full volume this guard exists for is + # exactly where stat()/disk_usage() is most likely to fail, and + # proceeding would take the copy that finishes off the disk. Repair + # then waits (HARD STOP) for a human to free space — the safe side. + reason = ( + f"could not determine free space on {db_path.parent} ({exc}); " + "refusing the forensic copy rather than risk filling the " + f"volume. Free disk space, then retry (or recover manually " + f'with `sqlite3 {db_path} ".recover"`).' + ) + logger.error("Refusing forensic backup of %s: %s", db_path, reason) + return None, reason + # Copy to a staging name OUTSIDE the ``.malformed-backup-`` prefix and + # rename into place only once every copy succeeded. A staging name + # inside the prefix (e.g. ``…-.incomplete``) counts as a backup, + # sorts NEWEST (so prune kept partials and deleted intact copies), and + # dedupe could return it as ``backup_path`` — passing the hard stop with + # no real forensic copy on disk. + staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}") + # (staging_src, final_dst) pairs. PUBLICATION ORDER MATTERS: the main + # DB name is the bundle's commit marker (what + # ``_existing_malformed_backups`` counts), so sidecars go FIRST and the + # main DB LAST; a failure partway then never leaves a countable main + # backup over a missing sidecar that would pass the hard stop and dedupe. + staged_sidecars: "List[Tuple[Path, Path, Path]]" = [] + for suffix in _DB_SIDECAR_SUFFIXES: + sidecar = db_path.with_name(db_path.name + suffix) + if sidecar.exists(): + side_staging = staging.with_name(staging.name + suffix) + side_dst = backup_path.with_name(backup_path.name + suffix) + staged_sidecars.append((sidecar, side_staging, side_dst)) + main_pair = (staging, backup_path) + published: "List[Path]" = [] + all_staging_srcs = [staging] + [s for _src, s, _d in staged_sidecars] + try: + shutil.copy2(db_path, staging) + for sidecar, side_staging, _side_dst in staged_sidecars: + shutil.copy2(sidecar, side_staging) + publish_order = [ + (s, d) for _src, s, d in staged_sidecars + ] + [main_pair] + for src, dst in publish_order: + os.replace(src, dst) + published.append(dst) + except Exception: + # Roll back unpublished staging files AND anything already promoted, + # so a failure after the main os.replace leaves no official backup_path. + for src in all_staging_srcs: + try: + src.unlink(missing_ok=True) + except OSError: + pass + for dst in published: + try: + dst.unlink(missing_ok=True) + except OSError: + pass + try: + staging.unlink(missing_ok=True) + except OSError: + pass + raise + _prune_malformed_backups(db_path) + return backup_path, None + except Exception as exc: # pragma: no cover - best effort + logger.warning("Could not back up malformed DB %s: %s", db_path, exc) + return None, f"backup copy failed: {exc}" + + +def preflight_db_writability( + db_path: Path, + *, + db_label: str = "state.db", +) -> None: + """Refuse-or-repair read-only DB files BEFORE the first connection opens. + + A stray read-only ``state.db`` / ``-wal`` / ``-shm`` (sudo run, restored + backup, copied dotfiles) otherwise surfaces as an opaque "attempt to write + a readonly database" deep inside ``_init_schema``, and the obvious wrong + "fix" (deleting the ``-wal``) silently loses committed transactions. + Repairs with ``chmod u+rw`` only inside the Hermes home tree (Hermes owns + those files, and ``chmod`` fails on files the user doesn't own, which + bounds the repair exactly); otherwise fails fast naming the exact file and + ``chmod`` command. Never deletes or truncates a WAL sidecar — once + writable, the normal open path checkpoints its committed frames. + ``:memory:`` and ``file:`` URIs are skipped. Shared with ``kanban_db``. + """ + raw = str(db_path) + if raw == ":memory:" or raw.startswith("file:"): + return + + try: + home: Optional[Path] = Path(get_hermes_home()).resolve() + except Exception: # pragma: no cover - defensive + home = None + + def _in_repair_scope(p: Path) -> bool: + if home is None: + return False + try: + return p.resolve().is_relative_to(home) + except (OSError, ValueError): + return False + + def _ensure_writable(p: Path, *, is_dir: bool = False) -> None: + import stat as _stat + + if os.access(p, os.R_OK | os.W_OK): + return + if _in_repair_scope(p): + try: + add = _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0) + os.chmod(p, p.stat().st_mode | add) + except OSError: + pass + if os.access(p, os.R_OK | os.W_OK): + logger.info( + "%s preflight: repaired read-only %s (chmod u+rw%s)", + db_label, + p, + "x" if is_dir else "", + ) + return + kind = "directory" if is_dir else "file" + wal_note = ( + " Do NOT delete the -wal file — it contains committed data that " + "will be merged into the database once it is writable." + if p.name.endswith("-wal") + else "" + ) + raise sqlite3.OperationalError( + f"{db_label} is not writable: {kind} {p} is read-only for this " + f"user. Hermes needs read-write access to open the database. " + f"Fix with: chmod u+rw{'x' if is_dir else ''} '{p}'" + f" (files owned by another user may need sudo/chown).{wal_note}" + ) + + parent = db_path.parent + if parent.is_dir(): + # SQLite needs a writable directory in every journal mode (WAL/SHM + # sidecars, or the rollback journal in DELETE mode). + _ensure_writable(parent, is_dir=True) + + for suffix in ("", "-wal", "-shm"): + p = db_path.with_name(db_path.name + suffix) if suffix else db_path + if p.is_file(): + _ensure_writable(p) + + +def _connect_repair_durable( + db_path: Path, *, timeout: float = 5.0 +) -> sqlite3.Connection: + """``sqlite3.connect`` for the repair/probe paths, with macOS write barriers. + + These paths open ``state.db`` directly (not via ``SessionDB`` / + :func:`apply_wal_with_fallback`), so they inherited ``synchronous=NORMAL`` + and no ``checkpoint_fullfsync`` — on Darwin, where ``fsync()`` guarantees + neither data-on-platter nor ordering, an interrupted rewrite leaves + half-written b-tree pages, and ``REINDEX``/``VACUUM``/``writable_schema`` + surgery rewrite nearly every page. Autocommit (``isolation_level=None``) + is preserved: DDL and ``VACUUM`` are illegal inside an implicit + transaction. Barriers are best-effort by necessity: SQLite loads the schema + before any statement, so on a malformed schema even ``PRAGMA + synchronous=FULL`` raises — and a malformed DB is this helper's input. + Whole-file rewrites call :func:`_reapply_durability_barriers` once the + schema parses again. + """ + conn = sqlite3.connect(str(db_path), timeout=timeout, isolation_level=None) + _reapply_durability_barriers(conn) + return conn + + +def _reapply_durability_barriers(conn: sqlite3.Connection) -> bool: + """Best-effort (re)application of the macOS write barriers. Never raises. + + True when the pragmas were accepted. Call before ``VACUUM``/``REINDEX`` + once the schema parses: a connection opened on a malformed schema could + not take them at open time. + """ + from hermes_state import _apply_macos_checkpoint_barrier, _enforce_macos_synchronous_full + try: + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return True + except sqlite3.DatabaseError: + # Schema still unparseable — pragmas cannot be set yet. + return False + except Exception: + return False + + +def apply_durability_barriers(conn: sqlite3.Connection) -> bool: + """Apply state-store durability barriers without changing journal mode. + + Public entry point for secondary users of ``state.db`` that must inherit + its owner's journal mode. Also applies the configured + ``database.synchronous`` level, a per-connection pragma that otherwise + only rides on the journal-mode setup path guests must not run. + """ + from hermes_state import _apply_synchronous_pragma + ok = _reapply_durability_barriers(conn) + try: + # Local import: avoids a circular import with hermes_cli.config. + from hermes_cli.config import cfg_get, load_config_readonly + + cfg = load_config_readonly() + raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) + if raw_synchronous is not None: + _apply_synchronous_pragma( + conn, raw_synchronous, db_label="state.db (guest)" + ) + except Exception: + pass + return ok + + +@contextmanager +def _exclusive_repair_db_guard(db_path: Path): + """Yield one live connection that excludes writers for repair surgery. + + ``locking_mode=EXCLUSIVE`` retains file-level exclusion after the short + ``BEGIN EXCLUSIVE`` is rolled back. The rollback is essential: + ``Connection.backup`` uses this connection as *source* and later as the + promotion *destination*, both of which require it transaction-free. It + stays open across the whole snapshot -> strategies -> promotion window, so + no other writer can commit a change promotion would overwrite. Existing + readers make acquisition fail rather than being disturbed: repair fails + closed unless this process owns the whole window. + """ + guard: Optional[sqlite3.Connection] = None + try: + # The cross-process repair lock already serializes repairers. Do not + # wait behind an ordinary application connection: a partial repair is + # less safe than an explicit "stop the gateway and retry". + guard = _connect_repair_durable(db_path, timeout=0.0) + guard.execute("PRAGMA locking_mode=EXCLUSIVE") + guard.execute("BEGIN EXCLUSIVE") + guard.execute("ROLLBACK") + except (sqlite3.Error, OSError) as exc: + if guard is not None: + try: + guard.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + guard.close() + yield None, exc + return + + try: + yield guard, None + finally: + try: + # Release the exclusive locks before close; also keeps a close-time + # checkpoint from being mistaken for a repair write by callers that + # immediately reopen state.db. + guard.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + guard.close() + + +def _copy_database_snapshot( + source_path: Path, + destination_path: Path, + *, + source_connection: Optional[sqlite3.Connection] = None, + destination_connection: Optional[sqlite3.Connection] = None, +) -> None: + """Copy one complete SQLite snapshot without replacing either file inode. + + The online backup API folds committed WAL frames into the source snapshot + and writes the destination in one transaction (rolled back if interrupted), + so ``state.db`` is never swapped out from under handles that refer to it. + """ + # Compute the deadline before opening an owned source connection: a + # sidecar vanishing mid-stat must not leak a just-opened descriptor. + deadline_seconds = _repair_snapshot_timeout_seconds(source_path) + deadline = time.monotonic() + deadline_seconds + source = source_connection or _connect_repair_durable(source_path) + destination = destination_connection + own_source = source_connection is None + own_destination = destination_connection is None + + def _check_deadline(_status: int, _remaining: int, _total: int) -> None: + if time.monotonic() >= deadline: + raise TimeoutError( + "timed out copying SQLite repair snapshot after " + f"{deadline_seconds:.0f}s" + ) + + try: + if destination is None: + destination = _connect_repair_durable(destination_path) + elif destination.in_transaction: + # sqlite3_backup needs a transaction-free destination; the exclusive + # guard retains exclusion via locking_mode, not a transaction. + raise sqlite3.ProgrammingError( + "SQLite repair backup destination has an active transaction" + ) + source.backup( + destination, + pages=256, + progress=_check_deadline, + sleep=_REPAIR_LOCK_POLL_SECONDS, + ) + finally: + if own_destination and destination is not None: + destination.close() + if own_source: + source.close() + + +def _db_opens_cleanly(db_path: Path) -> Optional[str]: + """Probe a DB on a fresh connection. Returns None if healthy, else a reason. + + Runs the first statement that trips the malformed-schema parse (``PRAGMA + journal_mode``), ``integrity_check``, a ``sessions`` read, FTS5 MATCH + probes, and a rolled-back ``messages`` write — so FTS5 index corruption, + which leaves reads and ``integrity_check`` passing while every ``INSERT + INTO messages`` fails through the FTS triggers, is reported as unhealthy. + """ + from hermes_state import SessionDB, load_fts5_cjk_extension + conn = _connect_repair_durable(db_path) + try: + # Best-effort tokenizer load: messages_fts_cjk needs cjk_unicode61 + # before any statement (incl. the trigger-driven write probe) can touch + # it. Without it this probe sees the DB as a tokenizer-less SessionDB + # would (which drops the cjk triggers), so tokenizer absence must never + # classify as corruption. + load_fts5_cjk_extension(conn) + conn.execute("PRAGMA journal_mode").fetchone() + rows = conn.execute("PRAGMA integrity_check").fetchall() + problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"] + if problems: + return "; ".join(problems[:3]) + conn.execute("SELECT COUNT(*) FROM sessions").fetchone() + + # FTS5 read probe. The write probe below misses partial shadow-table + # corruption where MATCH / snippet / rank raise DatabaseError("database + # disk image is malformed"), silently breaking session_search and + # /resume title resolution while check-only reports healthy. + for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"): + try: + # Trigram backs title resolution, so probe it too. MATCH '""' + # (empty phrase) parses, scans zero rows and exercises the + # shadow-table read path; FTS5 rejects MATCH '' outright. + conn.execute( + f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1" + ).fetchone() + except sqlite3.OperationalError as exc: + # Canonical capability classifier: on builds without fts5 a + # legacy messages_fts table may exist and MATCH raises "no such + # module: fts5"; treating that as corruption would send the DB + # into repair, whose final fallback deletes the messages_fts% + # schema. Covers "no such tokenizer: trigram" too. + if SessionDB._is_fts5_unavailable_error(exc): + continue + msg = str(exc).lower() + if "no such table" in msg or "no such column" in msg: + # FTS5 not built yet (brand new file mid-init). + continue + return f"fts5 read probe failed on {fts_table}: {exc}" + except sqlite3.DatabaseError as exc: + # Partial shadow-table damage: MATCH raises though the table parses. + return f"fts5 read probe failed on {fts_table}: {exc}" + + # FTS write probe: drive a row through the messages_fts* triggers in a + # transaction that is always rolled back. Missing messages/sessions + # tables (brand new file mid-init) mean "not yet populated", not corruption. + probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}" + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute( + "INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)", + (probe_session_id, "_health_probe", time.time()), + ) + conn.execute( + "INSERT INTO messages (session_id, role, content, timestamp) " + "VALUES (?, ?, ?, ?)", + (probe_session_id, "user", "_fts_health_probe", time.time()), + ) + conn.execute("ROLLBACK") + except sqlite3.OperationalError as exc: + # Missing tables / FTS disabled — not the corruption class we probe. + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + msg = str(exc).lower() + if "no such table" in msg or "no such column" in msg: + return None + if "no such tokenizer: cjk_unicode61" in msg: + # This process couldn't load the cjk extension while the DB + # carries the cjk index — capability gap, not corruption. A + # tokenizer-less SessionDB self-heals by dropping the triggers. + return None + return str(exc) + return None + except sqlite3.DatabaseError as exc: + return str(exc) + finally: + conn.close() + + +def _live_writer_holds_db(db_path: Path) -> bool: + """True when a connection outside this call still holds ``db_path`` open. + + Asks SQLite for what a repair needs and a live holder cannot grant: + ``PRAGMA locking_mode=EXCLUSIVE`` then ``BEGIN IMMEDIATE``. In WAL mode + that needs exclusive locks on the WAL index, so any other open connection + fails it with SQLITE_BUSY; neither statement parses the schema, so it + works on malformed DBs. Fails **open** (False) on anything but a positive + busy/locked signal — refusing to repair a DB nobody holds would strand the + self-heal path. + + Scope: WAL mode only. In ``journal_mode=DELETE`` (Hermes's fallback on + WAL-reset-vulnerable builds and NFS/SMB) a held reader takes only SHARED + and this returns False; repair is then serialised only by the cross-process + repairer lock. Broadening to DELETE mode is a follow-up. + """ + probe = None + try: + probe = _connect_repair_durable(db_path, timeout=0.0) + probe.execute("PRAGMA locking_mode=EXCLUSIVE") + probe.execute("BEGIN IMMEDIATE") + probe.execute("ROLLBACK") + return False + except sqlite3.OperationalError as exc: + lowered = str(exc).lower() + return "locked" in lowered or "busy" in lowered + except sqlite3.DatabaseError: + # Malformed/unreadable: no evidence of a live holder either way. + return False + except Exception: + return False + finally: + if probe is not None: + try: + # Drop exclusive mode before close so the probe never leaves + # the file pinned. + probe.execute("PRAGMA locking_mode=NORMAL") + except Exception: + pass + try: + probe.close() + except Exception: + pass + + +def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, Any]: + """Repair a state.db whose ``sqlite_master`` is malformed or whose FTS + indexes reject writes. + + Two corruption classes: malformed schema / "duplicate object definition" + (even ``PRAGMA`` fails), and FTS write-corruption (base tables read fine, + ``integrity_check`` passes, writes fail through ``messages_fts*`` + triggers). Least-destructive first: (1) rebuild FTS in place via FTS5 + ``'rebuild'``; (2) de-duplicate ``sqlite_master`` (lowest rowid per + ``type``/``name``), FTS preserved; (3) drop the FTS schema + ``VACUUM``, + rebuilt on the next ``SessionDB()`` open. Canonical rows are never + modified by a failed attempt: strategies run on a complete SQLite snapshot + and a successful result is copied back transactionally. A raw backup is + taken first unless ``backup=False``. Surgery is serialised across + processes (:func:`_cross_process_repair_lock`): the gateway, Desktop + backend and CLI all open the same file, and concurrent ``writable_schema`` + surgery is itself a corruption source. + + Returns ``{repaired: bool, strategy: str|None, backup_path: str|None, + error: str|None}``. + """ + from hermes_state import _cross_process_repair_lock, _db_opens_cleanly, _live_writer_holds_db, _persistent_repair_attempts_exhausted, _probe_journal_mode_for_repair, _record_repair_outcome, _repair_state_db_schema_locked + report: Dict[str, Any] = { + "repaired": False, + "strategy": None, + "backup_path": None, + "error": None, + } + + # Startup-watchdog progress lease: repair is I/O-bound (near-zero CPU), + # which the watchdog's CPU fallback would misread as a parked deadlock. A + # single lease (clamped to _MAX_LEASE_S=900) is deliberate: up to that much + # zombie time on a wedged repair beats per-chunk renewal complexity. + report_startup_progress(900.0, phase="state_db_repair") + + db_path = Path(db_path) + if not db_path.exists(): + report["error"] = f"{db_path} does not exist" + return report + + # Cross-restart attempt cap: the in-memory claim bounds one process, but a + # class the strategies cannot heal (b-tree page damage) used to re-run the + # whole surgery, with a fresh forensic backup, on EVERY restart. After + # _MAX_PERSISTENT_REPAIR_ATTEMPTS failures on the same file, stop. + if _persistent_repair_attempts_exhausted(db_path): + report["error"] = _persistent_repair_exhausted_error(db_path) + logger.error("state.db repair skipped: %s", report["error"]) + return report + + result = report + with _cross_process_repair_lock(db_path) as holding_lock: + if not holding_lock: + # Another process is inside its critical section, or the lock file + # could not be opened. It may have healed the file already (long + # VACUUM after a successful strategy), so re-probe before failing. + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "repaired_by_other_process" + else: + report["error"] = ( + "could not obtain the state.db repair lock (held by " + "another process, or the lock file was unopenable); " + "skipped schema surgery to avoid racing a concurrent " + "repairer" + ) + else: + # Recheck exhaustion after acquisition: a queued repairer can have + # recorded the final failure while this process waited, and this + # process must not start a fourth attempt. + if _persistent_repair_attempts_exhausted(db_path): + report["error"] = _persistent_repair_exhausted_error(db_path) + logger.error("state.db repair skipped: %s", report["error"]) + # WAL-holder preflight: fail-closed for active readers before a + # forensic backup is taken. Not the race defence — the exclusive + # guard in the locked routine excludes writers through promotion and + # rejects DELETE-mode readers this probe cannot see. + elif _live_writer_holds_db(db_path): + report["error"] = ( + "a live writer still holds state.db; skipped schema surgery " + "to avoid tearing b-tree pages under a concurrent writer. " + "Stop the gateway (hermes gateway stop) and retry." + ) + logger.error("state.db repair skipped: %s", report["error"]) + else: + # Probe the journal mode BEFORE surgery: a rebuilt file comes + # back in the default (delete) mode and nothing else records + # the flip (see _restore_journal_mode_after_repair). The probe + # may fail on a damaged file; then database.journal_mode is + # the restore target. + before_mode = _probe_journal_mode_for_repair(db_path) + result = _repair_state_db_schema_locked( + db_path, backup=backup, report=report + ) + if result.get("repaired"): + result["journal_mode_before"] = before_mode + _restore_journal_mode_after_repair(db_path, before_mode) + # Environmental aborts happen before a strategy mutates the + # snapshot; they are retriable, not proof a strategy was exhausted. + # Keep that private marker out of the public report. The ledger + # update stays under the same cross-process lock as surgery so two + # repairers cannot lose each other's updates; a queued loser must + # not record at all. + attempted = bool(result.pop("_repair_attempted", False)) + if attempted or result.get("repaired"): + _record_repair_outcome( + db_path, repaired=bool(result.get("repaired")) + ) + return result + + +def _probe_journal_mode_for_repair(db_path: Path) -> Optional[str]: + """Best-effort journal-mode probe for a (possibly malformed) DB file. + + Returns ``wal``/``delete``, or ``None`` when the file cannot be opened or + probed (malformed header, concurrent opener's locks — both expected on + the repair path); callers then fall back to ``database.journal_mode``. + """ + from hermes_state import _on_disk_journal_mode + try: + conn = _connect_repair_durable(db_path) + try: + return _on_disk_journal_mode(conn) + finally: + conn.close() + except (sqlite3.Error, OSError): + return None + + +def _restore_journal_mode_after_repair(db_path: Path, before_mode: Optional[str]) -> None: + """Re-apply the journal mode after schema surgery. + + A rebuilt SQLite file comes back in the default (delete) mode; without + this, a corruption event silently moves a WAL store out of WAL (the + open-time WAL-reset gate never sees a flip made inside repair). Routed + through :func:`apply_wal_with_fallback`, not a direct pragma, so it + inherits the vulnerable-SQLite WAL-reset gate (a rebuilt file IS a new + database; on a vulnerable runtime the gate deliberately keeps DELETE, and + "could not reach WAL" is expected there), the macOS-NFS silent-refusal + handling, and the WAL companions (size limit, checkpoint barrier, + synchronous=FULL). ``before_mode`` (None if unprobeable) is only for the + log comparison; the target comes from ``database.journal_mode``. + Best-effort: the repair already succeeded, so failures log at WARNING. + """ + from hermes_state import apply_wal_with_fallback + try: + conn = _connect_repair_durable(db_path) + try: + after = apply_wal_with_fallback(conn, db_label=db_path.name) + finally: + conn.close() + if before_mode and after != before_mode: + logger.warning( + "state.db repair changed journal_mode %r -> %r " + "(pre-surgery probe %r; restore resolved through " + "apply_wal_with_fallback per database.journal_mode and the " + "WAL-reset gate)", + before_mode, after, before_mode, + ) + except (sqlite3.Error, OSError) as exc: + logger.warning( + "state.db repair at %s: post-surgery journal-mode restore " + "failed (%s); verify with PRAGMA journal_mode on the next open", + db_path, exc, + ) + + +def _repair_state_db_schema_locked( + db_path: Path, *, backup: bool, report: Dict[str, Any] +) -> Dict[str, Any]: + """Repair strategies for :func:`repair_state_db_schema`. + + Caller must hold the cross-process repair lock for *db_path*. Strategies + run on a SCRATCH COPY; the result is copied back through SQLite's + transactional backup API only once proven to open cleanly, so a failed + repair cannot modify or lose committed canonical data. (A WAL checkpoint + of already-committed frames on guard release is not a repair mutation.) + + WHY not in place: Strategy 2 ends in ``VACUUM``, which rebuilds the file + from the schema SQLite can still parse. When the damage IS in the schema + b-tree — the ``malformed database schema ()`` class handled here — every + table hanging off the unreadable part is silently dropped, the probe then + correctly reports STILL malformed, and repair returned ``repaired=False`` + having destroyed what it was asked to save. The forensic backup does not + close this (nothing reads it back). Not mutating the original is the + property that holds without a human in the loop. + """ + from hermes_state import _backup_db_file, _copy_database_snapshot, _db_opens_cleanly, _repair_scratch_space_error, _run_repair_strategies, _unlink_db_triple + scratch = db_path.with_name(f"{db_path.name}.repair-scratch") + cleanup_error = _unlink_db_triple(scratch) + if cleanup_error is not None: + report["error"] = ( + "could not remove a stale repair snapshot before probing state.db: " + f"{cleanup_error}" + ) + logger.error("state.db repair aborted: %s", report["error"]) + return report + + # Re-probe under the lock: a process we queued behind may have just + # repaired the file; redoing surgery would undo its work (the + # repair/re-corrupt cascade this lock exists to break). + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "already_healthy" + return report + + if backup: + bpath, backup_error = _backup_db_file(db_path) + report["backup_path"] = str(bpath) if bpath else None + if bpath is None: + # HARD STOP: the forensic image is still required when corruption + # defeats every strategy, even though strategies run on a snapshot. + report["error"] = ( + "pre-repair backup refused; aborting schema repair to avoid " + f"mutating the only copy of the damaged DB: {backup_error}" + ) + logger.error("state.db repair aborted: %s", report["error"]) + return report + + # The forensic copy deliberately precedes this guard: its raw-copy safety + # checks inspect real live holders and would be poisoned by our exclusive + # connection. Everything affecting the repair image or live promotion + # happens only after writer exclusion is held. + with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error): + if live_guard is None: + report["error"] = ( + "could not acquire exclusive state.db repair ownership; " + "skipped schema surgery to avoid overwriting a concurrent " + f"writer. Stop the gateway and retry: {guard_error}" + ) + if guard_error is not None and _repair_failure_consumes_attempt( + guard_error + ): + report["_repair_attempted"] = True + logger.error("state.db repair skipped: %s", report["error"]) + return report + + space_error = _repair_scratch_space_error(db_path) + if space_error is not None: + report["error"] = space_error + logger.error("state.db repair aborted: %s", report["error"]) + return report + + try: + # Reuse live_guard rather than a second source connection: the + # guard owns the exclusion, and a second connection could be + # blocked by our own EXCLUSIVE lock on some SQLite builds. + _copy_database_snapshot( + db_path, scratch, source_connection=live_guard + ) + except (OSError, sqlite3.Error, TimeoutError) as exc: + report["error"] = ( + f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}" + ) + if _repair_failure_consumes_attempt(exc): + report["_repair_attempted"] = True + logger.error("state.db repair aborted: %s", report["error"]) + _unlink_db_triple(scratch) + return report + + try: + # Private marker consumed by the outer wrapper: a strategy failure + # consumes the persistent budget, but a later promotion failure is + # classified separately (disk/I/O/permission/lock = environmental). + report["_repair_attempted"] = True + _run_repair_strategies(scratch, report) + if report.get("repaired"): + try: + # Do not os.replace the live DB: Windows rejects replacement + # under open handles and POSIX would leave those handles on + # the old inode. The guard that staged the live image + # receives the promotion, keeping writer exclusion throughout. + _copy_database_snapshot( + scratch, + db_path, + destination_connection=live_guard, + ) + except (OSError, sqlite3.Error, TimeoutError) as exc: + report["repaired"] = False + report["strategy"] = None + report["_repair_attempted"] = _repair_failure_consumes_attempt( + exc + ) + report["error"] = ( + "repaired snapshot could not be promoted transactionally: " + f"{exc}" + ) + logger.error("state.db repair promotion failed: %s", exc) + else: + logger.warning( + "state.db repaired via '%s' and promoted transactionally: %s", + report.get("strategy"), + db_path, + ) + if not report.get("repaired"): + # Logged HERE, not in the strategies: they see the scratch copy, + # and the one message a human acts on must not name a path + # that no longer exists by the time they read it. + logger.error( + "state.db schema repair could not recover %s automatically " + "(no committed canonical data was modified or lost; backup: %s); " + "manual restore from backup may be required.", + db_path, + report["backup_path"], + ) + return report + finally: + # Never leave a half-repaired file beside the DB for a later probe + # or human to mistake for the real thing. + cleanup_error = _unlink_db_triple(scratch) + if cleanup_error is not None: + logger.warning( + "Could not remove state.db repair snapshot after repair: %s", + cleanup_error, + ) + + +def _unlink_db_triple(path: Path) -> Optional[str]: + """Remove *path* and every SQLite sidecar; return any cleanup failure.""" + from hermes_state import _IS_WINDOWS + failures: List[str] = [] + for suffix in ("", *_DB_SIDECAR_SUFFIXES): + victim = path if not suffix else path.with_name(path.name + suffix) + for attempt in range(10): + try: + victim.unlink() + break + except FileNotFoundError: + break + except PermissionError as exc: + # Windows may retain a just-closed SQLite handle for a few + # scheduler ticks; bounded retry. A later open still fails + # safely if the handle truly remains live. + if _IS_WINDOWS and attempt < 9: + time.sleep(0.05) + continue + failures.append(f"{victim}: {exc}") + break + except OSError as exc: + failures.append(f"{victim}: {exc}") + break + return "; ".join(failures) or None + + +def _run_repair_strategies( + db_path: Path, report: Dict[str, Any] +) -> Dict[str, Any]: + """Escalating repair attempts, applied to *db_path* IN PLACE. + + Every strategy mutates its argument, so this is only ever called by + :func:`_repair_state_db_schema_locked` on a scratch copy nothing else + holds open — never on the user's database. + """ + from hermes_state import _db_opens_cleanly, load_fts5_cjk_extension + # ── Strategy 0: rebuild FTS indexes in place (FTS write-corruption) ── + # FTS5 'rebuild' rewrites the index from the content table: the + # least-destructive fix for an index that rejects writes while reads work. + try: + conn = _connect_repair_durable(db_path) + try: + # The cjk index can only be rebuilt with its tokenizer loaded + # (best-effort; a tokenizer-less host skips it below). + load_fts5_cjk_extension(conn) + for table_name in ( + "messages_fts", "messages_fts_trigram", "messages_fts_cjk" + ): + try: + conn.execute( + f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')" + ) + except sqlite3.OperationalError: + # Table absent (FTS disabled / trigram off / cjk not present + # or tokenizer unavailable). + continue + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "rebuild_fts" + logger.warning( + "state.db FTS indexes rebuilt in place (schema preserved): %s", + db_path, + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db FTS in-place rebuild pass failed: %s", exc) + + # ── Strategy 0.5: rebuild stale B-tree indexes ── + # integrity_check reports "wrong # of entries in index" when a B-tree index + # drifts from its base table; REINDEX rewrites it from the canonical rows. + try: + conn = _connect_repair_durable(db_path) + try: + # REINDEX rewrites every index b-tree; take the barriers now that + # the schema parses, in case the open-time attempt was refused. + _reapply_durability_barriers(conn) + conn.execute("REINDEX") + conn.commit() + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "reindex_btree" + logger.warning( + "state.db B-tree indexes rebuilt via REINDEX: %s", db_path + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db REINDEX pass failed: %s", exc) + + # ── Strategy 1: de-duplicate sqlite_master (keeps FTS index) ── + try: + conn = _connect_repair_durable(db_path) + try: + conn.execute("PRAGMA writable_schema=ON") + dupes = conn.execute( + "SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep " + "FROM sqlite_master GROUP BY type, name HAVING c > 1" + ).fetchall() + for type_, name, _count, keep in dupes: + conn.execute( + "DELETE FROM sqlite_master " + "WHERE type IS ? AND name IS ? AND rowid <> ?", + (type_, name, keep), + ) + if dupes: + _bump_schema_cookie(conn) + conn.execute("PRAGMA writable_schema=OFF") + conn.commit() + finally: + conn.close() + if _db_opens_cleanly(db_path) is None: + report["repaired"] = True + report["strategy"] = "dedup_schema" + logger.warning( + "state.db schema repaired by de-duplicating sqlite_master " + "(FTS index preserved): %s", db_path + ) + return report + except sqlite3.DatabaseError as exc: + logger.warning("state.db dedup repair pass failed: %s", exc) + + # ── Strategy 2: drop all FTS schema, VACUUM, rebuild on next open ── + # The destructive one, and why this path runs on a scratch copy: on a + # damaged schema b-tree VACUUM silently drops every table hanging off the + # unreadable part (see _repair_state_db_schema_locked). + try: + conn = _connect_repair_durable(db_path) + try: + conn.execute("PRAGMA writable_schema=ON") + conn.execute("DELETE FROM sqlite_master WHERE name LIKE 'messages_fts%'") + _bump_schema_cookie(conn) + conn.execute("PRAGMA writable_schema=OFF") + conn.commit() + # The schema parses now, so the barriers can finally stick — and + # VACUUM rewrites the entire file, the worst operation to lose halfway. + _reapply_durability_barriers(conn) + conn.execute("VACUUM") + finally: + conn.close() + reason = _db_opens_cleanly(db_path) + if reason is None: + report["repaired"] = True + report["strategy"] = "drop_fts_rebuild" + logger.warning( + "state.db schema repaired by dropping FTS schema; indexes " + "will rebuild from messages on next open: %s", db_path + ) + return report + report["error"] = reason + except sqlite3.DatabaseError as exc: + report["error"] = str(exc) + + # The "could not recover" log lives in the caller: it must name the user's + # database, not the scratch copy. + return report diff --git a/hermes_state_schema.py b/hermes_state_schema.py index 01801a4870..0130fd434f 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -1,20 +1,21 @@ """Schema creation, column reconciliation, and FTS DDL management for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import datetime +import hashlib import logging import json +import os import sqlite3 +import tempfile import time import uuid -from typing import Dict, Optional, Sequence +from typing import Dict, List, Optional, Sequence from hermes_constants import get_hermes_home @@ -37,36 +38,29 @@ from hermes_state_common import ( fts_rebuild_admission, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") _FTS_HOLDER_ESCALATE_ATTEMPTS = 3 _FTS_HOLDER_ESCALATE_SECONDS = 60.0 -# Minimum spacing between in-process retries of a deferred stale-FTS rebuild -# (``retry_deferred_fts_recovery``). The startup open already paid the full -# admission wait once; later retries are non-blocking probes on this cadence -# so a live holder never stalls a long-lived writer. +# In-process retry cadence for a deferred stale-FTS rebuild +# (``retry_deferred_fts_recovery``): startup paid the full admission wait once; +# later retries are non-blocking probes so a live holder never stalls a +# long-lived writer. Each failed retry doubles the spacing up to the cap, so a +# permanent holder costs one deferral warning per hour, not per minute. _FTS_STALE_RETRY_SECONDS = 60.0 -# Each failed retry doubles the spacing up to this cap, so a holder that never -# goes away (a second long-lived writer) costs one deferral warning per hour, -# not one per minute. A successful rebuild clears the stale state entirely. _FTS_STALE_RETRY_MAX_SECONDS = 3600.0 -# Cache for schema_read_probe_statements() — parsing SCHEMA_SQL spins up an -# in-memory SQLite database, so derive the statements once per process. +# schema_read_probe_statements() cache — deriving it parses SCHEMA_SQL in an +# in-memory SQLite database, so do it once per process. _READ_PROBE_STATEMENTS: Optional[tuple] = None -# _FTS_TRIGGERS is the full canonical set, but its two halves have different -# availability: the trigram triggers are declared ONLY by FTS_TRIGRAM_SQL / -# LEGACY_FTS_TRIGRAM_SQL, whose CREATE VIRTUAL TABLE needs the trigram -# tokenizer (SQLite >= 3.34). On a build without it, _ensure_fts_schema -# soft-fails that DDL, so those three triggers can never exist and any check -# for "all six are present" is permanently unsatisfiable. Split the set so a -# trigger's absence is only ever measured against the DDL that can create it. -# The two subsets are exhaustive and disjoint by construction (base is the -# complement of trigram); test_fts_trigger_subsets_match_the_ddl pins them -# against the DDL those triggers actually come from. +# The trigram triggers come ONLY from FTS_TRIGRAM_SQL / LEGACY_FTS_TRIGRAM_SQL, +# whose CREATE VIRTUAL TABLE needs the trigram tokenizer (SQLite >= 3.34); +# without it _ensure_fts_schema soft-fails that DDL and "all six present" is +# permanently unsatisfiable. Split the set so a trigger's absence is only +# measured against the DDL that can create it. Exhaustive and disjoint by +# construction; pinned by test_fts_trigger_subsets_match_the_ddl. _FTS_TRIGRAM_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if "_trigram_" in n) _FTS_BASE_TRIGGERS = tuple(n for n in _FTS_TRIGGERS if n not in _FTS_TRIGRAM_TRIGGERS) @@ -76,25 +70,17 @@ def schema_read_probe_statements() -> tuple: Read-only opens skip ``_reconcile_columns()`` by design (no DDL against another profile's live DB), so a store created before a schema addition - keeps 500ing on read paths until something opens it writable. Callers - that heal on staleness (see ``_open_session_db_at_path`` in - ``hermes_cli/web_server.py``) run these probes right after a read-only - open: any missing table raises "no such table" and any missing column - raises "no such column", both at prepare time. + keeps failing on read paths until something opens it writable. Healing + callers (``_open_session_db_at_path`` in the web server) run these probes + after a read-only open: a missing table/column raises at prepare time. - Derived from SCHEMA_SQL — the same source of truth the writable - reconciler diffs against — so a column added there is covered here - automatically. A hand-maintained probe list went stale within days of - shipping (it never learned ``sessions.last_activity_at``, so the sidebar - served an empty session list after `hermes update` until the user's - first message forced a writable open). - - Each statement is ``LIMIT 0``: column resolution happens at prepare - time, so the probe reads zero rows. Column references are qualified - with the table name — an unqualified double-quoted identifier that - fails to resolve silently degrades to a string literal (SQLite's - double-quoted-string misfeature), which would make the probe pass on - exactly the stale store it exists to catch. + Derived from SCHEMA_SQL so a column added there is covered automatically + (a hand-maintained list went stale within days). Each statement is + ``LIMIT 0`` (resolution at prepare time, zero rows read). Column + references are table-qualified: an unqualified double-quoted identifier + that fails to resolve silently degrades to a string literal (SQLite + misfeature), which would make the probe pass on exactly the stale store + it exists to catch. """ global _READ_PROBE_STATEMENTS if _READ_PROBE_STATEMENTS is None: @@ -120,15 +106,11 @@ class SessionSchemaMixin: def _dedupe_legacy_system_prompts(self, cursor: sqlite3.Cursor) -> None: """Move inline prompt snapshots into the shared content-addressed table. - Contention-safe by design: a ``database is locked`` (or any other - ``OperationalError``) mid-loop returns instead of raising. Partial - migration is safe — the legacy ``system_prompt`` column is kept as a - read fallback for unmigrated rows, and the next schema init picks up - the remainder. Letting the error propagate aborted schema init - entirely, left the version below 25, and made every subsequent - ``SessionDB.__init__`` re-enter this migration against the same - contended DB (enterprise field report, 2026-08-14: gateway watchdog - crash loop). + Contention-safe: any ``OperationalError`` mid-loop returns instead of + raising. Partial migration is safe — the legacy ``system_prompt`` + column stays a read fallback and the next schema init resumes. + Propagating the error aborted schema init, left the version below + 25, and re-entered this migration on every open (gateway crash loop). """ try: rows = cursor.execute( @@ -138,9 +120,7 @@ class SessionSchemaMixin: except sqlite3.OperationalError: return - for row in rows: - session_id = row["id"] if isinstance(row, sqlite3.Row) else row[0] - prompt = row["system_prompt"] if isinstance(row, sqlite3.Row) else row[1] + for session_id, prompt in rows: try: prompt_hash = self._store_system_prompt(cursor, prompt) cursor.execute( @@ -182,15 +162,10 @@ class SessionSchemaMixin: cursor: sqlite3.Cursor, names: Sequence[str] = _FTS_TRIGGERS, ) -> int: - """Count how many of *names* currently exist as triggers. - - Defaults to the full canonical set so existing callers are unchanged; - callers that need to know whether one HALF of the set is intact pass - _FTS_BASE_TRIGGERS or _FTS_TRIGRAM_TRIGGERS. - """ + """Count how many of *names* currently exist as triggers (pass + _FTS_BASE_TRIGGERS / _FTS_TRIGRAM_TRIGGERS to check one half).""" if not names: - # "name IN ()" is a syntax error in SQLite, and nothing can be - # missing from an empty set anyway. + # "name IN ()" is a SQLite syntax error. return 0 placeholders = ",".join("?" for _ in names) row = cursor.execute( @@ -198,8 +173,7 @@ class SessionSchemaMixin: f"WHERE type = 'trigger' AND name IN ({placeholders})", tuple(names), ).fetchone() - return int(row[0] if not isinstance(row, sqlite3.Row) else row[0]) - + return int(row[0]) @staticmethod def _fts_update_trigger_needs_narrowing(sql: Optional[str]) -> bool: @@ -217,26 +191,20 @@ class SessionSchemaMixin: def _migrate_broad_fts_update_triggers(self, cursor: sqlite3.Cursor) -> int: """Replace broad AFTER UPDATE FTS triggers with AFTER UPDATE OF variants. - ``CREATE TRIGGER IF NOT EXISTS`` will not replace an existing broad - trigger, so installs that already created ``AFTER UPDATE ON messages`` - would keep firing on every messages row touch (status/compaction - writes included). Inspect ``sqlite_master``, drop any still-broad - UPDATE triggers, and re-apply the current DDL constants. - - No FTS rebuild: content correctness was already gated by WHEN clauses - on modern installs; OF only skips unnecessary trigger evaluation. - - Returns the number of triggers dropped (0 when already converged). + ``CREATE TRIGGER IF NOT EXISTS`` never replaces an existing broad + trigger, so it would keep firing on every messages row touch. Drop + still-broad UPDATE triggers and re-apply the current DDL. No FTS + rebuild: correctness was already gated by WHEN clauses; OF only skips + unnecessary trigger evaluation. Returns the number dropped. """ - # CJK is a v23-only surface. Decide the layout before selecting - # destructive candidates so the legacy branch never drops a trigger - # it does not recreate. + # CJK is v23-only. Decide the layout before selecting destructive + # candidates so the legacy branch never drops a trigger it won't recreate. legacy_layout = self._db_has_legacy_inline_fts(cursor) update_names = ( "messages_fts_update", "messages_fts_trigram_update", ) - if not legacy_layout and hasattr(self, "_ensure_fts_cjk_schema"): + if not legacy_layout: update_names += ("messages_fts_cjk_update",) placeholders = ", ".join("?" for _ in update_names) rows = cursor.execute( @@ -244,12 +212,10 @@ class SessionSchemaMixin: f"WHERE type = 'trigger' AND name IN ({placeholders})", update_names, ).fetchall() - to_drop = [] - for row in rows: - name = row[0] if not isinstance(row, sqlite3.Row) else row["name"] - sql = row[1] if not isinstance(row, sqlite3.Row) else row["sql"] - if self._fts_update_trigger_needs_narrowing(sql): - to_drop.append(name) + to_drop = [ + name for name, sql in rows + if self._fts_update_trigger_needs_narrowing(sql) + ] if not to_drop: return 0 @@ -270,12 +236,11 @@ class SessionSchemaMixin: self._ensure_fts_schema( cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL ) - # CJK triggers live on the host SessionDB; only recreate one that - # this migration actually dropped. ``_ensure_fts_cjk_schema`` is - # documented never-raises and soft-fails OperationalError by - # clearing availability — raise-path handling alone is not - # enough. After ensure, require a narrowed CJK UPDATE trigger or - # durable quarantine (stale breadcrumb + unavailable). + # Only recreate the CJK trigger this migration actually dropped. + # ``_ensure_fts_cjk_schema`` soft-fails OperationalError by clearing + # availability (never raises), so after ensure require a narrowed + # CJK UPDATE trigger or durable quarantine (stale breadcrumb + + # unavailable). if "messages_fts_cjk_update" in to_drop: try: self._ensure_fts_cjk_schema(cursor) @@ -306,20 +271,14 @@ class SessionSchemaMixin: "WHERE type = 'trigger' AND name = ?", ("messages_fts_cjk_update",), ).fetchone() - if not row: - return False - sql = row[0] if not isinstance(row, sqlite3.Row) else row["sql"] - return not self._fts_update_trigger_needs_narrowing(sql) + return bool(row) and not self._fts_update_trigger_needs_narrowing(row[0]) def _quarantine_cjk_after_update_of_migration( self, cursor: sqlite3.Cursor ) -> None: - """Fail-closed after dropping CJK UPDATE during OF migration. - - Clears availability, persists ``fts_cjk_stale``, and drops any - residual broad/partial CJK UPDATE trigger so a later open cannot - ``CREATE TRIGGER IF NOT EXISTS`` a gap without rebuild. - """ + """Fail closed after dropping the CJK UPDATE trigger mid-migration: + clear availability, persist ``fts_cjk_stale``, drop any residual CJK + UPDATE trigger so a later open cannot IF-NOT-EXISTS over a gap.""" self._fts_cjk_available = False try: self.set_meta(FTS_CJK_STALE_KEY, "1", cursor=cursor) @@ -343,18 +302,15 @@ class SessionSchemaMixin: *, include_trigram: bool = True, ) -> None: - # Both FTS tables are external-content (v23+): the special 'rebuild' - # command wipes the inverted index and repopulates it from the - # content source (messages for the standard index, the tool-row- - # excluding messages_fts_trigram_src view for the trigram index). + # v23+ external-content tables: 'rebuild' repopulates the inverted + # index from the content source (messages / messages_fts_trigram_src). cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')") if include_trigram: cursor.execute( "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')" ) - # 'rebuild' indexes EVERY row, so any deferred-backfill markers are - # now satisfied — clear them, otherwise the background worker would - # re-insert rows the rebuild already covered (duplicate entries). + # 'rebuild' indexes EVERY row: clear deferred-backfill markers or the + # worker would re-insert rows already covered (duplicates). cursor.execute( "DELETE FROM state_meta WHERE key IN " "('fts_rebuild_high_water', 'fts_rebuild_progress')" @@ -366,49 +322,37 @@ class SessionSchemaMixin: *, include_trigram: bool = True, ) -> None: - """Rebuild the LEGACY inline FTS indexes (pre-v23) from messages. + """Rebuild the LEGACY inline (pre-v23) FTS indexes from messages. - Used only to repair a legacy DB whose triggers degraded under an - earlier no-FTS5 runtime. Inline tables have no external-content - 'rebuild' source, so we DELETE + reinsert the concatenated content - the legacy triggers produced. Never touches the v23 shape. + Inline tables have no external-content 'rebuild' source, so DELETE + + reinsert the concatenated content the legacy triggers produced. + Never touches the v23 shape. """ - cursor.execute("DELETE FROM messages_fts") - cursor.execute( - "INSERT INTO messages_fts(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) - if not include_trigram: - return - cursor.execute("DELETE FROM messages_fts_trigram") - cursor.execute( - "INSERT INTO messages_fts_trigram(rowid, content) " - "SELECT id, " - "COALESCE(content, '') || ' ' || " - "COALESCE(tool_name, '') || ' ' || " - "COALESCE(tool_calls, '') " - "FROM messages" - ) + tables = ("messages_fts", "messages_fts_trigram") if include_trigram else ("messages_fts",) + for tbl in tables: + cursor.execute(f"DELETE FROM {tbl}") + cursor.execute( + f"INSERT INTO {tbl}(rowid, content) " + "SELECT id, " + "COALESCE(content, '') || ' ' || " + "COALESCE(tool_name, '') || ' ' || " + "COALESCE(tool_calls, '') " + "FROM messages" + ) def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]: try: cursor.execute(f"SELECT * FROM {table_name} LIMIT 0") return True except (sqlite3.OperationalError, UnicodeDecodeError) as exc: - # UnicodeDecodeError can occur when FTS shadow tables or content - # columns hold invalid UTF-8 bytes. On some Python/SQLite builds - # it surfaces as a bare UnicodeDecodeError (ValueError subclass, - # not sqlite3.Error); on others as OperationalError("Could not - # decode to UTF-8 column ..."). Catch both so the probe never - # kills the connection or raises to writable-init/recovery flows. + # Invalid UTF-8 in FTS content surfaces as a bare UnicodeDecodeError + # (not sqlite3.Error) on some builds and as OperationalError("Could + # not decode to UTF-8 ...") on others; catch both so the probe never + # raises into writable-init/recovery flows. if isinstance(exc, sqlite3.OperationalError): if self._is_fts5_unavailable_error(exc): - # Only disable FTS entirely when the whole module is missing. - # A missing trigram tokenizer only affects trigram searches. + # A missing trigram tokenizer only affects trigram search; + # only a missing FTS5 module disables FTS entirely. if self._is_trigram_unavailable_error(exc): self._warn_trigram_unavailable(exc) else: @@ -416,13 +360,11 @@ class SessionSchemaMixin: return None if "no such table" in str(exc).lower(): return False - # Re-raise any other OperationalError (e.g. malformed schema, - # corrupt vtable that isn't a decode error). + # Anything else (malformed schema, corrupt vtable) re-raises. if "decode to utf-8" not in str(exc).lower(): raise - # Swallow: decode error means the index is degraded but the - # store remains accessible. Writable init / recovery will - # schedule a rebuild or degrade to LIKE. + # Decode error: index degraded, store accessible; writable init / + # recovery schedules a rebuild or degrades to LIKE. logger.warning( "%s probe encountered invalid UTF-8 in FTS content; " "search may return incomplete results until FTS is rebuilt: %s", @@ -449,8 +391,7 @@ class SessionSchemaMixin: (FTS_REBUILD_DEFERRAL_KEY,), ).fetchone() if row: - raw = row["value"] if isinstance(row, sqlite3.Row) else row[0] - parsed = json.loads(raw) + parsed = json.loads(row[0]) if isinstance(parsed, dict): record = parsed except (sqlite3.Error, TypeError, ValueError, json.JSONDecodeError): @@ -513,13 +454,10 @@ class SessionSchemaMixin: attempts, ) return False - # Full structural rebuild: admit through the single cross-process - # authority (fail closed). Losing the race means another process is - # already performing this exact recovery; the stale breadcrumb stays - # set, so this process simply keeps FTS detached and retries later. - with fts_rebuild_admission( - getattr(self, "db_path", None), timeout_seconds=timeout_seconds - ) as admitted: + # Full structural rebuild: admit through the cross-process authority + # (fail closed). Losing the race means another process is doing this + # exact recovery; the breadcrumb stays set and we retry later. + with fts_rebuild_admission(self.db_path, timeout_seconds=timeout_seconds) as admitted: if not admitted: logger.warning( "Deferred stale state.db FTS rebuild: another process " @@ -532,24 +470,18 @@ class SessionSchemaMixin: def retry_deferred_fts_recovery(self) -> bool: """Retry a deferred stale-FTS rebuild on this open SessionDB. - ``_recover_stale_fts`` runs at open and fails closed when foreign - holders or the rebuild lock are busy, leaving ``_fts_stale`` set and - search on the LIKE fallback. Live write/search paths must never start - a full rebuild (#97940), so on a short-lived CLI that deferral is - cleared by the next process open — but a gateway opens state.db - once and stays up for days, so "next open" never came (#100108). - This is the in-process retry: bounded backoff from - ``_FTS_STALE_RETRY_SECONDS`` doubling to ``_FTS_STALE_RETRY_MAX_SECONDS``, - non-blocking admission (``timeout=0``) so a live holder is skipped and - tried again later, no new thread — the caller is an existing periodic - tick (gateway housekeeping). + ``_recover_stale_fts`` fails closed at open when holders or the rebuild + lock are busy, leaving ``_fts_stale`` set and search on LIKE. Live + write/search paths must never start a full rebuild, and a gateway + opens state.db once for days, so "next open" never comes. This is the + in-process retry from the gateway housekeeping tick: bounded backoff + (``_FTS_STALE_RETRY_SECONDS`` doubling to the max), non-blocking + admission (``timeout=0``), no new thread. Returns True only when the index was rebuilt and sync triggers restored. Never raises. """ - if not getattr(self, "_fts_stale", False): - return False - if getattr(self, "read_only", False) or getattr(self, "_conn", None) is None: + if not self._fts_stale or self.read_only or self._conn is None: return False now = time.monotonic() if now < getattr(self, "_fts_stale_retry_after", 0.0): @@ -596,8 +528,8 @@ class SessionSchemaMixin: try: trigram_status = self._fts_table_probe(cursor, "messages_fts_trigram") except (sqlite3.DatabaseError, UnicodeDecodeError): - # A corrupt vtable may fail even a LIMIT 0 probe. It still needs - # to be included in the drop-and-recreate recovery below. + # A corrupt vtable may fail even a LIMIT 0 probe; it must still be + # included in the drop-and-recreate below. trigram_status = True include_trigram = trigram_status is True @@ -665,8 +597,8 @@ class SessionSchemaMixin: self._conn.rollback() except sqlite3.Error: pass - # Stale indexes must remain detached even on SQLite builds whose - # DDL transaction behavior differs. + # Stale indexes must stay detached even on SQLite builds whose DDL + # transaction behavior differs. self._drop_all_fts_triggers(cursor) self._conn.commit() logger.error( @@ -687,34 +619,22 @@ class SessionSchemaMixin: @staticmethod def _parse_schema_columns(schema_sql: str) -> Dict[str, Dict[str, str]]: - """Extract expected columns per table from SCHEMA_SQL. + """Expected columns per table, parsed from SCHEMA_SQL. - Uses an in-memory SQLite database to parse the SQL — SQLite itself - handles all syntax (DEFAULT expressions with commas, inline - REFERENCES, CHECK constraints, etc.) so there are zero regex - edge cases. The in-memory DB is opened, the schema DDL is - executed, and PRAGMA table_info extracts the column metadata. - - Adding a column to SCHEMA_SQL is all that's needed; the - reconciliation loop picks it up automatically. - - The parse result is memoized on disk keyed by a hash of the DDL: - executing SCHEMA_SQL (FTS5 virtual tables included) in the scratch - DB costs ~85ms on every startup, but the output is a pure function - of the DDL text, which only changes when the shipped code changes. - Reconciliation itself (diffing the LIVE database) still runs every - startup — only the reference-side parse is cached. A corrupt or - stale cache degrades to recomputation. + Executes the DDL in an in-memory SQLite database and reads PRAGMA + table_info, so SQLite handles every syntax edge case (no regex). + The result is memoized on disk keyed by a hash of the DDL (~85ms per + startup otherwise; a pure function of the DDL text). Only the + reference-side parse is cached — diffing the LIVE database still runs + every startup. A corrupt or stale cache degrades to recomputation. """ - import hashlib as _hashlib - import json as _json - cache_path = None - schema_hash = _hashlib.sha256(schema_sql.encode("utf-8")).hexdigest() + schema_hash = hashlib.sha256(schema_sql.encode("utf-8")).hexdigest() try: - from hermes_constants import get_hermes_home - cache_path = get_hermes_home() / "cache" / "schema_columns.json" - blob = _json.loads(cache_path.read_text(encoding="utf-8")) + # Late import: resolves a test-patched hermes_constants.get_hermes_home. + from hermes_constants import get_hermes_home as _home + cache_path = _home() / "cache" / "schema_columns.json" + blob = json.loads(cache_path.read_text(encoding="utf-8")) if ( isinstance(blob, dict) and blob.get("schema_hash") == schema_hash @@ -739,15 +659,9 @@ class SessionSchemaMixin: "WHERE type='table' AND name NOT LIKE 'sqlite_%'" ).fetchall(): cols: Dict[str, str] = {} - for row in ref.execute( + for _cid, col_name, col_type, notnull, default, pk in ref.execute( f'PRAGMA table_info("{tbl}")' ).fetchall(): - # row: (cid, name, type, notnull, dflt_value, pk) - col_name = row[1] - col_type = row[2] or "" - notnull = row[3] - default = row[4] - pk = row[5] # Reconstruct the type expression for ALTER TABLE ADD COLUMN parts = [col_type] if col_type else [] if notnull and not pk: @@ -761,48 +675,33 @@ class SessionSchemaMixin: if cache_path is not None: try: - import os as _os - import tempfile as _tempfile cache_path.parent.mkdir(parents=True, exist_ok=True) - fd, tmp = _tempfile.mkstemp( + fd, tmp = tempfile.mkstemp( dir=str(cache_path.parent), prefix=".schema_columns." ) - with _os.fdopen(fd, "w", encoding="utf-8") as fh: - _json.dump( - {"schema_hash": schema_hash, "tables": table_columns}, fh - ) - _os.replace(tmp, cache_path) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump({"schema_hash": schema_hash, "tables": table_columns}, fh) + os.replace(tmp, cache_path) except Exception: pass # cache write is best-effort return table_columns def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None: - """Ensure live tables have every column declared in SCHEMA_SQL. + """ADD every SCHEMA_SQL column missing from the live tables. - Follows the Beets/sqlite-utils pattern: the CREATE TABLE definition - in SCHEMA_SQL is the single source of truth for the desired schema. - On every startup this method diffs the live columns (via PRAGMA - table_info) against the declared columns, and ADDs any that are - missing. - - This makes column additions a declarative operation — just add - the column to SCHEMA_SQL and it appears on the next startup. - Version-gated migration blocks are no longer needed for ADD COLUMN. + Beets/sqlite-utils pattern: SCHEMA_SQL is the single source of truth; + column additions are declarative and need no version-gated migration. """ expected = self._parse_schema_columns(SCHEMA_SQL) for table_name, declared_cols in expected.items(): - # Get current columns from the live table try: rows = cursor.execute( f'PRAGMA table_info("{table_name}")' ).fetchall() except sqlite3.OperationalError: continue # Table doesn't exist yet (shouldn't happen after executescript) - live_cols = set() - for row in rows: - # PRAGMA table_info returns (cid, name, type, notnull, dflt_value, pk) - name = row[1] if isinstance(row, (tuple, list)) else row["name"] - live_cols.add(name) + # PRAGMA table_info rows: (cid, name, type, notnull, dflt_value, pk) + live_cols = {row[1] for row in rows} for col_name, col_type in declared_cols.items(): if col_name not in live_cols: @@ -814,78 +713,49 @@ class SessionSchemaMixin: except sqlite3.OperationalError as exc: message = str(exc).lower() if "duplicate column" in message: - # Expected: a sibling process won the race to ADD - # this column between our PRAGMA diff and the - # ALTER. The store ends up correct either way. + # A sibling process won the ADD race; store is correct. logger.debug( "reconcile %s.%s: %s", table_name, col_name, exc, ) continue if "locked" in message or "busy" in message: - # Lock contention (e.g. an orphaned sibling - # backend holding the write lock, #79531). This - # used to be swallowed at DEBUG, leaving the - # store half-reconciled: startup "succeeded" and - # every session-list read then failed with - # "no such column" until an unrelated writable - # open. Re-raise instead so the open-time lock - # patience in _connect_and_init_with_lock_patience - # retries the WHOLE init (executescript is - # idempotent CREATE IF NOT EXISTS) with jittered - # backoff rather than serving a stale schema. + # Lock contention: swallowing it left the store + # half-reconciled ("no such column" on every read). + # Re-raise so _connect_and_init_with_lock_patience + # retries the WHOLE init (idempotent) with backoff. raise - # Anything else ("Cannot add a NOT NULL column with - # default value NULL", ...) is a schema mistake that - # permanently strands the store behind SCHEMA_SQL — - # be loud, don't bury it at DEBUG. + # Anything else is a schema mistake that permanently + # strands the store behind SCHEMA_SQL — be loud. logger.warning( "reconcile %s.%s failed; store remains behind " "SCHEMA_SQL: %s", table_name, col_name, exc, ) + @staticmethod + def _live_pk_columns(cursor: sqlite3.Cursor, table: str) -> Optional[List[str]]: + """PRIMARY KEY column names of *table* in key order; None when the + table is missing or has no columns (SCHEMA_SQL creates it correctly).""" + try: + rows = cursor.execute(f'PRAGMA table_info("{table}")').fetchall() + except sqlite3.OperationalError: + return None + if not rows: + return None + # row: (cid, name, type, notnull, dflt_value, pk) + return [r[1] for r in sorted((r for r in rows if r[5]), key=lambda r: r[5])] + def _heal_gateway_routing_pk(self, cursor: sqlite3.Cursor) -> None: """Rebuild ``gateway_routing`` when its PRIMARY KEY predates scoping. - Early builds of the routing-index migration (#59203) created the - table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column. - ``_reconcile_columns()`` ADDs the missing ``scope`` column on those - databases, but SQLite cannot ALTER a primary key, so the shipped - composite ``PRIMARY KEY (scope, session_key)`` never lands. On such - tables every write path is broken: - - * ``save_gateway_routing_entry`` fails with "ON CONFLICT clause does - not match any PRIMARY KEY or UNIQUE constraint" (its upsert targets - the composite key), and - * ``replace_gateway_routing_entries`` fails with "UNIQUE constraint - failed: gateway_routing.session_key" whenever the same session_key - exists under a different scope — the exact isolation the composite - key exists to provide. - - Each failed save logs a warning and falls back to sessions.json, - so a legacy-shaped table produces endless per-save warning spam. - Rebuild it once, preserving rows. On a session_key collision across - scopes (possible while the PK was wrong) the newest row wins. + Early builds used ``session_key TEXT PRIMARY KEY``; the reconciler ADDs + ``scope`` but SQLite cannot ALTER a PK, so the composite key never + lands and every routing write fails (ON CONFLICT mismatch / UNIQUE + violation across scopes) with per-save warning spam. Rebuild once, + preserving rows; on a cross-scope session_key collision the newest + row wins. """ - try: - rows = cursor.execute( - 'PRAGMA table_info("gateway_routing")' - ).fetchall() - except sqlite3.OperationalError: - return - if not rows: - return - - def _col(row, idx, name): - return row[idx] if isinstance(row, (tuple, list)) else row[name] - - pk_cols = [ - _col(r, 1, "name") - for r in sorted( - (r for r in rows if _col(r, 5, "pk")), - key=lambda r: _col(r, 5, "pk"), - ) - ] - if pk_cols == ["scope", "session_key"]: + pk_cols = self._live_pk_columns(cursor, "gateway_routing") + if pk_cols is None or pk_cols == ["scope", "session_key"]: return logger.info( @@ -905,9 +775,7 @@ class SessionSchemaMixin: PRIMARY KEY (scope, session_key) )""" ) - # INSERT OR REPLACE + updated_at ordering: if the broken PK ever let - # two scopes race over one session_key, keep the newest row per - # (scope, session_key) pair. + # INSERT OR REPLACE in updated_at order: newest row per key wins. cursor.execute( "INSERT OR REPLACE INTO gateway_routing " "(scope, session_key, entry_json, updated_at) " @@ -919,40 +787,15 @@ class SessionSchemaMixin: def _heal_session_model_usage_pk(self, cursor: sqlite3.Cursor) -> None: """Rebuild ``session_model_usage`` when its PRIMARY KEY lacks ``task``. - Installs whose ``state.db`` reached ``schema_version >= 22`` before - the ``task`` dimension was added carry a 5-column PRIMARY KEY - ``(session_id, model, billing_provider, billing_base_url, - billing_mode)``. ``_reconcile_columns()`` ADDs the ``task`` column - as a bare nullable, but SQLite cannot ALTER a primary key, so the - shipped composite 6-column key never lands. The version-gated v22 - rebuild is unreachable on those installs (``current_version < 22`` - is already false), so every upsert in ``_record_model_usage()`` - fails with "ON CONFLICT clause does not match any PRIMARY KEY or - UNIQUE constraint" — aborting the enclosing write transaction and - silently zeroing all token *and* cost accounting (#73823). - - Idempotent; runs unconditionally on every open, same pattern as - :meth:`_heal_gateway_routing_pk` above. On healthy databases the - PRAGMA check short-circuits and this is a no-op. + Installs already at v22+ when ``task`` landed carry the 5-column PK; + the reconciler ADDs ``task`` as a bare nullable but SQLite cannot + ALTER a PK, and the version-gated v22 rebuild is unreachable there. + Every ``_record_model_usage()`` upsert then fails (ON CONFLICT + mismatch), aborting the write transaction and silently zeroing token + and cost accounting. Idempotent; no-op on healthy databases. """ - try: - rows = cursor.execute( - 'PRAGMA table_info("session_model_usage")' - ).fetchall() - except sqlite3.OperationalError: - return - if not rows: - # Table doesn't exist yet — SCHEMA_SQL creates it correctly. - return - - def _col(row, idx, name): - return row[idx] if isinstance(row, (tuple, list)) else row[name] - - pk_cols = { - _col(r, 1, "name") for r in rows if _col(r, 5, "pk") - } - if "task" in pk_cols: - # task is already in the PK — healthy. + pk_cols = self._live_pk_columns(cursor, "session_model_usage") + if pk_cols is None or "task" in pk_cols: return logger.info( @@ -960,15 +803,10 @@ class SessionSchemaMixin: "rebuilding with composite 6-column key", sorted(pk_cols), ) - # FK-off window: the connection enables PRAGMA foreign_keys=ON - # before _init_schema runs, and session_model_usage.session_id - # REFERENCES sessions(id). INSERT OR IGNORE does NOT suppress - # foreign-key violations (OR IGNORE only covers uniqueness/NOT - # NULL conflicts), so an orphaned usage row — possible after a - # partial prune while accounting was broken — would abort the - # whole rebuild. Disable FK enforcement for the copy and restore - # it afterwards. PRAGMA foreign_keys is a no-op inside a - # transaction, which is fine here: _init_schema runs on an + # FK-off window: INSERT OR IGNORE does NOT suppress foreign-key + # violations, so an orphaned usage row (partial prune while accounting + # was broken) would abort the whole rebuild. PRAGMA foreign_keys is a + # no-op inside a transaction — fine here, _init_schema runs on an # isolation_level=None connection with no transaction open. cursor.execute("PRAGMA foreign_keys=OFF") try: @@ -999,10 +837,8 @@ class SessionSchemaMixin: PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) )""" ) - # OR IGNORE: while the PK was wrong the reconciler may have left - # ``task`` NULL on old rows; COALESCE to '' can theoretically - # collide with a genuine ''-task row — keep the first, drop the - # duplicate rather than fail the heal. + # OR IGNORE: COALESCE(task, '') on legacy NULL rows can collide + # with a genuine ''-task row — keep the first rather than fail. cursor.execute( """INSERT OR IGNORE INTO session_model_usage ( session_id, model, billing_provider, billing_base_url, @@ -1037,57 +873,34 @@ class SessionSchemaMixin: cursor.execute("PRAGMA foreign_keys=ON") def _init_schema(self): - """Create tables and FTS if they don't exist, reconcile columns. + """Create tables and FTS if missing, reconcile columns, run data migrations. - Schema management follows the declarative reconciliation pattern - (Beets, sqlite-utils): SCHEMA_SQL is the single source of truth. - On existing databases, _reconcile_columns() diffs live columns - against SCHEMA_SQL and ADDs any missing ones. This eliminates - the version-gated migration chain for column additions, making - it impossible for reordered or inserted migrations to skip columns. - - The schema_version table is retained for future data migrations - (transforming existing rows) which cannot be handled declaratively. + SCHEMA_SQL is the single source of truth: column additions are + declarative via _reconcile_columns(), so reordered migrations can + never skip a column. schema_version remains for data migrations + (row transforms) that cannot be expressed declaratively. """ - # Declare a startup-watchdog progress lease before potentially long - # synchronous work: on multi-GB state.db files the reconciliation + - # version-gated data migrations below are legitimately slow and can - # be I/O-bound (near-zero CPU), which the watchdog's CPU fallback - # would misread as a parked deadlock (OOF-298 / PR #89750). - # Single lease is deliberate: this is the one pre-loop phase that can - # legitimately exceed the 300s default deadline (multi-GB DBs), and - # the lease is clamped to _MAX_LEASE_S=900. Honest worst case: a - # genuinely wedged DB init delays supervisor respawn by up to the - # lease duration. Per-chunk renewal would shrink that, but adds - # complexity to the migration loops for a rare failure mode. + # Startup-watchdog progress lease: on multi-GB state.db files the + # reconciliation + data migrations are legitimately slow and I/O-bound + # (near-zero CPU), which the watchdog's CPU fallback would misread as + # a parked deadlock. Single lease is deliberate (clamped to + # _MAX_LEASE_S=900): a genuinely wedged init delays supervisor respawn + # by up to the lease; per-chunk renewal isn't worth the complexity. report_startup_progress(600.0, phase="state_db_init_schema") cursor = self._conn.cursor() cursor.executescript(SCHEMA_SQL) - # ── Declarative column reconciliation ────────────────────────── - # Diff live tables against SCHEMA_SQL and ADD any missing columns. - # This is idempotent and self-healing: even if a version-gated - # migration was skipped (e.g. due to version renumbering), the - # column gets created here. + # Idempotent, self-healing column reconciliation, then the two + # table-shape repairs ADD COLUMN cannot express (PK rebuilds). self._reconcile_columns(cursor) - - # Rebuild gateway_routing if it still carries the pre-scope PRIMARY - # KEY (session_key alone). ADD COLUMN cannot fix a PK, so this is - # the one table-shape repair reconciliation can't express. self._heal_gateway_routing_pk(cursor) - - # Rebuild session_model_usage if its PRIMARY KEY lacks the ``task`` - # column (5-column PK on installs already at v22+ when the column - # landed — the version-gated rebuild is unreachable there, #73823). - # Same PK-rebuild constraint as gateway_routing above. self._heal_session_model_usage_pk(cursor) - # Indexes that reference reconciler-added columns must be created - # AFTER _reconcile_columns runs — declaring them in SCHEMA_SQL - # makes the initial executescript fail on legacy DBs (the index's - # WHERE clause references a column that doesn't exist yet). + # Indexes referencing reconciler-added columns must be created AFTER + # _reconcile_columns — in SCHEMA_SQL the initial executescript would + # fail on legacy DBs (WHERE references a not-yet-existing column). try: cursor.execute( "CREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id " @@ -1097,20 +910,14 @@ class SessionSchemaMixin: except sqlite3.OperationalError as exc: logger.debug("idx_messages_platform_msg_id create skipped: %s", exc) - # Deferred indexes that reference the reconciler-added ``active`` - # column (idx_messages_session_active) — same ordering constraint. + # Same ordering constraint (idx_messages_session_active on ``active``). cursor.executescript(DEFERRED_INDEX_SQL) - # Heal NULL ``active`` rows unconditionally on every startup. - # On real-world DBs the reconciler-added ``active`` column can lack - # its NOT NULL DEFAULT 1 (older reconciler builds reconstructed the - # type without the default — see #51646: PRAGMA shows - # (17,'active','INTEGER',0,None,0) in the wild), so INSERTs that - # omitted the column wrote NULL and the ``WHERE active = 1`` - # transcript loaders hid the whole history. The INSERTs now set - # active=1 explicitly; this idempotent repair un-hides rows written - # before the fix. It was previously gated at ``current_version < - # 12`` which never re-ran for already-v12+ databases. + # Heal NULL ``active`` rows on every startup: older reconciler builds + # added ``active`` without its NOT NULL DEFAULT 1, so INSERTs omitting + # it wrote NULL and ``WHERE active = 1`` loaders hid whole histories. + # Unconditional because a ``current_version < 12`` gate never re-ran + # for already-v12+ databases. try: cursor.execute( "UPDATE messages SET active = 1 WHERE active IS NULL" @@ -1125,28 +932,23 @@ class SessionSchemaMixin: (FTS_STALE_KEY,), ).fetchone() is not None if self._fts_stale: - # A prior process deliberately detached FTS after corruption. - # Keep every FTS writer detached until a full rebuild succeeds. + # A prior process detached FTS after corruption; keep every FTS + # writer detached until a full rebuild succeeds. self._drop_all_fts_triggers(cursor) if not fts5_available: - # Existing FTS triggers can still fire on messages INSERT/UPDATE - # even though the current sqlite runtime cannot read the virtual - # tables they target. Drop only the triggers so core persistence - # continues; if a future runtime has FTS5, _ensure_fts_schema() - # recreates them. + # Existing FTS triggers would still fire on messages writes even + # though this runtime cannot read their targets. Drop only the + # triggers so persistence continues; a future FTS5 runtime's + # _ensure_fts_schema() recreates them. self._drop_fts_triggers(cursor) - # ── Schema version bookkeeping ───────────────────────────────── - # Bump to current so future data migrations (if any) can gate on - # version. No version-gated column additions remain. - cursor.execute("SELECT version FROM schema_version LIMIT 1") - row = cursor.fetchone() + row = cursor.execute("SELECT version FROM schema_version LIMIT 1").fetchone() if row is None: cursor.execute( "INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,), ) - # Record store provenance on creation so fresh vs wiped stores are distinguishable (#97568) + # Store provenance so fresh vs wiped stores are distinguishable. now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() instance_id = str(uuid.uuid4()) cursor.executemany( @@ -1158,27 +960,16 @@ class SessionSchemaMixin: ) else: - current_version = row["version"] if isinstance(row, sqlite3.Row) else row[0] - # Renew the progress lease: the version-gated chain below can - # rewrite whole tables (PK rebuilds, backfills) on large DBs. - # Same deliberate single-lease trade-off as _init_schema: honest - # worst case is up to the lease duration of zombie time on a - # wedged migration, accepted over per-chunk renewal complexity. + current_version = row[0] + # Renew the lease: the version-gated chain can rewrite whole tables + # on large DBs (same single-lease trade-off as above). report_startup_progress(600.0, phase="state_db_data_migrations") - # Data migrations that can't be expressed declaratively (row - # backfills, index changes tied to a specific version step) stay - # in a version-gated chain. Column additions are handled by - # _reconcile_columns() above and no longer need entries here. + # Version-gated chain for data migrations only (row backfills, + # version-specific index changes); column additions never belong here. if current_version < 10 and SCHEMA_VERSION == 10: - # v10: trigram FTS5 table for CJK/substring search. The - # virtual table + triggers are created unconditionally via - # FTS_TRIGRAM_SQL below, but existing rows need a one-time - # backfill into the FTS index. - # - # Only run this when v10 itself is the target schema. Current - # v11+ code drops and rebuilds both FTS tables below, so doing - # the v10-only trigram backfill first only burns startup time - # and WAL space before v11 throws the work away. + # v10: one-time trigram backfill. Only when v10 itself is the + # target: v11+ drops and rebuilds both FTS tables, so the + # backfill would only burn startup time and WAL space. if fts5_available: _fts_trigram_exists = self._fts_table_probe( cursor, "messages_fts_trigram" @@ -1197,20 +988,11 @@ class SessionSchemaMixin: fts_migrations_complete = False else: fts_migrations_complete = False - if current_version < 11 and SCHEMA_VERSION < 23: - # v11 (SUPERSEDED by v23): re-index FTS5 tables to cover - # tool_name + tool_calls in inline mode (#16751). v23 drops - # and rebuilds both FTS tables in external-content form, so - # running the v11 inline backfill first would only burn - # startup time and WAL space before v23 throws the work - # away — and its inline INSERT shape no longer matches the - # current external-content FTS_SQL anyway. Kept only for - # source archaeology; unreachable while SCHEMA_VERSION >= 23. - pass + # (v11 inline FTS re-index was superseded by v23 and removed.) if current_version < 16: # v16: tag delegate subagent rows so pickers stay clean after - # parent deletes that used to orphan them (parent_session_id → NULL). - # The shared predicate excludes user-visible reset children. + # parent deletes orphan them. The shared predicate excludes + # user-visible reset children. try: cursor.execute( "UPDATE sessions SET model_config = json_set(" @@ -1235,28 +1017,17 @@ class SessionSchemaMixin: except sqlite3.OperationalError: pass if current_version < 18: - # v18: gateway metadata consolidation (#9006). Backfill - # display_name / origin_json / expiry_finalized from - # sessions.json so pre-migration gateway sessions are - # discoverable from state.db without the JSON index. + # v18: backfill gateway metadata from sessions.json. Best-effort: + # missing metadata just means consumers fall back to + # sessions.json until the gateway rewrites those rows. try: self._backfill_gateway_metadata_from_sessions_json(cursor) except Exception as exc: - # Backfill is best-effort: sessions.json may be absent, - # corrupted, or partially stale. Missing metadata simply - # means consumers fall back to sessions.json for those - # rows until the gateway rewrites them. logger.debug("v18 gateway metadata backfill skipped: %s", exc) if current_version < 20: - # v20: per-model usage attribution (issue #51607). Going - # forward update_token_counts() records each API call into - # session_model_usage keyed by the live model, but existing - # sessions only have their aggregate totals on the sessions - # row. Seed one usage row per historical session from those - # aggregates so insights reads uniformly from the new table. - # INSERT OR IGNORE keeps it idempotent: if newer code already - # wrote a (session_id, model, provider) row for a session, the - # PK conflict skips the stale aggregate rather than doubling it. + # v20: seed one session_model_usage row per historical session + # from the sessions aggregates. INSERT OR IGNORE: a row newer + # code already wrote wins over the stale aggregate. try: cursor.execute( """INSERT OR IGNORE INTO session_model_usage ( @@ -1292,15 +1063,10 @@ class SessionSchemaMixin: except sqlite3.OperationalError: pass if current_version < 22: - # v22: task-dimension usage attribution (issue #23270). - # session_model_usage gains a ``task`` column ('' = main agent - # loop; 'vision'/'compression'/'title_generation'/... = - # auxiliary calls) so aux model spend is visible in analytics. - # The column participates in the PRIMARY KEY and SQLite cannot - # ALTER a PK, so rebuild the table. The reconciler will have - # already ADDed the plain column on legacy DBs (harmless); - # the rebuild bakes it into the PK properly. Existing rows are - # main-loop accounting by definition → task=''. + # v22: ``task`` joins the session_model_usage PRIMARY KEY ('' = + # main loop; 'vision'/'compression'/... = aux calls). SQLite + # cannot ALTER a PK, so rebuild; existing rows are main-loop + # accounting → task=''. try: legacy_pk = cursor.execute( "SELECT COUNT(*) FROM pragma_table_info('session_model_usage') " @@ -1357,57 +1123,34 @@ class SessionSchemaMixin: ) except sqlite3.OperationalError as exc: logger.debug("v22 session_model_usage rebuild skipped: %s", exc) - if current_version < 23: - # v23: FTS storage redesign (issues #22478, #43690, #55233). - # The v11 inline-mode FTS tables each store a full private - # copy of every message (content || tool_name || tool_calls), - # and the trigram index additionally covers role='tool' rows - # (~90% of message bytes: base64 payloads, file dumps) at - # ~2.6x amplification — together ~75% of state.db on heavy - # installs (observed: 18.9 GB of a 25 GB DB). - # - # OPT-IN, NOT AUTOMATIC. The transition (demote old vtables → - # new external-content schema → backfill → teardown → VACUUM) - # is disk-heavy (transient ~2x file size to fully reclaim via - # VACUUM) and long (~1-2h background on a 25 GB DB). Doing it - # silently on every big user's next open — with a completeness - # guarantee that depends on the process staying alive long - # enough — is the wrong default. So on an EXISTING install we - # touch nothing here: the v22 inline FTS keeps working exactly - # as before, and we only record a flag advertising that the - # optimization is available. `hermes sessions optimize-storage` - # performs the whole transition as one deliberate, disk-checked, - # progress-reported foreground operation. - # - # DECOUPLED VERSIONING. Crucially, this does NOT hold back the - # main schema_version. The FTS storage LAYOUT is tracked by an - # independent `fts_storage_version` marker (see - # _fts_storage_version / SETTLE below), so schema_version - # advances to SCHEMA_VERSION here like every other migration — - # future v24+ migrations land automatically for legacy-FTS - # users too. Only the FTS *layout* waits for opt-in. - if fts5_available and self._db_has_legacy_inline_fts(cursor): - self.set_meta("fts_optimize_available", "1", cursor=cursor) + # v23: FTS storage redesign (external-content tables; inline v11 + # tables were ~75% of state.db on heavy installs). OPT-IN, NOT + # AUTOMATIC: the transition is disk-heavy (~2x transient) and long + # (hours on a 25 GB DB), so an existing install only gets a flag + # advertising it; `hermes sessions optimize-storage` performs it as + # a deliberate foreground operation. DECOUPLED VERSIONING: the FTS + # layout is tracked by the independent `fts_storage_version` + # marker, so schema_version still advances here and future + # migrations land for legacy-FTS users too. + if ( + current_version < 23 + and fts5_available + and self._db_has_legacy_inline_fts(cursor) + ): + self.set_meta("fts_optimize_available", "1", cursor=cursor) if current_version < 25: - # v25: de-duplicate per-session system prompt snapshots into - # a shared content-addressed table. Keep the old column as a - # read fallback for partially migrated or externally written - # rows, but clear migrated rows so future writes do not keep - # one large prompt copy per session. + # v25: de-duplicate system prompt snapshots into the shared + # content-addressed table; the old column stays a read fallback + # for partially migrated or externally written rows. self._dedupe_legacy_system_prompts(cursor) - # The FTS storage layout is versioned independently of the main - # schema (see the v23 note above). Stamp the current layout so the - # main version can always advance: a fresh/optimized DB is at - # FTS_STORAGE_VERSION; a legacy DB is left at whatever it had - # (absent/0) until `optimize-storage` runs. An INTERRUPTED - # optimize (legacy vtables already demoted, but rebuild markers - # or demoted trash tables still present, or an empty external - # index against non-empty messages) is NOT stamped either — - # the marker is the source of truth for "fully optimized", and - # `fts_optimize_available()` keeps offering the resume until the - # transition actually completes. + # Stamp the FTS layout version (fresh/optimized DBs) so the main + # version can always advance; a legacy DB keeps its absent/0 marker + # until optimize-storage runs. An INTERRUPTED optimize (rebuild + # markers, trash tables, or an empty external index against + # non-empty messages) is NOT stamped: the marker is the source of + # truth for "fully optimized" and keeps the resume offer alive. if ( fts5_available and not self._db_has_legacy_inline_fts(cursor) @@ -1422,12 +1165,10 @@ class SessionSchemaMixin: "fts_storage_version", str(FTS_STORAGE_VERSION), cursor=cursor ) - # Advance schema_version to current for ALL non-FTS-layout - # migrations. This is deliberately NOT gated on the FTS opt-in — - # holding the whole version back would block every future schema - # migration for a user who never optimizes. FTS5 being unavailable - # is the one case we skip (we can't have created the current FTS - # objects, so claiming the current schema would be a lie). + # Advance schema_version — deliberately NOT gated on the FTS opt-in + # (that would block every future migration for a user who never + # optimizes). FTS5 unavailable is the one skip: we can't have + # created the current FTS objects, so claiming current would lie. if ( current_version < SCHEMA_VERSION and fts_migrations_complete @@ -1438,9 +1179,8 @@ class SessionSchemaMixin: (SCHEMA_VERSION,), ) - # Unique title index — always ensure it exists. Older databases may - # contain duplicate aliases from before the constraint was enforced; - # preserve every session while letting the newest one retain the alias. + # Unique title index. Older DBs may hold duplicate aliases from before + # the constraint; keep every session, the newest retains the alias. title_index_sql = ( "CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique " "ON sessions(title) WHERE title IS NOT NULL" @@ -1448,8 +1188,7 @@ class SessionSchemaMixin: try: cursor.execute(title_index_sql) except sqlite3.IntegrityError: - # The index is an optimization — its creation must never abort - # opening the database, so the repair itself is also guarded. + # The index must never abort opening the DB; guard the repair too. try: cursor.execute( """UPDATE sessions AS older @@ -1475,30 +1214,28 @@ class SessionSchemaMixin: pass # Index already exists if fts5_available: - # FTS5 setup. Run the DDL even when the virtual table exists so - # CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation from - # an earlier no-FTS5 runtime. - # - # OPT-IN v23 boundary: a legacy v22 install (inline-content FTS, - # not yet opted into `hermes db optimize`) must keep its EXISTING - # inline schema + triggers. Running the v23 external-content DDL - # here would create the trigram source VIEW and leave the DB in a - # mixed inline/external state. So for a legacy DB we only ensure - # its inline triggers exist (via the legacy DDL), and skip the - # v23 view/external tables entirely. Fresh installs and opted-in - # DBs have no legacy inline FTS, so they get the v23 DDL. + # Run the FTS DDL even when the vtable exists so CREATE TRIGGER IF + # NOT EXISTS repairs trigger-only degradation from a no-FTS5 runtime. + # OPT-IN v23 boundary: a legacy v22 inline install must keep its + # inline schema + triggers — the v23 external-content DDL would + # create the trigram source VIEW and leave a mixed state — so it + # gets the legacy DDL only; fresh/opted-in DBs get v23. legacy_fts = self._db_has_legacy_inline_fts(cursor) if self._fts_stale: if self._recover_stale_fts(cursor, legacy=legacy_fts): - # CJK was detached alongside the corrupt base indexes and - # has its own stale marker. Its existing ensure path keeps - # it offline until its dedicated rebuild. + # CJK was detached alongside the base indexes and has its + # own stale marker; its ensure path decides when it returns. self._ensure_fts_cjk_schema(cursor) else: self._fts_enabled = False self._trigram_available = False self._fts_cjk_available = False - elif legacy_fts: + else: + base_sql, trigram_sql, rebuild = ( + (LEGACY_FTS_SQL, LEGACY_FTS_TRIGRAM_SQL, self._rebuild_legacy_fts_indexes) + if legacy_fts + else (FTS_SQL, FTS_TRIGRAM_SQL, self._rebuild_fts_indexes) + ) # Measure BEFORE the DDL below runs, so these describe the # pre-repair state. Whether the trigram half is even # creatable is only known AFTER _ensure_fts_schema, which is @@ -1512,11 +1249,14 @@ class SessionSchemaMixin: < len(_FTS_TRIGRAM_TRIGGERS) ) self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", LEGACY_FTS_SQL + cursor, "messages_fts", base_sql ) if self._fts_enabled: + # Trigram FTS5 for CJK/substring search is optional + # relative to the main table; if it cannot be created, + # CJK search falls back to LIKE. trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", LEGACY_FTS_TRIGRAM_SQL + cursor, "messages_fts_trigram", trigram_sql ) self._trigram_available = trigram_enabled if base_triggers_missing or ( @@ -1524,49 +1264,16 @@ class SessionSchemaMixin: ): self._run_admitted_startup_rebuild( cursor, - lambda: self._rebuild_legacy_fts_indexes( - cursor, include_trigram=trigram_enabled - ), + lambda: rebuild(cursor, include_trigram=trigram_enabled), ) - else: - # Same split as the legacy branch above, same reason. - base_triggers_missing = ( - self._fts_trigger_count(cursor, _FTS_BASE_TRIGGERS) - < len(_FTS_BASE_TRIGGERS) - ) - trigram_triggers_missing = ( - self._fts_trigger_count(cursor, _FTS_TRIGRAM_TRIGGERS) - < len(_FTS_TRIGRAM_TRIGGERS) - ) - self._fts_enabled = self._ensure_fts_schema( - cursor, "messages_fts", FTS_SQL - ) - - # Trigram FTS5 for CJK/substring search. This is optional - # relative to the main FTS table; if it cannot be created, - # CJK search falls back to LIKE. - if self._fts_enabled: - trigram_enabled = self._ensure_fts_schema( - cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL - ) - self._trigram_available = trigram_enabled - if base_triggers_missing or ( - trigram_enabled and trigram_triggers_missing - ): - self._run_admitted_startup_rebuild( - cursor, - lambda: self._rebuild_fts_indexes( - cursor, - include_trigram=trigram_enabled, - ), - ) - # CJK-bigram index (cjk_unicode61). Strictly additive to - # the surfaces above and gated on the loadable tokenizer: - self._ensure_fts_cjk_schema(cursor) + if not legacy_fts: + # CJK-bigram index (cjk_unicode61): strictly additive + # and gated on the loadable tokenizer. + self._ensure_fts_cjk_schema(cursor) # Replace any pre-existing broad AFTER UPDATE triggers with # AFTER UPDATE OF variants. IF NOT EXISTS cannot rewrite them. - if getattr(self, "_fts_enabled", False): + if self._fts_enabled: self._migrate_broad_fts_update_triggers(cursor) self._conn.commit() @@ -1574,24 +1281,18 @@ class SessionSchemaMixin: def _run_admitted_startup_rebuild(self, cursor, rebuild_fn) -> None: """Run a full trigger-repair FTS rebuild under cross-process admission. - ``_init_schema`` reaches here when the sync triggers were missing and - the DDL just recreated them, so the index has a gap of unknown extent - and must be rebuilt in full. Two processes opening the same DB after - an update commonly hit this path simultaneously — the exact + Reached when the sync triggers were missing and the DDL just recreated + them: the index has a gap of unknown extent. Two processes opening the + same DB after an update commonly hit this simultaneously — the concurrent-rebuild interleaving that structurally corrupted state.db - in production (PR #93200) — so the rebuild admits through - ``fts_rebuild_admission`` and FAILS CLOSED. - - On deferral (another process holds the rebuild authority) the - just-repaired triggers are dropped again and the durable stale - breadcrumb is persisted, mirroring ``_enter_fts_fail_open``'s - ordering contract: triggers must never be live over an index with an - unrebuilt gap. FTS stays detached for this instance; the winner's - rebuild — or ``retry_deferred_fts_recovery`` from the gateway - housekeeping tick, or ``_recover_stale_fts`` at the next startup — restores - the index and triggers atomically. + in production — so this admits through ``fts_rebuild_admission`` and + FAILS CLOSED. On deferral the just-repaired triggers are dropped again + and the stale breadcrumb persisted (``_enter_fts_fail_open``'s + ordering contract: triggers must never be live over an unrebuilt gap); + the winner's rebuild, ``retry_deferred_fts_recovery``, or + ``_recover_stale_fts`` at next startup restores index and triggers. """ - with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: + with fts_rebuild_admission(self.db_path) as admitted: if admitted: rebuild_fn() return @@ -1615,12 +1316,7 @@ class SessionSchemaMixin: self, cursor: sqlite3.Cursor ) -> None: """One-time v18 backfill of gateway metadata from sessions.json. - - Existing gateway sessions predate the display_name / origin_json / - expiry_finalized columns; copy what sessions.json knows so consumers - can switch to state.db without losing pre-migration sessions. - Only fills NULL columns — never overwrites data written by newer code. - """ + Only fills NULL columns — never overwrites data written by newer code.""" sessions_file = get_hermes_home() / "sessions" / "sessions.json" if not sessions_file.exists(): return diff --git a/hermes_state_search.py b/hermes_state_search.py index 3dafeebc4a..7abfe3801f 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -1,11 +1,9 @@ """Full-text / trigram / CJK message search and FTS maintenance for SessionDB. -Mixin contract: this is a plain mixin class consumed by -``hermes_state.SessionDB``. It defines no ``__init__`` and no state of its -own; methods access the host's attributes (``self._conn``, ``self.db_path``, -``self._execute_write`` and other SessionDB methods) established by -``SessionDB.__init__``. It must never import hermes_state (cycle) — shared -module-level constants live in hermes_state_common. +Plain mixin consumed by ``hermes_state.SessionDB``: no ``__init__``, no state +of its own; methods use host attributes established by ``SessionDB.__init__``. +Must never import hermes_state (cycle) — shared constants live in +hermes_state_common. """ import logging @@ -30,22 +28,78 @@ from hermes_state_common import ( fts_rebuild_admission, ) -# Moved methods logged under the "hermes_state" logger before the split; -# keep that logger identity so log filtering/capture behavior is unchanged. +# Keep the pre-split logger identity so log filtering/capture is unchanged. logger = logging.getLogger("hermes_state") # Characters FTS5's query grammar rejects outside a quoted phrase. Anything -# missing from this set reaches MATCH raw and raises, which the execute site -# swallows into zero results — the failure this strip step exists to prevent. -# Assembled through re.escape so the backslash cannot be eaten as a regex -# escape inside the class (it was, while the set was written as a literal). -# -# ``%`` is deliberately excluded: a CJK query falls back to a LIKE search that -# needs it preserved as a literal (that path escapes wildcards itself), so -# stripping it here widened those queries onto unrelated rows. +# missing here reaches MATCH raw and raises, which the execute site swallows +# into zero results. Assembled through re.escape so the backslash cannot be +# eaten as a regex escape. ``%`` is deliberately excluded: the CJK LIKE +# fallback needs it as a literal (that path escapes wildcards itself). _FTS5_SPECIAL_CHARS = '+{}():"^@/#&|~[]<>,;!?$=\\\'' _FTS5_SPECIAL_RE = re.compile(f"[{re.escape(_FTS5_SPECIAL_CHARS)}]") +_FTS_OPERATORS = frozenset({"AND", "OR", "NOT"}) + +# Column list shared by every search route (snippet + metadata, never content). +_SEARCH_SELECT_TAIL = ( + "m.timestamp, m.tool_name, s.source, s.model, s.started_at AS session_started" +) +_LIKE_SNIPPET_SQL = "substr(m.content, max(1, instr(m.content, ?) - 40), 120) AS snippet" +_LIKE_ANY_COLUMN_SQL = ( + "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " + "OR m.tool_calls LIKE ? ESCAPE '\\')" +) + + +def _meta_row(conn, key: str) -> Optional[sqlite3.Row]: + """Point-read one ``state_meta`` row (``None`` when absent).""" + return conn.execute( + "SELECT value FROM state_meta WHERE key = ?", (key,) + ).fetchone() + + +def _delete_meta(conn, *keys: str) -> None: + conn.execute( + f"DELETE FROM state_meta WHERE key IN ({','.join('?' for _ in keys)})", keys + ) + + +def _quote_fts_tokens(raw_query: str) -> str: + """Quote each non-operator token (neutralising FTS5 special characters) + while preserving AND/OR/NOT.""" + return " ".join( + tok if tok.upper() in _FTS_OPERATORS else '"' + tok.replace('"', '""') + '"' + for tok in raw_query.split() + ) + + +def _search_filter_clauses( + where: List[str], + params: list, + *, + include_inactive: bool, + source_filter: Optional[List[str]], + exclude_sources: Optional[List[str]], + role_filter: Optional[List[str]], +) -> None: + """Append the visibility/source/role predicates every search route shares. + + Live rows (active=1) AND compaction-archived rows (compacted=1) are + discoverable; only rewind/undo rows (active=0, compacted=0) are hidden. + """ + if not include_inactive: + where.append("(m.active = 1 OR m.compacted = 1)") + if source_filter is not None: + where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") + params.extend(source_filter) + if exclude_sources is not None: + where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") + params.extend(exclude_sources) + if role_filter: + where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") + params.extend(role_filter) + class SessionSearchMixin: """See module docstring — mixin for SessionDB (Search cluster).""" @@ -89,35 +143,31 @@ class SessionSearchMixin: max_pages=self._FTS_MERGE_MAX_PAGES_PER_INDEX ) except Exception as exc: # noqa: BLE001 - post-commit maintenance - # The canonical write is already committed before this cadence - # runs. No maintenance failure — including the bare SystemError - # the CPython sqlite3 layer can raise under cross-thread errmsg - # scrambling — may escape and make the caller replay an - # ambiguous, possibly-durable write (#90734, #85079). + # The canonical write is already committed. No maintenance failure + # — including the bare SystemError CPython's sqlite3 layer can raise + # under cross-thread errmsg scrambling — may escape and make the + # caller replay an ambiguous, possibly-durable write. logger.warning("FTS incremental merge failed after commit: %s", exc) def fts_rebuild_status(self) -> Optional[Dict[str, Any]]: - """Return deferred-rebuild progress, or None when no rebuild pending. + """Deferred-rebuild progress ``{"pending", "total", "indexed", + "percent"}``, or None when no rebuild is pending. - Shape: {"pending": True, "total": , - "indexed": , "percent": <0-100 int>}. - Consumed by search_messages() notes and by status surfaces - (dashboard/desktop can poll this to render a progress indicator). - - Reads state_meta directly via _read_ctx instead of calling - get_meta() (which takes self._lock) so search_messages doesn't - block on the writer lock when checking rebuild status. + Reads state_meta via the pooled reader rather than get_meta() (which + takes self._lock) so search_messages never blocks on the writer lock. """ - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_rebuild_high_water", "fts_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_rebuild_high_water") + return self._rebuild_status("fts_rebuild") + + def _rebuild_status(self, prefix: str) -> Optional[Dict[str, Any]]: + rows = self._read_all( + "SELECT key, value FROM state_meta WHERE key IN (?, ?)", + (f"{prefix}_high_water", f"{prefix}_progress"), + ) + meta = {r["key"]: r["value"] for r in rows} + high_water = meta.get(f"{prefix}_high_water") if high_water is None: return None - progress = int(meta.get("fts_rebuild_progress") or 0) + progress = int(meta.get(f"{prefix}_progress") or 0) total = int(high_water) if total <= 0: return None @@ -127,69 +177,53 @@ class SessionSearchMixin: def _fts_rebuild_finish(self) -> None: """Finalize the deferred rebuild: boundary sweep + clear markers. - The sweep is cheap insurance against any write that slipped through - the migration-boundary instant (between high_water capture and - trigger activation): re-index any row near the boundary that the - index is missing. docsize has one row per indexed doc, so the - anti-join is exact and runs on a narrow id range. - - The trigram half of the sweep is gated on ``self._trigram_available`` - for the same reason ``fts_rebuild_step()`` gates its backfill INSERT: - when the SQLite build has no trigram tokenizer (or the table was - never created), an unconditional INSERT raises ``no such table`` - and aborts the whole rebuild — taking ``optimize_fts_storage()`` - down with it. + The sweep is cheap insurance against a write that slipped through the + migration-boundary instant (between high_water capture and trigger + activation). The trigram half is gated on ``_trigram_available``: + without the tokenizer/table an unconditional INSERT raises ``no such + table`` and aborts the whole rebuild (and optimize_fts_storage()). """ - include_trigram = self._trigram_available - - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - # Sweep a generous window around the boundary. - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - if include_trigram: - conn.execute( - "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_rebuild_high_water', 'fts_rebuild_progress')" - ) - self._execute_write(_do) + sweeps = [self._BOUNDARY_SWEEP_SQL.format(table="messages_fts", extra="")] + if self._trigram_available: + sweeps.append(self._BOUNDARY_SWEEP_SQL.format( + table="messages_fts_trigram", extra="AND m.role <> 'tool' " + )) + self._rebuild_finish("fts_rebuild", sweeps) logger.info("Deferred FTS rebuild complete — all messages indexed.") + # Re-index rows in an id window the index is missing. docsize has one row + # per indexed doc, so the anti-join is exact. + _BOUNDARY_SWEEP_SQL = ( + "INSERT INTO {table}(rowid, content, tool_name, tool_calls) " + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m " + "WHERE m.id > ? AND m.id <= ? {extra}" + "AND NOT EXISTS (SELECT 1 FROM {table}_docsize d WHERE d.id = m.id)" + ) + + def _rebuild_finish(self, prefix: str, sweep_sqls: List[str]) -> None: + """Sweep a generous window around the high-water boundary, then clear + the ``{prefix}_high_water`` / ``{prefix}_progress`` markers.""" + def _do(conn): + hw_row = _meta_row(conn, f"{prefix}_high_water") + if hw_row is not None: + hw = int(hw_row[0]) + for sql in sweep_sqls: + conn.execute(sql, (hw - 1000, hw + 1000)) + _delete_meta(conn, f"{prefix}_high_water", f"{prefix}_progress") + self._execute_write(_do) + def _fts_teardown_trash_step(self) -> bool: - """Tear down one chunk of a demoted v22 FTS shadow table. + """Tear down one chunk of a demoted v22 FTS shadow table; True while + work remains. - The trash tables are PLAIN tables (their vtable parent was demoted - away during the migration), so chunked DELETE + final DROP involve - no FTS5 machinery at all. Returns True while teardown work remains. - - Single-column-key trash tables (the common shape — FTS shadow - tables carry a rowid/integer PK) are drained with a high-water - marker mirroring :meth:`fts_rebuild_step`: each chunk deletes only - rows after the previously-drained key, so the per-chunk scan is - bounded instead of re-scanning from the start of the table every - chunk (O(n²) total on large trash tables, #79324). Compound-key - trash tables (multi-column PK) cannot use a scalar high-water - comparison, so they keep the legacy chunked ``LIMIT`` delete — - those shadow tables are small by construction. + Trash tables are PLAIN tables (their vtable parent was demoted), so + chunked DELETE + final DROP involve no FTS5 machinery. Integer + single-column-key tables are drained with a high-water marker so each + chunk's scan is bounded (re-scanning from the start was O(n²) on + large tables). Compound-key tables cannot use a scalar high-water + comparison and keep the chunked ``LIMIT`` delete — they are small by + construction. """ with self._lock: trash = [ @@ -214,18 +248,13 @@ class SessionSearchMixin: key = ", ".join(pk_cols) if pk_cols else "rowid" if len(pk_cols) == 1 and (not pk_info or pk_info[0][1] == "INTEGER"): - # High-water drain: delete only rows past the marker key. - # The marker is read/written inside the same BEGIN IMMEDIATE - # transaction as the DELETE, so concurrent callers claim - # disjoint key ranges instead of re-deleting. Only integer - # PKs can anchor a numeric high-water comparison — the FTS - # config shadow table (TEXT pk like 'version') falls back to - # the legacy chunked delete below. + # High-water drain. The marker is read/written in the same + # BEGIN IMMEDIATE as the DELETE, so concurrent callers claim + # disjoint key ranges. Only integer PKs can anchor the numeric + # comparison — the config shadow table (TEXT pk) falls through + # to the chunked delete below. marker_key = f"fts_teardown_{tbl}_progress" - row = conn.execute( - "SELECT value FROM state_meta WHERE key = ?", - (marker_key,), - ).fetchone() + row = _meta_row(conn, marker_key) high_water = int(row[0]) if row is not None else 0 # Claim the chunk's upper bound: the LAST row of the @@ -238,9 +267,7 @@ class SessionSearchMixin: if not upper_rows: # Drained — the DROP is cheap now. conn.execute(f"DROP TABLE IF EXISTS {tbl}") - conn.execute( - "DELETE FROM state_meta WHERE key = ?", (marker_key,) - ) + _delete_meta(conn, marker_key) logger.info("Old FTS shadow table %s torn down.", tbl) return True @@ -250,17 +277,11 @@ class SessionSearchMixin: (high_water, upper), ) if cur.rowcount > 0: - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (marker_key, str(upper)), - ) + self.set_meta(marker_key, str(upper), cursor=conn) return True - # Compound-key or rowid trash table: legacy chunked delete. - # These shadow tables are small, so the quadratic re-scan is - # not a concern (#79324 keeps the high-water path for the big - # single-key tables). + # Compound-key or rowid trash table: chunked delete (small tables, + # quadratic re-scan is not a concern). cur = conn.execute( f"DELETE FROM {tbl} WHERE ({key}) IN " f"(SELECT {key} FROM {tbl} LIMIT {self._FTS_REBUILD_CHUNK_ROWS})" @@ -278,29 +299,44 @@ class SessionSearchMixin: return True def fts_rebuild_step(self) -> bool: - """Backfill one chunk of the deferred FTS rebuild. - - Returns True when more work remains, False when the rebuild is - complete (or none is pending). Safe to call from any process at any - time; chunks are claimed atomically inside the write transaction, so - concurrent callers interleave instead of duplicating rows. - """ + """Backfill one chunk of the deferred FTS rebuild; True while work + remains. Safe from any process: chunks are claimed atomically inside + the write transaction, so concurrent callers interleave instead of + duplicating rows.""" if not self._fts_enabled: return False - high_water_raw = self.get_meta("fts_rebuild_high_water") + inserts = [self._CHUNK_INSERT_SQL.format(table="messages_fts", extra="")] + if self._trigram_available: + inserts.append(self._CHUNK_INSERT_SQL.format( + table="messages_fts_trigram", extra=" AND role <> 'tool'" + )) + return self._rebuild_step( + "fts_rebuild", inserts, + fail_msg="FTS rebuild chunk failed (will retry): %s", + finish=self._fts_rebuild_finish, + ) + + _CHUNK_INSERT_SQL = ( + "INSERT INTO {table}(rowid, content, tool_name, tool_calls) " + "SELECT id, content, tool_name, tool_calls FROM messages " + "WHERE id > ? AND id <= ?{extra}" + ) + + def _rebuild_step( + self, prefix: str, insert_sqls: List[str], *, fail_msg: str, finish + ) -> bool: + """Shared chunk engine for the base and CJK deferred backfills.""" + high_water_raw = self.get_meta(f"{prefix}_high_water") if high_water_raw is None: return False high_water = int(high_water_raw) - include_trigram = self._trigram_available chunk = self._FTS_REBUILD_CHUNK_ROWS def _do(conn): # Re-read progress inside the write transaction (BEGIN IMMEDIATE # is already held by _execute_write) — this is the claim: two # workers can't read the same progress value concurrently. - row = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" - ).fetchone() + row = _meta_row(conn, f"{prefix}_progress") if row is None: return False # finished (or cleared) by another process progress = int(row[0]) @@ -310,179 +346,87 @@ class SessionSearchMixin: # The chunk upper bound is an id, not a row count, so gaps from # deleted rows don't shrink chunks below the claimed range. upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ?", - (progress, upper), - ) - if include_trigram: - conn.execute( - "INSERT INTO messages_fts_trigram" - "(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) + for sql in insert_sqls: + conn.execute(sql, (progress, upper)) # Publish progress in the same transaction as the rows it # covers — crash-atomic: either both land or neither does. conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_rebuild_progress'", - (str(upper),), + "UPDATE state_meta SET value = ? WHERE key = ?", + (str(upper), f"{prefix}_progress"), ) return upper < high_water try: more = self._execute_write(_do) except sqlite3.OperationalError as exc: - logger.debug("FTS rebuild chunk failed (will retry): %s", exc) + logger.debug(fail_msg, exc) return True # transient (lock contention) — caller retries if more is False: - status = self.fts_rebuild_status() + status = self._rebuild_status(prefix) if status is not None and status["indexed"] >= status["total"]: - self._fts_rebuild_finish() + finish() return False return bool(more) def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]: """CJK-index backfill progress, or None when none is pending.""" - with self._read_ctx() as conn: - row = conn.execute( - "SELECT key, value FROM state_meta WHERE key IN (?, ?)", - ("fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress"), - ).fetchall() - meta = {r["key"]: r["value"] for r in row} - high_water = meta.get("fts_cjk_rebuild_high_water") - if high_water is None: - return None - progress = int(meta.get("fts_cjk_rebuild_progress") or 0) - total = int(high_water) - if total <= 0: - return None - pct = min(100, int(100 * progress / total)) - return {"pending": True, "total": total, "indexed": progress, "percent": pct} + return self._rebuild_status("fts_cjk_rebuild") def fts_cjk_rebuild_step(self) -> bool: """Backfill one chunk of the CJK index. True while work remains.""" if not self._fts_enabled or not self._fts_cjk_loaded: return False - high_water_raw = self.get_meta("fts_cjk_rebuild_high_water") - if high_water_raw is None: - return False - high_water = int(high_water_raw) - chunk = self._FTS_REBUILD_CHUNK_ROWS - - def _do(conn): - row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_progress'" - ).fetchone() - if row is None: - return False # finished (or cleared) by another process - progress = int(row[0]) - if progress >= high_water: - return False - upper = min(progress + chunk, high_water) - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", - (progress, upper), - ) - conn.execute( - "UPDATE state_meta SET value = ? " - "WHERE key = 'fts_cjk_rebuild_progress'", - (str(upper),), - ) - return upper < high_water - - try: - more = self._execute_write(_do) - except sqlite3.OperationalError as exc: - logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc) - return True - if more is False: - status = self.fts_cjk_rebuild_status() - if status is not None and status["indexed"] >= status["total"]: - self._fts_cjk_rebuild_finish() - return False - return bool(more) + return self._rebuild_step( + "fts_cjk_rebuild", + [self._CHUNK_INSERT_SQL.format( + table="messages_fts_cjk", extra=" AND role <> 'tool'" + )], + fail_msg="CJK FTS rebuild chunk failed (will retry): %s", + finish=self._fts_cjk_rebuild_finish, + ) def _fts_cjk_rebuild_finish(self) -> None: """Boundary sweep + clear the cjk markers; index becomes servable.""" - def _do(conn): - hw_row = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_cjk_rebuild_high_water'" - ).fetchone() - if hw_row is not None: - hw = int(hw_row[0]) - lo, hi = hw - 1000, hw + 1000 - conn.execute( - "INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) " - "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " - "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)", - (lo, hi), - ) - conn.execute( - "DELETE FROM state_meta WHERE key IN " - "('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')" + self._rebuild_finish("fts_cjk_rebuild", [ + self._BOUNDARY_SWEEP_SQL.format( + table="messages_fts_cjk", extra="AND m.role <> 'tool' " ) - self._execute_write(_do) + ]) self._fts_cjk_available = True logger.info("CJK FTS index backfill complete — serving CJK search.") def _fts_cjk_reset_if_stale(self) -> None: - """Rebuild path for a stale cjk index (triggers were dropped). - - The gap's extent is unknown, so the only safe recovery is a from- - scratch rebuild: drop the table + triggers, clear the breadcrumb, - recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill - markers on a populated DB). Called from ``optimize_fts_storage`` on - a tokenizer-capable host; no-op when not stale. - """ + """From-scratch rebuild of a stale cjk index (triggers were dropped, + gap extent unknown): drop table + triggers, clear the breadcrumb, + recreate via ``_ensure_fts_cjk_schema`` (fresh backfill markers on a + populated DB). No-op when not stale.""" if not self._fts_cjk_loaded: return def _do(conn): - stale = conn.execute( - "SELECT 1 FROM state_meta WHERE key = ?", - (FTS_CJK_STALE_KEY,), - ).fetchone() - if not stale: + if _meta_row(conn, FTS_CJK_STALE_KEY) is None: return False for trig in _FTS_CJK_TRIGGERS: conn.execute(f"DROP TRIGGER IF EXISTS {trig}") conn.execute("DROP TABLE IF EXISTS messages_fts_cjk") conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src") - conn.execute( - "DELETE FROM state_meta WHERE key IN " - f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', " - "'fts_cjk_rebuild_progress')" + _delete_meta( + conn, FTS_CJK_STALE_KEY, "fts_cjk_rebuild_high_water", "fts_cjk_rebuild_progress" ) return True was_stale = self._execute_write(_do) if was_stale: # Recreate outside the write transaction — _ensure_fts_cjk_schema - # uses executescript(), which implicitly commits any pending - # transaction and must not run inside _execute_write's BEGIN - # IMMEDIATE. Sets fresh backfill markers on a populated DB. + # uses executescript(), which implicitly commits and must not run + # inside _execute_write's BEGIN IMMEDIATE. with self._lock: self._ensure_fts_cjk_schema(self._conn) self._conn.commit() def _fts_external_index_empty_with_messages(self, conn) -> bool: - """True when the base FTS table exists but indexes nothing while - ``messages`` has rows. Caller must hold ``self._lock``. - - This is the post-demote empty-index shape: external-content FTS with - zero ``messages_fts_docsize`` rows against a non-empty messages table. - Healthy installs (and mid-backfill installs that still hold markers) - never match. - """ + """True when the base FTS table indexes nothing while ``messages`` has + rows (the post-demote empty-index shape). Caller holds ``self._lock``. + Healthy and mid-backfill installs never match.""" try: has_msg = conn.execute( "SELECT EXISTS(SELECT 1 FROM messages)" @@ -490,11 +434,9 @@ class SessionSearchMixin: if not has_msg: return False # docsize is the authoritative "is this rowid indexed" surface for - # external-content FTS5; probing the virtual table itself is - # not reliable across SQLite builds. EXISTS instead of COUNT(*): - # this runs on every writable open via the _init_schema stamp - # condition, and COUNT(*) is a full b-tree scan (~100ms on a - # 2M-row table) while EXISTS is O(1). + # external-content FTS5 (probing the vtable is unreliable across + # builds). EXISTS not COUNT(*): this runs on every writable open + # and COUNT(*) is a full b-tree scan. has_fts = conn.execute( "SELECT EXISTS(SELECT 1 FROM messages_fts_docsize)" ).fetchone()[0] @@ -504,11 +446,8 @@ class SessionSearchMixin: return False def _fts_index_known_empty(self, conn) -> bool: - """True when the base external-content index holds no rows. - - A missing table counts as empty: the schema ensure that follows - creates it fresh. - """ + """True when the base external-content index holds no rows; a missing + table counts as empty (the schema ensure that follows creates it).""" try: n = conn.execute( "SELECT COUNT(*) FROM messages_fts_docsize" @@ -518,18 +457,14 @@ class SessionSearchMixin: return True def _reset_fts_index_to_empty(self, conn) -> None: - """Delete every indexed row from the v23 external-content tables. + """Truncate the v23 external-content tables via FTS5 ``'delete-all'``. - Uses the FTS5 ``'delete-all'`` special command — the documented O(1) - truncate for external-content tables. A plain no-WHERE ``DELETE`` is - O(rows) on external-content FTS5 (each row's delete tokens are - regenerated from the content table; measured ~12µs/row, minutes on a - large index, while holding the write lock) and corrupts the index if - indexed rows have diverged from ``messages`` — precisely the broken- - bookkeeping shape this repair path handles. The backfill chunk worker - replays its whole selected id range with no anti-join, so a replay - from zero is only safe once the index is known empty — this is how a - partially indexed DB gets there. + A plain ``DELETE`` is O(rows) on external-content FTS5 (minutes on a + large index, holding the write lock) and corrupts the index when + indexed rows have diverged from ``messages`` — precisely the shape + this repair handles. The backfill worker replays its id range with no + anti-join, so a replay from zero is only safe once the index is known + empty; this is how a partially indexed DB gets there. """ for tbl in ("messages_fts", "messages_fts_trigram"): try: @@ -539,90 +474,49 @@ class SessionSearchMixin: def _seed_fts_rebuild_markers(self, conn, *, force: bool = False) -> int: """Write ``fts_rebuild_high_water`` / ``fts_rebuild_progress`` for a - full backfill. Returns the high-water id. + full backfill; returns the high-water id. - When ``force`` is False and high_water is already set, only repairs a - missing progress key (stuck no-op when high_water exists alone), and - only after the index is known empty: the chunk worker replays its - whole selected id range without an anti-join, so a partially indexed - DB is first reset to a known-empty surface rather than rebuilt from - zero on top of surviving rows. Caller must hold the write - transaction / lock as appropriate. + Without ``force`` and with high_water already set, only repairs a + missing progress key, resetting a partially indexed DB to a + known-empty surface first (the chunk worker replays without an + anti-join). Caller holds the write transaction. """ - existing_hw = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_high_water'" - ).fetchone() + existing_hw = _meta_row(conn, "fts_rebuild_high_water") if existing_hw is not None and not force: - hw = int(existing_hw[0]) - progress = conn.execute( - "SELECT value FROM state_meta WHERE key = 'fts_rebuild_progress'" - ).fetchone() - if progress is None: + if _meta_row(conn, "fts_rebuild_progress") is None: # high_water without progress: fts_rebuild_step treats missing # progress as "done by another process" and optimize would # no-op then stamp. Re-seed progress so the chunk loop runs. if not self._fts_index_known_empty(conn): self._reset_fts_index_to_empty(conn) - conn.execute( - "INSERT INTO state_meta (key, value) VALUES " - "('fts_rebuild_progress', '0') " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value" - ) - return hw + self.set_meta("fts_rebuild_progress", "0", cursor=conn) + return int(existing_hw[0]) - hw = conn.execute( - "SELECT COALESCE(MAX(id), 0) FROM messages" - ).fetchone()[0] - for k, v in ( - ("fts_rebuild_high_water", str(hw)), - ("fts_rebuild_progress", "0"), - ): - conn.execute( - "INSERT INTO state_meta (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (k, v), - ) + hw = conn.execute("SELECT COALESCE(MAX(id), 0) FROM messages").fetchone()[0] + self.set_meta("fts_rebuild_high_water", str(hw), cursor=conn) + self.set_meta("fts_rebuild_progress", "0", cursor=conn) return int(hw) def _repair_optimize_bookkeeping(self) -> None: """Heal interrupted demote/backfill bookkeeping before optimize runs. - Covers two post-#65798 failure classes: + 1. Empty external-content index with messages and no markers (demote + crash window / settle without backfill): seed a full backfill. + 2. high_water without progress: seed progress (resetting a partially + populated index first so the anti-join-free replay cannot + duplicate rows). - 1. Empty external-content index with messages present and no rebuild - markers (demote crash window after empty v23 tables landed but - before markers, or settle that stamped without backfill). Seed a - full backfill. - 2. ``fts_rebuild_high_water`` present without ``fts_rebuild_progress`` - (partial meta) — seed progress so the chunk loop is not a no-op, - resetting a partially populated index to a known-empty surface - first so the anti-join-free chunk replay cannot duplicate rows. - - Must not invent markers on a still-legacy inline DB: that would make - ``optimize_fts_storage`` skip demote (``legacy and not pending``) and - attempt v23-shaped INSERTs against the inline table forever. + Must not invent markers on a still-legacy inline DB: optimize would + then skip demote and attempt v23 INSERTs against the inline table forever. """ def _do(conn): - existing_hw = conn.execute( - "SELECT value FROM state_meta " - "WHERE key = 'fts_rebuild_high_water'" - ).fetchone() - - if existing_hw is not None: + if _meta_row(conn, "fts_rebuild_high_water") is not None: # Repair orphan high_water-without-progress only. Never # invent a fresh claim on a healthy complete index. - progress = conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_progress'" - ).fetchone() - if progress is None: + if _meta_row(conn, "fts_rebuild_progress") is None: if not self._fts_index_known_empty(conn): self._reset_fts_index_to_empty(conn) - conn.execute( - "INSERT INTO state_meta (key, value) VALUES " - "('fts_rebuild_progress', '0') " - "ON CONFLICT(key) DO UPDATE SET value = '0'" - ) + self.set_meta("fts_rebuild_progress", "0", cursor=conn) return # No markers. On a still-legacy DB demote owns marker creation. @@ -632,65 +526,48 @@ class SessionSearchMixin: # Non-legacy empty external index (demote crash window / premature # stamp): seed a full backfill claim. if self._fts_external_index_empty_with_messages(conn): - conn.execute( - "DELETE FROM state_meta WHERE key = 'fts_storage_version'" - ) + _delete_meta(conn, "fts_storage_version") self._seed_fts_rebuild_markers(conn, force=True) self._execute_write(_do) def fts_optimize_available(self) -> bool: - """True when `optimize_fts_storage()` has work to do: either this DB - is a legacy inline-FTS install that can be optimized to the v23 - external-content schema, or a previous optimize run was interrupted - (legacy vtables already demoted, but backfill markers and/or trash - tables remain) and re-running would resume it, or the CJK-bigram - index needs a backfill/rebuild on this tokenizer-capable host, or - a prior demote left an empty external-content index without markers - (healable on re-run). - False for fresh and fully-optimized installs (and when FTS5 is - unavailable).""" + """True when `optimize_fts_storage()` has work: legacy inline FTS to + migrate, an interrupted optimize to resume (markers/trash remain), a + CJK-bigram backfill/rebuild on this tokenizer-capable host, or an + empty external index left without markers. False for fresh and + fully-optimized installs and when FTS5 is unavailable.""" if not self._fts_enabled or self.read_only: return False with self._lock: if self._db_has_legacy_inline_fts(self._conn): return True - # Interrupted optimize: demotion already removed the legacy - # vtables (so the check above is False), but the transition is - # unfinished until the backfill markers are cleared and the - # demoted trash tables are torn down. Search stays complete - # through the gap supplement meanwhile; re-running resumes. - if self._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone(): + # Interrupted optimize: legacy vtables already demoted, but the + # transition is unfinished until markers clear and trash is gone. + if _meta_row(self._conn, "fts_rebuild_high_water") is not None: return True # CJK-bigram index work — only offerable when THIS process can # tokenize: a pending backfill (markers set at creation on a # populated DB) or a stale index awaiting a from-scratch rebuild. - if self._fts_cjk_loaded and self._conn.execute( - "SELECT 1 FROM state_meta WHERE key IN " - f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1" - ).fetchone(): + if self._fts_cjk_loaded and ( + _meta_row(self._conn, "fts_cjk_rebuild_high_water") is not None + or _meta_row(self._conn, FTS_CJK_STALE_KEY) is not None + ): return True if self._has_fts_trash(self._conn): return True - # Pre-fix crash window: empty external-content index with - # messages still present, no markers, no trash (teardown already - # finished or never needed). Re-run seeds markers and backfills. + # Crash window: empty external index, messages present, no markers, + # no trash. Re-run seeds markers and backfills. return self._fts_external_index_empty_with_messages(self._conn) def _demote_legacy_fts_to_trash(self) -> int: """Demote the legacy inline FTS vtables and stage their shadow tables - for chunked teardown. Returns MAX(messages.id) as the rebuild high - water. O(1) schema surgery — the heavy delete is deferred to the - chunked teardown, exactly as the validated auto path did. + for chunked teardown; returns MAX(messages.id) as the rebuild high + water. O(1) schema surgery — the heavy delete is deferred. - Markers are written in the same BEGIN IMMEDIATE as the demote, *before* - the empty v23 schema is created. Schema creation uses - ``executescript`` and therefore cannot run inside that transaction - (it issues an implicit COMMIT — see the CJK recreate path). Creating - the empty schema only after markers are durable closes the crash - window where trash + empty v23 tables exist with no backfill claim. + Markers are written in the same BEGIN IMMEDIATE as the demote, BEFORE + the empty v23 schema is created (``executescript`` implicitly COMMITs + and cannot run inside that transaction). This closes the crash window + where trash + empty v23 tables exist with no backfill claim. """ def _stage(conn): self._drop_fts_triggers(conn) @@ -717,23 +594,26 @@ class SessionSearchMixin: ] for sh in shadows: conn.execute(f"ALTER TABLE {sh} RENAME TO fts_v22_trash_{sh}") - # Claim the backfill *before* empty v23 tables exist. A crash - # between this commit and schema ensure still leaves markers, so - # optimize-storage resumes instead of tearing down trash and - # stamping an empty index as complete. + # Claim the backfill BEFORE empty v23 tables exist so a crash + # before schema ensure resumes instead of stamping an empty index. hw = self._seed_fts_rebuild_markers(conn, force=True) - conn.execute( - "DELETE FROM state_meta WHERE key = 'fts_optimize_available'" - ) + _delete_meta(conn, "fts_optimize_available") return hw hw = int(self._execute_write(_stage)) - # Create the empty v23 schema outside the write transaction — - # ``_ensure_fts_schema`` uses executescript(), which implicitly - # commits any pending transaction and must not run inside - # ``_execute_write``'s BEGIN IMMEDIATE (same rule as the CJK recreate - # path above). Markers are already durable. + # Outside the write transaction: ``_ensure_fts_schema`` uses + # executescript(), which implicitly commits. Markers are durable. + self._ensure_v23_fts_tables( + "failed to create v23 messages_fts during optimize-storage demote" + ) + return hw + + def _ensure_v23_fts_tables(self, failure_message: str) -> None: + """Ensure the v23 external-content base + trigram tables under the + lock (IF NOT EXISTS, cheap); raise *failure_message* without the base + table, since the backfill loop would otherwise retry "no such table" + forever.""" with self._lock: base_ok = self._ensure_fts_schema(self._conn, "messages_fts", FTS_SQL) trigram_ok = self._ensure_fts_schema( @@ -741,11 +621,8 @@ class SessionSearchMixin: ) self._trigram_available = bool(trigram_ok) if not base_ok: - raise sqlite3.OperationalError( - "failed to create v23 messages_fts during optimize-storage demote" - ) + raise sqlite3.OperationalError(failure_message) self._conn.commit() - return hw def optimize_fts_storage( self, @@ -754,30 +631,21 @@ class SessionSearchMixin: vacuum: bool = True, ) -> Dict[str, Any]: """Migrate a legacy v22 inline-FTS DB to the v23 external-content - schema, foreground and to completion. Safe to re-run: if a previous - attempt was interrupted it resumes from the progress marker. - - ``progress_cb`` receives {"phase", "percent", "indexed", "total"} - dicts for a CLI progress bar. Returns a summary dict. - - The trigram tokenizer being unavailable is not fatal — the base index - is still rebuilt (CJK falls back to LIKE), mirroring normal startup. - """ + schema, foreground and to completion; re-running resumes an + interrupted attempt. ``progress_cb`` receives {"phase", "percent", + "indexed", "total"}. A missing trigram tokenizer is not fatal (CJK + falls back to LIKE, as at startup).""" if not self._fts_enabled: return {"ok": False, "reason": "fts5_unavailable"} if self.read_only: return {"ok": False, "reason": "read_only"} - # Heal empty-index / orphan-marker bookkeeping from an interrupted - # demote *before* deciding whether to demote again. This re-seeds - # markers when trash was already staged (or torn down) without a - # backfill claim so the phases below actually run. + # Heal empty-index / orphan-marker bookkeeping BEFORE deciding whether + # to demote again, so the phases below actually run. self._repair_optimize_bookkeeping() - # Only demote if we're actually still on the legacy shape. If a prior - # run already demoted (markers/trash present), skip straight to - # finishing the backfill + teardown — this is what makes re-running - # after an interruption safe. + # Demote only when still on the legacy shape; a prior demote + # (markers/trash present) skips straight to backfill + teardown. with self._lock: legacy = self._db_has_legacy_inline_fts(self._conn) pending = self.get_meta("fts_rebuild_high_water") is not None @@ -786,31 +654,15 @@ class SessionSearchMixin: elif pending and not legacy: # Resume mid-demote: markers exist, empty v23 tables may still be # missing if the process died between the staged demote commit and - # schema ensure. Re-ensure is IF NOT EXISTS and cheap. - with self._lock: - base_ok = self._ensure_fts_schema( - self._conn, "messages_fts", FTS_SQL - ) - trigram_ok = self._ensure_fts_schema( - self._conn, "messages_fts_trigram", FTS_TRIGRAM_SQL - ) - self._trigram_available = bool(trigram_ok) - if not base_ok: - # Fail fast: without the base table the backfill loop - # below would retry "no such table" errors forever. - raise sqlite3.OperationalError( - "failed to re-create v23 messages_fts " - "on optimize-storage resume" - ) - self._conn.commit() + # schema ensure. + self._ensure_v23_fts_tables( + "failed to re-create v23 messages_fts on optimize-storage resume" + ) - # A stale CJK index (triggers dropped by a tokenizer-less process) - # can only be recovered from scratch — reset it now so the cjk - # backfill phase below rebuilds it. No-op without the tokenizer. + # A stale CJK index can only be recovered from scratch; reset it so + # the cjk backfill phase rebuilds it. Then ensure table + markers + # exist (a v23 DB gaining the cjk index for the first time). self._fts_cjk_reset_if_stale() - # An optimized v23 DB gaining the cjk index for the first time (no - # legacy work left, tokenizer newly installed): ensure the table + - # markers exist so the backfill phase has work to claim. if self._fts_cjk_loaded: with self._lock: self._ensure_fts_cjk_schema(self._conn) @@ -830,57 +682,40 @@ class SessionSearchMixin: }) def _pause(chunk_seconds: float) -> None: - """Inter-chunk throttle (see the chunk-engine note above). - - The chunk methods themselves never sleep, so this loop is the - single place the duty cycle is enforced: without it, back-to-back - BEGIN IMMEDIATE chunks starve any live gateway/CLI process - sharing the DB out of its lock retries (the measured ~85% - write-lock ownership that froze concurrent sessions). - """ + """Inter-chunk throttle — the single place the duty cycle is + enforced. Without it back-to-back BEGIN IMMEDIATE chunks starve a + live gateway/CLI sharing the DB out of its lock retries.""" time.sleep(max( self._FTS_REBUILD_MIN_PAUSE, chunk_seconds * self._FTS_REBUILD_DUTY_FACTOR, )) - # Phase 1: backfill (foreground, throttled between chunks so a live - # gateway sharing the DB stays responsive). - _emit("backfill") - while True: - _t0 = time.monotonic() - if not self.fts_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) - _emit("backfill") + def _drive(phase: str, step) -> None: + """Run *step* to completion, emitting progress and throttling + between chunks so a live gateway sharing the DB stays responsive.""" + while True: + _t0 = time.monotonic() + if not step(): + break + _emit(phase) + _pause(time.monotonic() - _t0) - # Phase 1b: backfill the CJK-bigram index (its own marker pair; a - # no-op when the tokenizer isn't loadable or nothing is pending). - while True: - _t0 = time.monotonic() - if not self.fts_cjk_rebuild_step(): - break - _emit("backfill") - _pause(time.monotonic() - _t0) + # Phase 1: base backfill. Phase 1b: CJK-bigram backfill (its own + # marker pair; no-op without the tokenizer or pending work). + _emit("backfill") + _drive("backfill", self.fts_rebuild_step) + _emit("backfill") + _drive("backfill", self.fts_cjk_rebuild_step) # Phase 2: tear down the demoted legacy shadow tables in chunks. _emit("teardown") - while True: - _t0 = time.monotonic() - if not self._fts_teardown_trash_step(): - break - _emit("teardown") - _pause(time.monotonic() - _t0) + _drive("teardown", self._fts_teardown_trash_step) # Refuse to stamp "optimized" while work remains or the base index is - # still empty against a non-empty messages table. Pre-fix code could - # tear down trash and settle after a no-op backfill when markers were - # missing — permanent search-index loss for historical rows. + # empty against non-empty messages (settling after a no-op backfill + # meant permanent search-index loss for historical rows). with self._lock: - still_pending = self._conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() is not None + still_pending = _meta_row(self._conn, "fts_rebuild_high_water") is not None still_trash = self._has_fts_trash(self._conn) empty_index = self._fts_external_index_empty_with_messages(self._conn) if still_pending or still_trash or empty_index: @@ -904,23 +739,15 @@ class SessionSearchMixin: self._conn.execute("VACUUM") vacuum_ok = True except sqlite3.OperationalError as exc: - # Most common cause: not enough free disk for VACUUM's temp - # copy. The optimization still succeeded; space just isn't - # reclaimed until a later VACUUM. Non-fatal. + # Usually no free disk for VACUUM's temp copy; the optimization + # still succeeded, space is reclaimed by a later VACUUM. logger.warning("VACUUM after FTS optimize failed: %s", exc) vacuum_ok = False - # Best-effort: fold the WAL back into the main file so the on-disk - # size settles now rather than at close(). NOTE this is REFUSED - # (SQLITE_BUSY) while any other connection holds a WAL read-mark — - # e.g. a live gateway sharing the DB — so it is not sufficient on - # its own. Callers must therefore NOT size the result by stat()ing - # the file; use :meth:`logical_size_bytes`, which is truthful - # immediately regardless of readers. - # PASSIVE, not TRUNCATE: optimize-storage runs from a transient CLI - # process; a TRUNCATE reset here would race a live gateway writer - # and tear B-tree pages (#45383). (The TRUNCATE was already refused - # SQLITE_BUSY while the gateway holds a read-mark, per the note - # above; PASSIVE removes the reset attempt entirely.) + # Best-effort WAL fold-back. REFUSED (SQLITE_BUSY) while another + # connection holds a WAL read-mark, so callers must NOT size the + # result by stat()ing the file — use :meth:`logical_size_bytes`. + # PASSIVE, not TRUNCATE: a TRUNCATE reset from this transient CLI + # process would race a live gateway writer and tear B-tree pages. try: with self._lock: self._conn.execute("PRAGMA wal_checkpoint(PASSIVE)") @@ -930,30 +757,21 @@ class SessionSearchMixin: exc, ) - # Phase 4: stamp the FTS storage layout as current, clear the "available" - # flag, and advance schema_version if it was somehow still behind (the - # main version normally advances on open now, but bump defensively so a - # DB opened only by pre-decoupling code still settles). The FTS-layout - # marker is the source of truth for "is this DB optimized". + # Phase 4: stamp the FTS layout (the source of truth for "optimized"), + # clear the "available" flag, and advance schema_version if a DB + # opened only by pre-decoupling code left it behind. def _settle(conn): # Re-check inside the write transaction so a concurrent writer # cannot race a stamp past incomplete work. Returns a refusal - # reason (stamping nothing) or None once the stamp is written. - if conn.execute( - "SELECT 1 FROM state_meta " - "WHERE key = 'fts_rebuild_high_water' LIMIT 1" - ).fetchone() is not None: + # reason (nothing stamped) or None. + if _meta_row(conn, "fts_rebuild_high_water") is not None: return "backfill_incomplete" if self._has_fts_trash(conn): return "teardown_incomplete" if self._fts_external_index_empty_with_messages(conn): return "backfill_incomplete" - conn.execute( - "INSERT INTO state_meta (key, value) VALUES ('fts_storage_version', ?) " - "ON CONFLICT(key) DO UPDATE SET value = excluded.value", - (str(FTS_STORAGE_VERSION),), - ) - conn.execute("DELETE FROM state_meta WHERE key = 'fts_optimize_available'") + self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn) + _delete_meta(conn, "fts_optimize_available") conn.execute( "UPDATE schema_version SET version = ? WHERE version < ?", (SCHEMA_VERSION, SCHEMA_VERSION), @@ -961,10 +779,8 @@ class SessionSearchMixin: return None refusal = self._execute_write(_settle) if refusal is not None: - # A concurrent process re-seeded markers, left trash, or emptied - # the index between the pre-vacuum check above and this write - # transaction. Nothing was stamped. Report the failure instead of - # crashing the CLI with a traceback; a re-run can still settle. + # A concurrent process changed state since the pre-vacuum check; + # report instead of crashing the CLI — a re-run can still settle. logger.warning( "FTS storage optimization settle refused (%s)", refusal ) @@ -983,37 +799,22 @@ class SessionSearchMixin: bookend: int = 3, keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant"), ) -> Dict[str, Any]: - """Return an anchored window plus session bookends. + """Anchored window (``get_messages_around``) plus session bookends. - Built on top of ``get_messages_around``. Three slices: + - ``window``: filtered to ``keep_roles``, EXCEPT the anchor itself is + always kept regardless of role. + - ``bookend_start`` / ``bookend_end``: first/last ``bookend`` messages + with ids strictly outside the window (empty when the window already + overlaps the head/tail). Empty-content rows (tool-call-only turns) + are skipped so they don't crowd out prose. - - ``window``: messages immediately surrounding the anchor. Filtered - to ``keep_roles`` (tool-response noise dropped by default), EXCEPT - the anchor itself is always preserved regardless of role. - - ``bookend_start``: first ``bookend`` user/assistant messages of the - session — but only those whose id is strictly before the window's - first message id. Empty when the window already overlaps the - session head. Empty-content messages (tool-call-only assistant - turns) are skipped so they don't crowd out actual prose openings. - - ``bookend_end``: last ``bookend`` user/assistant messages of the - session, same non-overlap rule at the tail. - - Bookends let an FTS5 hit anywhere in a long session yield the goal - (opening) and the resolution (closing) on a single call — without - loading the whole transcript. - - Returns ``{"window": [], "messages_before": 0, "messages_after": 0, - "bookend_start": [], "bookend_end": []}`` when the anchor isn't in - the session. - - ``keep_roles=None`` disables role filtering (raw window + raw - bookends). + Bookends let a hit anywhere in a long session yield the goal and the + resolution in one call. Empty slices + zero counts when the anchor + isn't in the session. ``keep_roles=None`` disables role filtering. """ if bookend < 0: bookend = 0 - # Reuse the primitive — handles anchor-existence, content decoding, - # tool_calls deserialisation, and boundary counts. primitive = self.get_messages_around( session_id, around_message_id, window=window ) @@ -1040,38 +841,28 @@ class SessionSearchMixin: window_min_id = window_rows[0]["id"] window_max_id = window_rows[-1]["id"] - # Fetch bookends only when there's room outside the window. SQL filters - # by id range, role, and non-empty content — tool-call-only assistant - # turns (content='' with tool_calls populated) are excluded so they - # don't crowd out actual prose openings/closings. bookend_start_rows: List[Any] = [] bookend_end_rows: List[Any] = [] if bookend > 0: + role_clause = "" + role_params: list = [] + if keep_roles is not None: + role_clause = f" AND role IN ({','.join('?' for _ in keep_roles)})" + role_params = list(keep_roles) + with self._read_ctx() as conn: - role_clause = "" - role_params: list = [] - if keep_roles is not None: - role_placeholders = ",".join("?" for _ in keep_roles) - role_clause = f" AND role IN ({role_placeholders})" - role_params = list(keep_roles) + def _bookend(op: str, boundary_id: int, order: str): + return conn.execute( + f"SELECT * FROM messages " + f"WHERE session_id = ? AND id {op} ?{role_clause} " + f"AND length(content) > 0 " + f"ORDER BY id {order} LIMIT ?", + (session_id, boundary_id, *role_params, bookend), + ).fetchall() - bookend_start_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id < ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id ASC LIMIT ?", - (session_id, window_min_id, *role_params, bookend), - ).fetchall() - - bookend_end_rows = conn.execute( - f"SELECT * FROM messages " - f"WHERE session_id = ? AND id > ?{role_clause} " - f"AND length(content) > 0 " - f"ORDER BY id DESC LIMIT ?", - (session_id, window_max_id, *role_params, bookend), - ).fetchall() - # End rows came back DESC for the LIMIT cap; flip to ASC. - bookend_end_rows = list(reversed(bookend_end_rows)) + bookend_start_rows = _bookend("<", window_min_id, "ASC") + # End rows come back DESC for the LIMIT cap; flip to ASC. + bookend_end_rows = list(reversed(_bookend(">", window_max_id, "DESC"))) def _hydrate(row) -> Dict[str, Any]: msg = dict(row) @@ -1103,43 +894,30 @@ class SessionSearchMixin: limit: int = 20, include_inactive: bool = False, ) -> List[Dict[str, Any]]: - """Return the *limit* most-recent user messages, newest first. + """The *limit* most-recent real user turns, newest first, as + ``{id, timestamp, preview}`` (preview = first 80 chars, whitespace + collapsed). Used by /rewind and ``/undo [N]``. - Each entry is a dict with keys ``id``, ``timestamp``, ``preview``. - ``preview`` is the first 80 characters of the message content - (with line breaks collapsed to spaces). Used by the /rewind - slash command picker, CLI/TUI/gateway ``/undo [N]``, and any other - caller that needs real user-turn targets. - - Bookkeeping timeline rows (``display_kind`` set — e.g. model_switch, - async_delegation_complete, auto_continue, hidden) are excluded. They - are durable ``role='user'`` rows for the API transcript, but no client - counts them as user turns (desktop demotes them to system / drops them; - the CLI already uses ``not m.get("display_kind")``). Including them here - made ``/undo`` soft-delete from a marker instead of the last real turn — - same class of index skew as the prompt.submit ordinal bug. - - By default only active messages are returned. + Bookkeeping timeline rows (``display_kind`` set) are excluded: they + are durable ``role='user'`` rows but no client counts them as user + turns, and including them made ``/undo`` soft-delete from a marker + instead of the last real turn. Only active messages by default. """ active_clause = "" if include_inactive else " AND active = 1" - # Match CLI/desktop: only real user turns, not timeline bookkeeping. display_clause = " AND (display_kind IS NULL OR display_kind = '')" - # Legacy standalone compaction handoffs (persisted pre-#80622) are - # durable role='user' rows with NO display_kind — SQL can't see them, - # so fetch with headroom and drop them in the decode loop below. - # Without this, /undo N and rewind pair an in-memory count that - # excludes handoffs with a DB pick that includes them, soft-deleting - # the wrong turn. + # Legacy standalone compaction handoffs are role='user' rows with NO + # display_kind — SQL can't see them, so fetch with headroom and drop + # them in the decode loop; otherwise /undo N pairs an in-memory count + # that excludes handoffs with a DB pick that includes them. fetch_limit = int(limit) * 2 + 5 with self._lock: - cursor = self._conn.execute( + rows = self._conn.execute( "SELECT id, timestamp, content FROM messages " "WHERE session_id = ? AND role = 'user'" f"{active_clause}{display_clause} " "ORDER BY id DESC LIMIT ?", (session_id, fetch_limit), - ) - rows = cursor.fetchall() + ).fetchall() from agent.context_compressor import ContextCompressor @@ -1149,8 +927,7 @@ class SessionSearchMixin: break decoded = self._decode_content(row["content"]) if ContextCompressor._is_context_summary_content(decoded): - # Compaction handoff — never a user-originated turn (#80622). - continue + continue # compaction handoff — never a user-originated turn if isinstance(decoded, list): # Multimodal — flatten text parts. text_parts = [ @@ -1169,41 +946,21 @@ class SessionSearchMixin: preview = " ".join(preview.split()) # collapse whitespace if len(preview) > 80: preview = preview[:77] + "..." - result.append( - { - "id": row["id"], - "timestamp": row["timestamp"], - "preview": preview, - } - ) + result.append({"id": row["id"], "timestamp": row["timestamp"], "preview": preview}) return result @staticmethod def _sanitize_fts5_query(query: str) -> str: - """Sanitize user input for safe use in FTS5 MATCH queries. - - FTS5 has its own query syntax where characters like ``"``, ``(``, ``)``, - ``+``, ``*``, ``{``, ``}``, the column-filter operator ``:`` and bare - boolean operators (``AND``, ``OR``, ``NOT``) have special meaning. - Passing raw user input directly to MATCH can cause - ``sqlite3.OperationalError``. - - Strategy: - - Preserve properly paired quoted phrases (``"exact phrase"``) - - Strip unmatched FTS5-special characters that would cause errors - - Wrap unquoted hyphenated and dotted terms in quotes so FTS5 - matches them as exact phrases instead of splitting on the - hyphen/dot (e.g. ``chat-send``, ``P2.2``, ``my-app.config.ts``) - """ - # Cap user-controlled FTS input before any regex processing. Search - # queries do not need to be arbitrarily large, and bounding them keeps - # sanitizer/runtime behavior predictable under adversarial input. + """Sanitize user input for FTS5 MATCH (raw special characters raise + ``sqlite3.OperationalError``): preserve paired quoted phrases, strip + unmatched special characters, and quote hyphenated/dotted terms so + FTS5 matches them as phrases instead of splitting (``chat-send``, + ``P2.2``, ``my-app.config.ts``).""" + # Cap before any regex processing so adversarial input stays bounded. query = query[:MAX_FTS5_QUERY_CHARS] - # Step 1: Extract balanced double-quoted phrases and protect them - # from further processing via numbered placeholders. Do this with a - # single linear scan rather than a regex so pathological quote runs - # cannot induce backtracking. + # Step 1: protect balanced quoted phrases via numbered placeholders. + # Linear scan, not regex, so pathological quote runs cannot backtrack. _quoted_parts: list = [] pieces: list[str] = [] i = 0 @@ -1215,8 +972,7 @@ class SessionSearchMixin: continue end = query.find('"', i + 1) if end == -1: - # Unmatched quote: replace with whitespace like the old - # sanitizer's special-char stripping step. + # Unmatched quote: replace with whitespace. pieces.append(" ") i += 1 continue @@ -1226,46 +982,30 @@ class SessionSearchMixin: sanitized = "".join(pieces) - # Step 2: Strip remaining (unmatched) FTS5-special characters. ``:`` is - # FTS5's column-filter operator (``col:term``); since the FTS table has a - # single ``content`` column, an unquoted colon query like ``TODO: fix`` - # parses as ``column:term`` and raises "no such column" — swallowed at - # the execute site into zero results. Strip it like the others. - # The class below is every character FTS5's query grammar rejects - # outside a quoted phrase. Anything omitted here reaches MATCH raw and - # raises, which the execute site swallows into zero results — the - # failure mode this step exists to prevent. Measured against a real - # FTS5 table: ``it's``, ``gateway/run.py``, ``user@host``, ``a,b`` and - # ``50%`` all raised before the class was completed. + # Step 2: strip remaining FTS5-special characters (see + # _FTS5_SPECIAL_CHARS); e.g. an unquoted ``TODO: fix`` parses as + # ``column:term`` and raises "no such column". sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized) - # Step 2b: ``%`` is excluded from the class above only to protect the - # CJK LIKE-fallback path (LIKE treats % as a wildcard the fallback - # builds itself). A non-CJK query never reaches that fallback - # (``is_cjk`` gates it), so ``50%`` would sail into MATCH raw and - # raise like the rest. Strip it whenever the query has no CJK. + # Step 2b: ``%`` is only spared for the CJK LIKE fallback; a non-CJK + # query never reaches it, so ``50%`` would hit MATCH raw and raise. if "%" in sanitized and not SessionSearchMixin._contains_cjk(sanitized): sanitized = sanitized.replace("%", " ") - # Step 3: Collapse repeated * (e.g. "***") into a single one, - # and remove leading * (prefix-only needs at least one char before *) + # Step 3: collapse repeated * and drop leading * (prefix needs a char). sanitized = re.sub(r"\*+", "*", sanitized) sanitized = re.sub(r"(^|\s)\*", r"\1", sanitized) - # Step 4: Remove dangling boolean operators at start/end that would - # cause syntax errors (e.g. "hello AND" or "OR world") + # Step 4: drop dangling boolean operators at start/end (syntax errors). sanitized = re.sub(r"(?i)^(AND|OR|NOT)\b\s*", "", sanitized.strip()) sanitized = re.sub(r"(?i)\s+(AND|OR|NOT)\s*$", "", sanitized.strip()) - # Step 5: Wrap unquoted dotted and/or hyphenated terms in double - # quotes. FTS5's tokenizer splits on dots and hyphens, turning - # ``chat-send`` into ``chat AND send`` and ``P2.2`` into ``p2 AND 2``. - # Quoting preserves phrase semantics. A single pass avoids the - # double-quoting bug that would occur if dotted, hyphenated and underscored - # patterns were applied sequentially (e.g. ``my-app.config``). + # Step 5: quote dotted/hyphenated/underscored terms in ONE pass (the + # tokenizer splits on them; sequential passes double-quote + # ``my-app.config``). sanitized = re.sub(r"\b(\w+(?:[._-]\w+)+)\b", r'"\1"', sanitized) - # Step 6: Restore preserved quoted phrases + # Step 6: restore preserved quoted phrases. for i, quoted in enumerate(_quoted_parts): sanitized = sanitized.replace(f"\x00Q{i}\x00", quoted) @@ -1284,17 +1024,7 @@ class SessionSearchMixin: @staticmethod def _contains_cjk(text: str) -> bool: """Check if text contains CJK (Chinese, Japanese, Korean) characters.""" - for ch in text: - cp = ord(ch) - if (0x4E00 <= cp <= 0x9FFF or # CJK Unified Ideographs - 0x3400 <= cp <= 0x4DBF or # CJK Extension A - 0x20000 <= cp <= 0x2A6DF or # CJK Extension B - 0x3000 <= cp <= 0x303F or # CJK Symbols - 0x3040 <= cp <= 0x309F or # Hiragana - 0x30A0 <= cp <= 0x30FF or # Katakana - 0xAC00 <= cp <= 0xD7AF): # Hangul Syllables - return True - return False + return any(SessionSearchMixin._is_cjk_codepoint(ord(ch)) for ch in text) @classmethod def _count_cjk(cls, text: str) -> int: @@ -1303,12 +1033,9 @@ class SessionSearchMixin: @classmethod def _has_lone_cjk_run(cls, query: str) -> bool: - """True when any maximal CJK run in the query is a single char. - - The cjk-bigram index stores bigrams for runs >=2 chars and unigrams - only for isolated chars, so a 1-char CJK term can't match inside - longer runs there — those queries keep the LIKE substring route. - """ + """True when any maximal CJK run in the query is a single char: the + cjk-bigram index stores unigrams only for isolated chars, so such a + term can't match inside longer runs — those queries keep LIKE.""" run = 0 for ch in query: if cls._is_cjk_codepoint(ord(ch)): @@ -1321,21 +1048,25 @@ class SessionSearchMixin: @staticmethod def _trigram_eligible_tokens(query: str) -> bool: - """True when every non-operator token is long enough for the trigram - tokenizer to match (>=3 chars). - - The trigram tokenizer indexes overlapping 3-character sequences, so a - token shorter than 3 chars produces no trigrams and can never match. - With FTS5's implicit-AND between tokens, a single short token makes the - whole MATCH return nothing, so the trigram path is only worth taking - when every searchable token qualifies. - """ + """True when every non-operator token is >=3 chars: a shorter token + produces no trigrams, and with FTS5's implicit AND one such token + makes the whole MATCH return nothing.""" tokens = [ - t for t in query.strip('"').strip().split() - if t.upper() not in {"AND", "OR", "NOT"} + t for t in query.strip('"').strip().split() if t.upper() not in _FTS_OPERATORS ] return bool(tokens) and all(len(t) >= 3 for t in tokens) + @classmethod + def _has_short_cjk_token(cls, raw_query: str) -> bool: + """True when any non-operator CJK token has fewer than 3 CJK chars — + the trigram tokenizer needs >=3 per token, so such a query returns + nothing there and must take the LIKE route.""" + return any( + cls._count_cjk(t) < 3 + for t in raw_query.split() + if t.upper() not in _FTS_OPERATORS and cls._contains_cjk(t) + ) + def _run_trigram_search( self, raw_query: str, @@ -1349,63 +1080,19 @@ class SessionSearchMixin: limit: int = 20, offset: int = 0, ) -> Optional[List[Dict[str, Any]]]: - """Run a search against a substring-capable FTS index. - - ``table`` is ``messages_fts_trigram`` (default) or - ``messages_fts_cjk``. The trigram tokenizer indexes overlapping - 3-byte sequences, so it matches substrings regardless of word - boundaries — both CJK phrases the unicode61 tokenizer splits into - single characters and Latin runs the unicode61 tokenizer fuses onto - adjacent CJK (e.g. ``修改youer服务端``). The cjk-bigram tokenizer - splits Latin runs off adjacent CJK, giving the same recovery as an - exact ranked token match. Each non-operator token is quoted to - neutralise FTS5 special characters while boolean operators - (AND/OR/NOT) are preserved. - - Returns the matching rows, or ``None`` when the query cannot be - executed (e.g. the tokenizer is unavailable at runtime) so the - caller can fall back to another strategy. - """ - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = [f"{table} MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM {table} - JOIN messages m ON m.id = {table}.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) + """Search a substring-capable index (``messages_fts_trigram`` or + ``messages_fts_cjk``): trigram matches substrings regardless of word + boundaries (CJK phrases unicode61 splits, Latin runs it fuses onto + adjacent CJK like ``修改youer服务端``); cjk-bigram splits Latin runs + off CJK for an exact ranked match. Returns ``None`` when the query + cannot execute (e.g. tokenizer unavailable) so the caller can fall + back.""" + tri_sql, tri_params = self._fts_match_sql( + table, _quote_fts_tokens(raw_query), order_by_sql, + include_inactive=include_inactive, source_filter=source_filter, + exclude_sources=exclude_sources, role_filter=role_filter, + limit=limit, offset=offset, + ) with self._read_ctx() as conn: try: tri_cursor = conn.execute(tri_sql, tri_params) @@ -1414,6 +1101,41 @@ class SessionSearchMixin: return None return [dict(row) for row in tri_cursor.fetchall()] + @staticmethod + def _fts_match_sql( + table: str, + match_query: str, + order_by_sql: str, + *, + include_inactive: bool, + source_filter: Optional[List[str]], + exclude_sources: Optional[List[str]], + role_filter: Optional[List[str]], + limit: int, + offset: int, + ) -> Tuple[str, list]: + """MATCH query + params against one FTS5 index joined to messages/sessions.""" + where = [f"{table} MATCH ?"] + params: list = [match_query] + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) + params.extend([limit, offset]) + sql = f""" + SELECT m.id, m.session_id, m.role, + snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet, + {_SEARCH_SELECT_TAIL} + FROM {table} + JOIN messages m ON m.id = {table}.rowid + JOIN sessions s ON s.id = m.session_id + WHERE {' AND '.join(where)} + {order_by_sql} + LIMIT ? OFFSET ? + """ + return sql, params + def search_messages( self, query: str, @@ -1426,14 +1148,10 @@ class SessionSearchMixin: include_inactive: bool = False, fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """Instrumented wrapper around :meth:`_search_messages_impl`. - - Logs one line per slow search with the routing path taken, so - production latency stays attributable per query shape (the 2026-07 - session_search investigation needed trace archaeology to discover - the LIKE full scans; this makes the next regression a grep). - Threshold: HERMES_SEARCH_SLOW_MS (default 1000; 0 logs every call). - """ + """Instrumented wrapper around :meth:`_search_messages_impl`: logs one + line per slow search with the routing path taken so latency stays + attributable per query shape. Threshold HERMES_SEARCH_SLOW_MS + (default 1000; 0 logs every call).""" started = time.time() rows = None try: @@ -1477,12 +1195,11 @@ class SessionSearchMixin: raw = sanitized.strip('"').strip() if self._fts_cjk_available and not self._has_lone_cjk_run(raw): return "fts_cjk" - tokens = [ - t for t in raw.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - short = any(self._count_cjk(t) < 3 for t in tokens) - if self._count_cjk(raw) >= 3 and not short and self._trigram_available: + if ( + self._count_cjk(raw) >= 3 + and not self._has_short_cjk_token(raw) + and self._trigram_available + ): return "trigram" return "like_scan" except Exception: @@ -1492,13 +1209,9 @@ class SessionSearchMixin: def _compile_like_boolean_query( query: str, ) -> Tuple[str, List[Any], Optional[str]]: - """Compile the supported FTS boolean subset into LIKE predicates. - - Terms within an OR group are ANDed by default, matching FTS5's - implicit conjunction. ``NOT`` negates the following term inside that - group instead of being discarded, so ``python NOT java`` becomes a - positive Python match plus a Java exclusion. - """ + """Compile the supported FTS boolean subset into LIKE predicates: + terms within an OR group are ANDed (FTS5's implicit conjunction) and + ``NOT`` negates the following term rather than being discarded.""" groups: List[List[Tuple[str, bool]]] = [[]] negate_next = False for raw_token in re.findall(r'"[^"]+"|\S+', query): @@ -1527,18 +1240,13 @@ class SessionSearchMixin: continue clauses: List[str] = [] for term, negated in group: - escaped = ( - term.replace("\\", "\\\\") - .replace("%", "\\%") - .replace("_", "\\_") - ) clause = ( "(COALESCE(m.content, '') LIKE ? ESCAPE '\\' OR " "COALESCE(m.tool_name, '') LIKE ? ESCAPE '\\' OR " "COALESCE(m.tool_calls, '') LIKE ? ESCAPE '\\')" ) clauses.append(f"NOT {clause}" if negated else clause) - params.extend([f"%{escaped}%"] * 3) + params.extend([f"%{_escape_like(term)}%"] * 3) if snippet_term is None and not negated: snippet_term = term compiled_groups.append(f"({' AND '.join(clauses)})") @@ -1563,52 +1271,45 @@ class SessionSearchMixin: return [] where = [f"({predicate})"] - if not include_inactive: - where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - params.extend(source_filter) - if exclude_sources is not None: - where.append( - f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})" - ) - params.extend(exclude_sources) - if role_filter: - where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - params.extend(role_filter) - + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) order = ( "ASC" if isinstance(sort, str) and sort.strip().lower() == "oldest" else "DESC" ) + return self._like_rows( + where, [snippet_term, *params, limit, offset], + order_by=f"ORDER BY m.timestamp {order}, m.id {order}", limit_sql="LIMIT ? OFFSET ?", + ) + + def _like_rows( + self, where: List[str], params: list, *, order_by: str, limit_sql: str + ) -> List[Dict[str, Any]]: + """Canonical-table LIKE scan; ``params[0]`` is the snippet anchor term.""" sql = f""" SELECT m.id, m.session_id, m.role, - substr(m.content, max(1, instr(m.content, ?) - 40), 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started + {_LIKE_SNIPPET_SQL}, + {_SEARCH_SELECT_TAIL} FROM messages m JOIN sessions s ON s.id = m.session_id WHERE {' AND '.join(where)} - ORDER BY m.timestamp {order}, m.id {order} - LIMIT ? OFFSET ? + {order_by} + {limit_sql} """ - with self._read_ctx() as conn: - rows = conn.execute( - sql, [snippet_term, *params, limit, offset] - ).fetchall() - return [dict(row) for row in rows] + return [dict(row) for row in self._read_all(sql, params)] def _refresh_fts_stale_state(self) -> None: """Observe fail-open initiated by another process sharing state.db.""" if self._fts_stale or not self._fts_enabled: return try: - with self._read_ctx() as conn: - stale = conn.execute( - "SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", - (FTS_STALE_KEY,), - ).fetchone() + stale = self._read_one( + "SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", (FTS_STALE_KEY,) + ) except sqlite3.Error: return if stale is not None: @@ -1622,13 +1323,9 @@ class SessionSearchMixin: matches: List[Dict[str, Any]], result_fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """Attach neighboring messages and trim full content from results. - - Context (1 message before + after each match) is only loaded when - the selected result projection consumes it. Each query takes its - own fresh read transaction via _read_ctx, so we never hold a lock - across N sequential queries. - """ + """Attach neighboring messages (1 before + after, only when the + projection consumes ``context``) and trim full content. Each context + query takes its own read transaction, never a lock across N queries.""" context_matches = ( matches if result_fields is None or "context" in result_fields else () ) @@ -1691,12 +1388,8 @@ class SessionSearchMixin: except Exception: match["context"] = [] - # Full message content is never selected by any search route: every - # SELECT returns snippet + metadata only (saves I/O on multi-MB tool - # rows and the tokens a content column would cost downstream). The - # context query above re-fetches its 3-message window by id, so - # nothing reads content from the match rows themselves. The pop stays - # as a belt-and-braces guard for any future route that selects it. + # No search route selects full content (snippet + metadata only); the + # pop is a guard for any future route that does. for match in matches: match.pop("content", None) @@ -1720,36 +1413,19 @@ class SessionSearchMixin: include_inactive: bool = False, fields: Optional[Collection[str]] = None, ) -> List[Dict[str, Any]]: - """ - Full-text search across session messages using FTS5. + """FTS5 search across session messages (keywords, ``"phrases"``, + AND/OR/NOT, ``prefix*``). - Supports FTS5 query syntax: - - Simple keywords: "docker deployment" - - Phrases: '"exact phrase"' - - Boolean: "docker OR kubernetes", "python NOT java" - - Prefix: "deploy*" + Returns snippet + session metadata + 1-message context per hit; + ``fields`` selects a projection (context is only loaded when it + consumes it). ``sort``: None = BM25 rank only; "newest"/"oldest" = + timestamp then rank. The short-CJK LIKE fallback orders by timestamp + DESC and ignores ``sort``. - Returns matching messages with session metadata, content snippet, - and surrounding context (1 message before and after the match). - ``fields`` selects a result projection; omitting it preserves the - complete legacy result. Context is only loaded when that projection - consumes it. - - ``sort`` controls temporal ordering: - - ``None`` (default): FTS5 BM25 relevance only. Time-neutral. - - ``"newest"``: order by message timestamp DESC, then by rank. - - ``"oldest"``: order by message timestamp ASC, then by rank. - - The short-CJK LIKE fallback already orders by timestamp DESC and - ignores ``sort``. The trigram CJK path honours ``sort`` like the main - FTS5 path. - - Rewound (``active=0``, ``compacted=0``) rows are excluded by default — - the user took those back. Compaction-archived rows (``active=0``, - ``compacted=1``) ARE included by default: they were summarized away from - the live context but remain part of the conversation's record, so the - pre-compaction transcript stays discoverable after in-place compaction - (#38763). Pass ``include_inactive=True`` to search every row regardless. + Rewound rows (``active=0, compacted=0``) are excluded by default; + compaction-archived rows (``compacted=1``) ARE included so the + pre-compaction transcript stays discoverable. ``include_inactive`` + searches every row. """ result_fields = self._search_message_fields(fields) @@ -1760,27 +1436,21 @@ class SessionSearchMixin: if not query: return [] + filters = dict( + include_inactive=include_inactive, source_filter=source_filter, + exclude_sources=exclude_sources, role_filter=role_filter, + ) self._refresh_fts_stale_state() if self._fts_stale: matches = self._search_messages_like_fallback( - query, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - sort=sort, - include_inactive=include_inactive, - ) - return self._finalize_search_matches( - matches, result_fields=result_fields + query, limit=limit, offset=offset, sort=sort, **filters ) + return self._finalize_search_matches(matches, result_fields=result_fields) if not self._fts_enabled: return [] - # Normalise sort. Anything not in the allowed set falls back to None - # (FTS5 rank-only) so callers can pass through user input without - # validation. + # Normalise sort; anything unknown falls back to rank-only so callers + # can pass through user input. if isinstance(sort, str): sort_norm = sort.strip().lower() if sort_norm not in ("newest", "oldest"): @@ -1788,8 +1458,6 @@ class SessionSearchMixin: else: sort_norm = None - # ORDER BY shared across the main FTS5 path and trigram CJK path. - # With sort set, timestamp is primary and rank is the tiebreaker. if sort_norm == "newest": order_by_sql = "ORDER BY m.timestamp DESC, rank" elif sort_norm == "oldest": @@ -1797,157 +1465,44 @@ class SessionSearchMixin: else: order_by_sql = "ORDER BY rank" - # Build WHERE clauses dynamically - where_clauses = ["messages_fts MATCH ?"] - params: list = [query] - if not include_inactive: - # Live rows (active=1) AND compaction-archived rows (compacted=1) - # are discoverable; only rewind/undo rows (active=0, compacted=0) - # are hidden. See archive_and_compact() / #38763. - where_clauses.append("(m.active = 1 OR m.compacted = 1)") - - if source_filter is not None: - source_placeholders = ",".join("?" for _ in source_filter) - where_clauses.append(f"s.source IN ({source_placeholders})") - params.extend(source_filter) - - if exclude_sources is not None: - exclude_placeholders = ",".join("?" for _ in exclude_sources) - where_clauses.append(f"s.source NOT IN ({exclude_placeholders})") - params.extend(exclude_sources) - - if role_filter: - role_placeholders = ",".join("?" for _ in role_filter) - where_clauses.append(f"m.role IN ({role_placeholders})") - params.extend(role_filter) - - where_sql = " AND ".join(where_clauses) - params.extend([limit, offset]) - - sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts - JOIN messages m ON m.id = messages_fts.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {where_sql} - {order_by_sql} - LIMIT ? OFFSET ? - """ - - # CJK queries bypass the unicode61 FTS5 table. The default tokenizer - # splits CJK characters into individual tokens, so "大别山项目" becomes - # "大 AND 别 AND 山 AND 项 AND 目" — producing false positives and - # missing exact phrase matches. - # - # For queries with 3+ CJK characters, we use the trigram FTS5 table - # (indexed substring matching with ranking and snippets). For shorter - # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8 - # bytes = 3 CJK chars), so we fall back to LIKE. + # CJK queries bypass the unicode61 table, whose tokenizer splits CJK + # into single characters ("大别山项目" -> "大 AND 别 AND ...": false + # positives, missed phrases). 3+ CJK chars -> trigram; shorter -> + # LIKE (trigram needs 9 UTF-8 bytes = 3 CJK chars). matches: List[Dict[str, Any]] = [] is_cjk = self._contains_cjk(query) if is_cjk: raw_query = query.strip('"').strip() - cjk_count = self._count_cjk(raw_query) - - # Per-token CJK length check (#20494): trigram needs >=3 CJK chars - # per token. A query like "广西 OR 桂林 OR 漓江" has cjk_count=6 - # (>=3) but each individual token is only 2 chars — trigram returns 0. - # Route to LIKE when any non-operator CJK token is <3 CJK chars. - _tokens_for_check = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t) - ] - _any_short_cjk = any( - self._count_cjk(t) < 3 for t in _tokens_for_check - ) - _trigram_succeeded = False - # Tool rows are excluded from the trigram index (they're ~90% of - # message bytes and machine noise — see FTS_TRIGRAM_SQL). A CJK - # query explicitly filtering on role='tool' must therefore use - # the LIKE fallback, which scans the base table directly. + # Tool rows are excluded from the trigram/cjk indexes (see + # FTS_TRIGRAM_SQL), so a role='tool' CJK query must use LIKE. _wants_tool_rows = bool(role_filter) and "tool" in role_filter - # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── - # When the bigram index is available it serves EVERY CJK query - # shape the legacy code split between trigram (>=3 chars/token) - # and LIKE full scans (1-2 char tokens) — the whole point of the - # index (PR #65544). Exceptions stay on the legacy routes: - # - role_filter=['tool'] queries (tool rows aren't in the cjk - # index, same exclusion as trigram), - # - queries containing a LONE 1-char CJK run: the index stores - # bigrams for runs >=2, so a single-char term can only match - # isolated chars — LIKE substring semantics are broader. + # CJK-bigram route: serves every CJK shape the legacy code split + # between trigram and LIKE full scans, except role='tool' queries + # and LONE 1-char CJK runs (the index stores bigrams for runs >=2, + # so a single-char term only matches isolated chars — LIKE is broader). if ( self._fts_cjk_available and not _wants_tool_rows and not self._has_lone_cjk_run(raw_query) ): - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - cjk_query = " ".join(parts) - cjk_where = ["messages_fts_cjk MATCH ?"] - cjk_params: list = [cjk_query] - if not include_inactive: - cjk_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - cjk_params.extend(source_filter) - if exclude_sources is not None: - cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - cjk_params.extend(exclude_sources) - if role_filter: - cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - cjk_params.extend(role_filter) - cjk_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_cjk - JOIN messages m ON m.id = messages_fts_cjk.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(cjk_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - cjk_params.extend([limit, offset]) + cjk_sql, cjk_params = self._fts_match_sql( + "messages_fts_cjk", _quote_fts_tokens(raw_query), order_by_sql, + limit=limit, offset=offset, **filters, + ) try: - with self._read_ctx() as conn: - cjk_cursor = conn.execute(cjk_sql, cjk_params) - matches = [dict(row) for row in cjk_cursor.fetchall()] - _trigram_succeeded = True + matches = [dict(row) for row in self._read_all(cjk_sql, cjk_params)] + _trigram_succeeded = True except sqlite3.OperationalError: - # Tokenizer missing on this connection / query syntax — - # the trigram + LIKE routes below still answer. + # Tokenizer missing / query syntax — trigram + LIKE still answer. logger.debug( "messages_fts_cjk query failed; falling back to " "trigram/LIKE", exc_info=True, ) except sqlite3.DatabaseError as exc: - # A full-message rebuild is unbounded and holds the writer - # lock, so a live search never performs one. Detach the - # derived indexes and answer from canonical rows instead. + # A live search never performs the unbounded full rebuild: + # detach the derived indexes and answer from canonical rows. # Non-FTS corruption is not safe to reinterpret here. if not self._enter_fts_fail_open(exc): raise @@ -1957,69 +1512,29 @@ class SessionSearchMixin: exc, ) + # Per-token CJK length check: trigram needs >=3 CJK chars per + # token. "广西 OR 桂林 OR 漓江" has 6 CJK chars total but 2 per + # token — trigram returns 0, so such queries take LIKE. if ( not _trigram_succeeded - and cjk_count >= 3 - and not _any_short_cjk + and self._count_cjk(raw_query) >= 3 + and not self._has_short_cjk_token(raw_query) and self._trigram_available and not _wants_tool_rows ): - # Trigram FTS5 path — quote each non-operator token to handle - # FTS5 special chars (%, *, etc.) while preserving boolean - # operators (AND, OR, NOT) for multi-term queries. - tokens = raw_query.split() - parts = [] - for tok in tokens: - if tok.upper() in {"AND", "OR", "NOT"}: - parts.append(tok) - else: - parts.append('"' + tok.replace('"', '""') + '"') - trigram_query = " ".join(parts) - tri_where = ["messages_fts_trigram MATCH ?"] - tri_params: list = [trigram_query] - if not include_inactive: - tri_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - tri_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - tri_params.extend(source_filter) - if exclude_sources is not None: - tri_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - tri_params.extend(exclude_sources) - if role_filter: - tri_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - tri_params.extend(role_filter) - tri_sql = f""" - SELECT - m.id, - m.session_id, - m.role, - snippet(messages_fts_trigram, -1, '>>>', '<<<', '...', 40) AS snippet, - m.timestamp, - m.tool_name, - s.source, - s.model, - s.started_at AS session_started - FROM messages_fts_trigram - JOIN messages m ON m.id = messages_fts_trigram.rowid - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(tri_where)} - {order_by_sql} - LIMIT ? OFFSET ? - """ - tri_params.extend([limit, offset]) + tri_sql, tri_params = self._fts_match_sql( + "messages_fts_trigram", _quote_fts_tokens(raw_query), order_by_sql, + limit=limit, offset=offset, **filters, + ) try: - with self._read_ctx() as conn: - tri_cursor = conn.execute(tri_sql, tri_params) - matches = [dict(row) for row in tri_cursor.fetchall()] - _trigram_succeeded = True + matches = [dict(row) for row in self._read_all(tri_sql, tri_params)] + _trigram_succeeded = True except sqlite3.OperationalError: # Trigram query failed at runtime — fall through to LIKE. pass except sqlite3.DatabaseError as exc: - # Preserve the same bounded recovery contract as the CJK - # and main FTS paths: detach derived indexes, then fall - # through to the canonical LIKE query. A non-FTS storage - # error remains fatal rather than being hidden as a miss. + # Same bounded recovery as the CJK/main paths; a non-FTS + # storage error stays fatal rather than hidden as a miss. if not self._enter_fts_fail_open(exc): raise logger.warning( @@ -2028,163 +1543,81 @@ class SessionSearchMixin: exc, ) if not _trigram_succeeded: - # Short / mixed CJK query, trigram unavailable, or trigram - # <3 CJK chars. Fall back to LIKE substring search. - # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"), - # build one LIKE condition per non-operator token so each term - # is matched independently (#20494). + # LIKE substring fallback; one clause per non-operator token so + # "广西 OR 桂林 OR 漓江" matches each term independently. non_op_tokens = [ - t for t in raw_query.split() - if t.upper() not in {"AND", "OR", "NOT"} + t for t in raw_query.split() if t.upper() not in _FTS_OPERATORS ] or [raw_query] - token_clauses = [] like_params: list = [] for tok in non_op_tokens: - esc = _escape_like(tok) - token_clauses.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"] - like_where = [f"({' OR '.join(token_clauses)})"] - if not include_inactive: - # Same visibility rule as the FTS5 paths: live rows and - # compaction-archived rows are discoverable; rewind/undo - # rows (active=0, compacted=0) are hidden (#38763). - like_where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - like_params.extend(source_filter) - if exclude_sources is not None: - like_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - like_params.extend(exclude_sources) - if role_filter: - like_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - like_params.extend(role_filter) - like_sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(like_where)} - ORDER BY m.timestamp DESC - LIMIT ? OFFSET ? - """ - like_params.extend([limit, offset]) + like_params += [f"%{_escape_like(tok)}%"] * 3 + like_where = [ + f"({' OR '.join([_LIKE_ANY_COLUMN_SQL] * len(non_op_tokens))})" + ] + _search_filter_clauses(like_where, like_params, **filters) # instr() for snippet uses first search token - like_params = [non_op_tokens[0]] + like_params - with self._read_ctx() as conn: - like_cursor = conn.execute(like_sql, like_params) - matches = [dict(row) for row in like_cursor.fetchall()] + matches = self._like_rows( + like_where, [non_op_tokens[0], *like_params, limit, offset], + order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ? OFFSET ?", + ) else: + sql, params = self._fts_match_sql( + "messages_fts", query, order_by_sql, limit=limit, offset=offset, **filters + ) try: - with self._read_ctx() as conn: - cursor = conn.execute(sql, params) - matches = [dict(row) for row in cursor.fetchall()] + matches = [dict(row) for row in self._read_all(sql, params)] except sqlite3.OperationalError: # FTS5 query syntax error despite sanitization — return empty return [] except sqlite3.DatabaseError as exc: - # A corrupt FTS index raises the malformed / "fts5: corrupt - # structure record" class on the MATCH read, the same class the - # write path handles (#66296). OperationalError (query syntax) - # is a subclass caught above; this arm is the corruption - # parent. Live search must remain bounded, so detach the - # derived indexes and answer from canonical message rows. The - # existing stale-open/repair paths retain rebuild ownership. + # Corruption parent class (OperationalError is caught above). + # Live search must stay bounded: detach the derived indexes and + # answer from canonical rows; repair paths own the rebuild. if not self._enter_fts_fail_open(exc): raise matches = self._search_messages_like_fallback( - query, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - sort=sort, - include_inactive=include_inactive, + query, limit=limit, offset=offset, sort=sort, **filters ) - # Deferred-rebuild supplement (schema v23): while the background - # backfill is pending, the FTS indexes only cover rows outside the - # (progress, high_water] gap. Top the results up with a bounded LIKE - # scan over just that id range so search never silently loses old - # messages mid-rebuild. The range shrinks as the backfill advances, - # so this cost decays to zero. The CJK LIKE-fallback path above - # already scans the whole base table and needs no supplement. + # Deferred-rebuild supplement: while the backfill is pending the FTS + # indexes miss the (progress, high_water] gap; top up with a bounded + # LIKE scan over that range so old messages never silently vanish + # mid-rebuild. The cost decays to zero as the backfill advances. rebuild_status = self.fts_rebuild_status() if rebuild_status is not None and len(matches) < limit: try: gap_matches = self._search_unindexed_gap( - query, - limit - len(matches), - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, + query, limit - len(matches), **filters ) seen_ids = {m["id"] for m in matches} matches.extend(m for m in gap_matches if m["id"] not in seen_ids) except sqlite3.OperationalError as exc: logger.debug("Unindexed-gap supplement skipped: %s", exc) - # Pure-Latin queries run against the unicode61 ``messages_fts`` table, - # whose tokenizer does not insert a boundary between Latin letters and - # adjacent CJK characters: "修改youer服务端" is indexed as one token, - # so MATCH "youer" finds nothing even though the substring is present - # (#54242). When the exact-token search returns nothing, retry on the - # substring-capable indexes. Preference order: - # 1. messages_fts_cjk (when built): its tokenizer splits Latin runs - # off adjacent CJK, so "youer" is an exact ranked token match. - # 2. messages_fts_trigram: substring matching, needs >=3-char - # tokens (shorter tokens produce no trigrams). - # Gated on a zero-result miss so successful Latin searches keep their - # unicode61 ranking — strictly additive, never reorders existing - # hits. Trade-off on the trigram leg: any zero-result Latin query - # gains substring semantics (e.g. "cat" can then match - # "concatenate"). Genuinely absent terms still return []. Skipped for - # role_filter=['tool'] queries — both fallback indexes exclude tool - # rows (v23), so a retry could never add hits. + # unicode61 puts no boundary between Latin and adjacent CJK + # ("修改youer服务端" is one token, so MATCH "youer" misses). On a + # zero-result Latin miss retry the substring-capable indexes: cjk + # first (splits Latin off CJK: exact ranked match), then trigram + # (needs >=3-char tokens). Gated on a miss so successful searches keep + # their ranking; trade-off: "cat" may then match "concatenate". + # Skipped for role='tool' (both indexes exclude tool rows). if ( not matches and not is_cjk and not (bool(role_filter) and "tool" in role_filter) ): _fb_query = query.strip('"').strip() + fb_kwargs = dict(order_by_sql=order_by_sql, limit=limit, offset=offset, **filters) if self._fts_cjk_available: - cjk_fb = self._run_trigram_search( - _fb_query, - table="messages_fts_cjk", - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if cjk_fb: - matches = cjk_fb + matches = self._run_trigram_search( + _fb_query, table="messages_fts_cjk", **fb_kwargs + ) or matches if ( not matches and self._trigram_available and self._trigram_eligible_tokens(query) ): - tri_matches = self._run_trigram_search( - _fb_query, - order_by_sql=order_by_sql, - include_inactive=include_inactive, - source_filter=source_filter, - exclude_sources=exclude_sources, - role_filter=role_filter, - limit=limit, - offset=offset, - ) - if tri_matches: - matches = tri_matches + matches = self._run_trigram_search(_fb_query, **fb_kwargs) or matches return self._finalize_search_matches(matches, result_fields=result_fields) @@ -2198,68 +1631,37 @@ class SessionSearchMixin: exclude_sources: Optional[List[str]] = None, role_filter: Optional[List[str]] = None, ) -> List[Dict[str, Any]]: - """LIKE-scan the rows the deferred rebuild hasn't indexed yet. - - Only touches ids in (fts_rebuild_progress, fts_rebuild_high_water] — - a range that shrinks to nothing as the backfill advances. The FTS - query is degraded to per-token substring terms (AND-joined; quoted - phrases kept whole), which is deliberately recall-over-precision: - temporary results beat silently missing ones mid-rebuild. - """ + """LIKE-scan ids in (fts_rebuild_progress, fts_rebuild_high_water] — + the rows the deferred rebuild hasn't indexed yet. The FTS query is + degraded to AND-joined substring terms (quoted phrases kept whole): + deliberately recall-over-precision mid-rebuild.""" status = self.fts_rebuild_status() if status is None or limit <= 0: return [] progress, high_water = status["indexed"], status["total"] - # Degrade the FTS query to LIKE terms: strip operators/wildcards, - # keep quoted phrases intact, AND the rest. terms: List[str] = [] for raw_tok in re.findall(r'"[^"]+"|\S+', fts_query): tok = raw_tok.strip('"').strip("*").strip() - if not tok or tok.upper() in {"AND", "OR", "NOT", "NEAR"}: - continue - terms.append(tok) + if tok and tok.upper() not in {"AND", "OR", "NOT", "NEAR"}: + terms.append(tok) if not terms: return [] where = ["m.id > ? AND m.id <= ?"] params: list = [progress, high_water] for term in terms: - esc = _escape_like(term) - where.append( - "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' " - "OR m.tool_calls LIKE ? ESCAPE '\\')" - ) - params += [f"%{esc}%"] * 3 - if not include_inactive: - where.append("(m.active = 1 OR m.compacted = 1)") - if source_filter is not None: - where.append(f"s.source IN ({','.join('?' for _ in source_filter)})") - params.extend(source_filter) - if exclude_sources is not None: - where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})") - params.extend(exclude_sources) - if role_filter: - where.append(f"m.role IN ({','.join('?' for _ in role_filter)})") - params.extend(role_filter) - - sql = f""" - SELECT m.id, m.session_id, m.role, - substr(m.content, - max(1, instr(m.content, ?) - 40), - 120) AS snippet, - m.timestamp, m.tool_name, - s.source, s.model, s.started_at AS session_started - FROM messages m - JOIN sessions s ON s.id = m.session_id - WHERE {' AND '.join(where)} - ORDER BY m.timestamp DESC - LIMIT ? - """ - params = [terms[0]] + params + [limit] - with self._read_ctx() as conn: - rows = conn.execute(sql, params).fetchall() - return [dict(r) for r in rows] + where.append(_LIKE_ANY_COLUMN_SQL) + params += [f"%{_escape_like(term)}%"] * 3 + _search_filter_clauses( + where, params, include_inactive=include_inactive, + source_filter=source_filter, exclude_sources=exclude_sources, + role_filter=role_filter, + ) + return self._like_rows( + where, [terms[0], *params, limit], + order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ?", + ) def search_sessions_by_id( self, @@ -2270,24 +1672,16 @@ class SessionSearchMixin: sources: List[str] = None, exclude_sources: List[str] = None, ) -> List[Dict[str, Any]]: - """Search surfaced sessions by exact/prefix/substring session id. - - Desktop search uses this alongside FTS message search so users can paste - a session id from logs, CLI output, or another Hermes surface and jump - straight to that conversation. Matching also checks ``_lineage_root_id`` - for projected compression-chain tips, so an old root id still resolves to - the live continuation row. - """ + """Search surfaced sessions by exact/prefix/substring session id (paste + an id from logs and jump to it). Also matches ``_lineage_root_id`` so + an old compression root id resolves to the live continuation row.""" needle = (query or "").strip().lower() if not needle or limit <= 0: return [] - # SQL-bounded: list_sessions_rich pushes the id LIKE filter into the - # query (matching the row's own id AND any id in its forward - # compression chain), so we only materialize matching rows instead of - # scanning every session. Fetch a small multiple of `limit` so the - # in-Python exact/prefix/substring ranking below has enough candidates - # to order, then truncate. + # list_sessions_rich pushes the id LIKE filter (own id + forward + # compression chain) into SQL; over-fetch so the in-Python + # exact/prefix/substring ranking has candidates, then truncate. candidates = self.list_sessions_rich( source=source, sources=sources, @@ -2320,32 +1714,17 @@ class SessionSearchMixin: self._conn.execute(f"SELECT 1 FROM {name} LIMIT 0") return True except sqlite3.DatabaseError: - # OperationalError ("no such table") or the broader - # DatabaseError class ("vtable constructor failed", raised when - # e.g. a required tokenizer is missing or the table is mid- - # teardown) — in every case the table is not queryable. + # "no such table", or "vtable constructor failed" (missing + # tokenizer / mid-teardown) — either way not queryable. return False def optimize_fts(self) -> int: - """Merge fragmented FTS5 b-tree segments into one per index. + """Merge fragmented FTS5 segments into one per index (``'optimize'``). - FTS5 indexes grow as a series of incremental segments — one per - ``INSERT`` batch driven by the message triggers. Over tens of - thousands of messages these segments accumulate, which both bloats - the ``*_data`` shadow tables and slows ``MATCH`` queries that must - scan every segment. The special ``'optimize'`` command rewrites each - index as a single merged segment. - - This is purely a maintenance operation — it changes neither search - results nor ``snippet()`` output, only on-disk layout and query - speed. It is complementary to VACUUM: ``optimize`` compacts the FTS - index internally, then VACUUM returns the freed pages to the OS. - - Skips any FTS table that does not exist (e.g. the trigram index when - disabled via ``HERMES_DISABLE_FTS_TRIGRAM`` or not yet created), so - it is safe to call unconditionally. - - Returns the number of FTS indexes that were optimized. + Pure maintenance: changes neither results nor ``snippet()`` output, + only layout and speed; complementary to VACUUM, which then returns + the freed pages. Skips absent tables, so it is safe unconditionally. + Returns the number of indexes optimized. """ optimized = 0 with self._lock: @@ -2353,8 +1732,6 @@ class SessionSearchMixin: if not self._fts_table_exists(tbl): continue try: - # The column name in the INSERT must match the table name - # for FTS5 special commands. self._conn.execute( f"INSERT INTO {tbl}({tbl}) VALUES('optimize')" ) @@ -2366,31 +1743,19 @@ class SessionSearchMixin: return optimized def rebuild_fts(self) -> int: - """Rebuild FTS5 indexes from the canonical ``messages`` table. + """Rebuild FTS5 indexes from ``messages`` (``'rebuild'``) — the + documented recovery for a corrupt index that rejects writes while + reads succeed. - Uses the FTS5 ``'rebuild'`` command, which rewrites the internal - b-tree segments from the content rows. This is the documented - recovery for a corrupt FTS index that rejects message writes while - reads still succeed (issue #50502). Unlike ``optimize_fts`` (which - merges existing segments), ``rebuild`` discards and recreates the - index data entirely. - - A full structural rebuild must never run concurrently in two - processes sharing one state.db — that interleaving has structurally - corrupted the database in production (PR #93200) — so this admits - through the cross-process ``fts_rebuild_admission`` authority and - FAILS CLOSED: if another process holds the rebuild lock beyond the - bounded wait, this call defers (returns 0) rather than racing it. - Callers already treat 0 as "rebuild made no progress" and fall back - to the stale-FTS breadcrumb path, which retries in-process from the - gateway housekeeping tick (``retry_deferred_fts_recovery``) and at - next startup. - - Safe to call when FTS tables don't exist (skips them). - Returns the number of FTS indexes that were rebuilt. + A full structural rebuild must never run concurrently in two processes + sharing one state.db (that interleaving corrupted production DBs), so + this admits through ``fts_rebuild_admission`` and FAILS CLOSED, + returning 0 on deferral; callers treat 0 as "no progress" and fall + back to the stale-FTS breadcrumb path. Skips absent tables. Returns + the number of indexes rebuilt. """ rebuilt = 0 - with fts_rebuild_admission(getattr(self, "db_path", None)) as admitted: + with fts_rebuild_admission(self.db_path) as admitted: if not admitted: logger.warning( "Deferred in-place FTS rebuild: another process holds " @@ -2419,34 +1784,23 @@ class SessionSearchMixin: ) -> int: """Run bounded FTS5 ``'merge'`` commands against each present index. - A positive merge rank tells SQLite to stop after approximately that - many output pages, so each command holds the write lock for - milliseconds regardless of index size — unlike ``'optimize'``, which - rewrites the whole index in one transaction (measured 9-18 s per - index on a 10 GB production DB, long enough to exhaust a competing - writer's entire lock-retry patience). - - Protocol (SQLite FTS5 §6.8-6.9): + A positive merge rank stops after ~that many output pages, so each + command holds the write lock for milliseconds regardless of index + size — unlike ``'optimize'`` (9-18 s per index on a 10 GB DB, enough + to exhaust a competing writer's lock-retry patience). - ``usermerge`` is lowered to its minimum of 2 (persisted in the - ``%_config`` shadow table, applied once per instance) so a - positive merge acts on ANY level holding >= 2 segments. With the - default of 4, levels below that threshold are never merged by a - positive-rank command and a fragmented index cannot converge. - - Up to *max_commands* merge commands run per index, stopping early - on the documented no-progress signal: the delta in - ``total_changes`` is < 2 (the command's own INSERT accounts - for 1 change; >= 2 means real merge work happened). + ``%_config`` shadow table, once per instance) so a positive merge + acts on ANY level with >= 2 segments; at the default 4 a fragmented + index cannot converge. + - Up to *max_commands* per index, stopping on the documented + no-progress signal: ``total_changes`` delta < 2 (the command's own + INSERT is 1 change). - Each command is its own implicit transaction (the connection runs - with ``isolation_level=None``), so the SQLite write lock is released - between commands and competing processes can interleave writes - mid-pass. Missing tables are valid schema variants (FTS variants are - optional, and ``optimize_fts_storage`` legitimately drops + backfills - these tables while writers keep running) and are skipped, mirroring - ``optimize_fts``. Other SQLite errors propagate to the caller. - - Returns the number of merge commands executed. + Each command is its own implicit transaction (``isolation_level=None``), + so competing processes interleave mid-pass. Missing tables are valid + variants (optimize_fts_storage drops + backfills them live) and are + skipped; other SQLite errors propagate. Returns commands executed. """ if isinstance(max_pages, bool) or not isinstance(max_pages, int): raise TypeError("max_pages must be an integer") @@ -2464,11 +1818,9 @@ class SessionSearchMixin: for tbl in self._FTS_TABLES: if not self._fts_table_exists(tbl): continue - # One-time (per instance) usermerge floor; the value is - # persisted in the index's config shadow table so future - # connections inherit it. Setting config is a metadata-only - # write — it never touches segment data. - if not getattr(self, "_fts_usermerge_floor_applied", False): + # One-time (per instance) usermerge floor; metadata-only write, + # persisted so future connections inherit it. + if not self._fts_usermerge_floor_applied: self._conn.execute( f"INSERT INTO {tbl}({tbl}, rank) " "VALUES('usermerge', 2)" diff --git a/hermes_state_telegram.py b/hermes_state_telegram.py new file mode 100644 index 0000000000..2c8dbf8501 --- /dev/null +++ b/hermes_state_telegram.py @@ -0,0 +1,529 @@ +"""Telegram DM topic-mode mixin for :class:`hermes_state.SessionDB`.""" + +from __future__ import annotations + +import logging +import sqlite3 +import time +from typing import Any, Dict, List, Optional + +from hermes_state_common import ( + _PREVIEW_ELIGIBLE_SQL, + _PREVIEW_RAW_SELECT, + _shape_preview, + _sql_session_last_active, +) + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +def _normalize_telegram_topic_profile_name(profile_name: Optional[str] = None) -> str: + """Empty/missing → ``"default"`` (single namespace for non-multiplexed + gateways). Multiplexed callers must pass the *routed* profile + (``source.profile``), never the process-global active profile.""" + name = str(profile_name or "").strip() + return name if name else "default" + + +class SessionTelegramTopicsMixin: + """Telegram DM topic-mode tables, bindings and lookups.""" + + def apply_telegram_topic_migration(self) -> None: + """Create Telegram DM topic-mode tables on explicit /topic opt-in. + + Deliberately NOT part of startup reconciliation: operators can upgrade + and keep the old bot behavior until a user runs /topic. + + Schema versions: v1 initial; v2 session_id FK ON DELETE CASCADE (pruning + clears bindings); v3 ``profile_name`` on both tables so multiplexed + gateways sharing one state.db isolate topic state per profile. + """ + # (table, column list, DDL body). profile_name leads the PK: a private + # chat_id is the user id, identical across bots sharing one state.db. + tables = ( + ( + "telegram_dm_topic_mode", + "profile_name, chat_id, user_id, enabled, activated_at, updated_at, " + "has_topics_enabled, allows_users_to_create_topics, " + "capability_checked_at, intro_message_id, pinned_message_id", + """ + profile_name TEXT NOT NULL DEFAULT 'default', + chat_id TEXT NOT NULL, + user_id TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + activated_at REAL NOT NULL, + updated_at REAL NOT NULL, + has_topics_enabled INTEGER, + allows_users_to_create_topics INTEGER, + capability_checked_at REAL, + intro_message_id TEXT, + pinned_message_id TEXT, + PRIMARY KEY (profile_name, chat_id) + """, + ), + ( + "telegram_dm_topic_bindings", + "profile_name, chat_id, thread_id, user_id, session_key, " + "session_id, managed_mode, linked_at, updated_at", + """ + profile_name TEXT NOT NULL DEFAULT 'default', + chat_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + user_id TEXT NOT NULL, + session_key TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + managed_mode TEXT NOT NULL DEFAULT 'auto', + linked_at REAL NOT NULL, + updated_at REAL NOT NULL, + PRIMARY KEY (profile_name, chat_id, thread_id) + """, + ), + ) + + def _do(conn): + for table, columns, ddl in tables: + conn.execute(f"CREATE TABLE IF NOT EXISTS {table} ({ddl})") + have = {row[1] for row in conn.execute(f"PRAGMA table_info('{table}')")} + if "profile_name" in have: + continue + # v1/v2 → v3. SQLite can't ALTER a PK or FK, so rebuild (this + # also supplies v2's ON DELETE CASCADE). Legacy rows land in + # "default" only, never replicated across profiles. + legacy_columns = columns.replace("profile_name, ", "", 1) + conn.executescript( + f""" + CREATE TABLE {table}_new ({ddl}); + INSERT INTO {table}_new ({columns}) + SELECT 'default', {legacy_columns} FROM {table}; + DROP TABLE {table}; + ALTER TABLE {table}_new RENAME TO {table}; + """ + ) + + # Indexes after any rebuild: the user index needs profile_name. + conn.executescript( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session + ON telegram_dm_topic_bindings(session_id); + + CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user + ON telegram_dm_topic_bindings(profile_name, user_id, chat_id); + """ + ) + + conn.execute( + "INSERT INTO state_meta (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ("telegram_dm_topic_schema_version", "3"), + ) + self._execute_write(_do) + + def enable_telegram_topic_mode( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + has_topics_enabled: Optional[bool] = None, + allows_users_to_create_topics: Optional[bool] = None, + ) -> None: + """Enable Telegram DM topic mode for one private chat/user. + + Owns the explicit topic migration; SessionDB startup must not create + these tables. Multiplexed callers pass the routed ``source.profile``, + not the process-global active profile. + """ + self.apply_telegram_topic_migration() + now = time.time() + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _to_int(value: Optional[bool]) -> Optional[int]: + if value is None: + return None + return 1 if value else 0 + + self._write_sql( + """ + INSERT INTO telegram_dm_topic_mode ( + profile_name, chat_id, user_id, enabled, activated_at, updated_at, + has_topics_enabled, allows_users_to_create_topics, + capability_checked_at + ) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id) DO UPDATE SET + user_id = excluded.user_id, + enabled = 1, + updated_at = excluded.updated_at, + has_topics_enabled = excluded.has_topics_enabled, + allows_users_to_create_topics = excluded.allows_users_to_create_topics, + capability_checked_at = excluded.capability_checked_at + """, + ( + profile_name, + str(chat_id), + str(user_id), + now, + now, + _to_int(has_topics_enabled), + _to_int(allows_users_to_create_topics), + now, + ), + ) + + def disable_telegram_topic_mode( + self, + *, + chat_id: str, + profile_name: str = "default", + clear_bindings: bool = True, + ) -> None: + """Disable Telegram DM topic mode for one private chat. + + ``clear_bindings`` also drops the chat's bindings so a later re-enable + starts clean. Never creates the tables; absent tables are a no-op. + """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _do(conn): + try: + conn.execute( + "UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? " + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, str(chat_id)), + ) + if clear_bindings: + conn.execute( + "DELETE FROM telegram_dm_topic_bindings " + "WHERE profile_name = ? AND chat_id = ?", + (profile_name, str(chat_id)), + ) + except sqlite3.OperationalError: + return + self._execute_write(_do) + + def is_telegram_topic_mode_enabled( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + ) -> bool: + """Return whether Telegram DM topic mode is enabled for this chat/user.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT enabled FROM telegram_dm_topic_mode + WHERE profile_name = ? AND chat_id = ? AND user_id = ? + """, + (profile_name, str(chat_id), str(user_id)), + ).fetchone() + except sqlite3.OperationalError: + return False + if row is None: + return False + enabled = row[0] + return bool(enabled) + + def get_telegram_topic_binding( + self, + *, + chat_id: str, + thread_id: str, + profile_name: str = "default", + ) -> Optional[Dict[str, Any]]: + """Return the session binding for a Telegram DM topic, if present.""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT * FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? + """, + (profile_name, str(chat_id), str(thread_id)), + ).fetchone() + except sqlite3.OperationalError: + return None + return dict(row) if row else None + + def list_telegram_topic_bindings_for_chat( + self, + *, + chat_id: str, + profile_name: str = "default", + ) -> List[Dict[str, Any]]: + """All bindings for one chat, newest first. Read-only: [] when the + table is absent (never triggers the migration).""" + profile_name = _normalize_telegram_topic_profile_name(profile_name) + with self._read_ctx() as conn: + try: + rows = conn.execute( + "SELECT * FROM telegram_dm_topic_bindings " + "WHERE profile_name = ? AND chat_id = ? " + "ORDER BY updated_at DESC", + (profile_name, str(chat_id)), + ).fetchall() + except sqlite3.OperationalError: + return [] + return [dict(row) for row in rows] + + def get_telegram_topic_binding_by_session( + self, + *, + session_id: str, + ) -> Optional[Dict[str, Any]]: + """Reverse lookup via the UNIQUE INDEX on session_id; None when + unbound or the table is absent.""" + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT * FROM telegram_dm_topic_bindings + WHERE session_id = ? + """, + (str(session_id),), + ).fetchone() + except sqlite3.OperationalError: + return None + return dict(row) if row else None + + def delete_telegram_topic_binding( + self, + *, + chat_id: str, + thread_id: str, + profile_name: str = "default", + ) -> int: + """Remove the binding row for one (chat, thread) pair. + + Called when the Bot API confirms a topic was deleted externally + (``Thread not found`` after the same-thread retry failed); otherwise + ``gateway.run._recover_telegram_topic_thread_id`` keeps redirecting + inbound messages to the dead topic. + + If this removes the chat's *last* binding, ``telegram_dm_topic_mode`` + is flipped to ``enabled = 0`` in the same transaction; otherwise the + chat stays in topic mode with zero lanes and a user who disabled topics + in the Telegram client (not via ``/topic off``) stays stuck. + + Returns the number of rows deleted; absent binding or unmigrated tables + are silent no-ops (never raise from a cleanup hot path). + """ + chat_id = str(chat_id) + thread_id = str(thread_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) + deleted = {"count": 0} + + def _do(conn): + try: + cursor = conn.execute( + """ + DELETE FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? AND thread_id = ? + """, + (profile_name, chat_id, thread_id), + ) + deleted["count"] = cursor.rowcount or 0 + except sqlite3.OperationalError: + deleted["count"] = 0 + return + if not deleted["count"]: + return + # Last binding gone → disable topic mode. Same transaction, so no + # read-after-prune race. + try: + remaining = conn.execute( + """ + SELECT 1 FROM telegram_dm_topic_bindings + WHERE profile_name = ? AND chat_id = ? LIMIT 1 + """, + (profile_name, chat_id), + ).fetchone() + if remaining is None: + conn.execute( + "UPDATE telegram_dm_topic_mode " + "SET enabled = 0, updated_at = ? " + "WHERE profile_name = ? AND chat_id = ?", + (time.time(), profile_name, chat_id), + ) + except sqlite3.OperationalError: + # telegram_dm_topic_mode absent — binding prune still stands. + pass + + self._execute_write(_do) + return deleted["count"] + + def bind_telegram_topic( + self, + *, + chat_id: str, + thread_id: str, + user_id: str, + session_key: str, + session_id: str, + managed_mode: str = "auto", + profile_name: str = "default", + ) -> None: + """Bind one Telegram DM topic thread to one Hermes session. + + A session may be linked to only one topic: rebinding the same pair is + idempotent; linking the session to a different topic raises ValueError. + """ + self.apply_telegram_topic_migration() + now = time.time() + chat_id = str(chat_id) + thread_id = str(thread_id) + user_id = str(user_id) + session_key = str(session_key) + session_id = str(session_id) + profile_name = _normalize_telegram_topic_profile_name(profile_name) + + def _do(conn): + existing_session = conn.execute( + """ + SELECT profile_name, chat_id, thread_id + FROM telegram_dm_topic_bindings + WHERE session_id = ? + """, + (session_id,), + ).fetchone() + if existing_session is not None: + if isinstance(existing_session, sqlite3.Row): + linked_profile = existing_session["profile_name"] + linked_chat = existing_session["chat_id"] + linked_thread = existing_session["thread_id"] + else: + linked_profile, linked_chat, linked_thread = existing_session + if ( + str(linked_profile) != profile_name + or str(linked_chat) != chat_id + or str(linked_thread) != thread_id + ): + raise ValueError("session is already linked to another Telegram topic") + + conn.execute( + """ + INSERT INTO telegram_dm_topic_bindings ( + profile_name, chat_id, thread_id, user_id, session_key, session_id, + managed_mode, linked_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(profile_name, chat_id, thread_id) DO UPDATE SET + user_id = excluded.user_id, + session_key = excluded.session_key, + session_id = excluded.session_id, + managed_mode = excluded.managed_mode, + updated_at = excluded.updated_at + """, + ( + profile_name, + chat_id, + thread_id, + user_id, + session_key, + session_id, + managed_mode, + now, + now, + ), + ) + self._execute_write(_do) + + def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool: + """True if the session is bound to any Telegram DM topic. Read-only: + absent tables (nobody ran ``/topic``) mean unbound → False.""" + with self._read_ctx() as conn: + try: + row = conn.execute( + """ + SELECT 1 FROM telegram_dm_topic_bindings + WHERE session_id = ? + LIMIT 1 + """, + (str(session_id),), + ).fetchone() + except sqlite3.OperationalError: + return False + return row is not None + + def list_unlinked_telegram_sessions_for_user( + self, + *, + chat_id: str, + user_id: str, + profile_name: str = "default", + limit: int = 10, + ) -> List[Dict[str, Any]]: + """This user's Telegram sessions not bound to a topic. + + Read-only: if the bindings table is absent, every session is unlinked + and a simpler query is used. Scoped by ``profile_name`` so multiplexed + profiles do not surface each other's sessions. + """ + profile_name = _normalize_telegram_topic_profile_name(profile_name) + # sessions.profile_name is NULL/empty for legacy rows → treat as default. + profile_clause = "AND COALESCE(NULLIF(TRIM(s.profile_name), ''), 'default') = ?" + with self._read_ctx() as conn: + try: + rows = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + AND {_PREVIEW_ELIGIBLE_SQL} + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + {_sql_session_last_active("s")} AS last_active + FROM sessions s + LEFT JOIN system_prompts sp + ON sp.hash = s.system_prompt_hash + WHERE s.source = 'telegram' + AND s.user_id = ? + {profile_clause} + AND NOT EXISTS ( + SELECT 1 FROM telegram_dm_topic_bindings b + WHERE b.session_id = s.id + ) + ORDER BY last_active DESC, s.started_at DESC + LIMIT ? + """, + (str(user_id), profile_name, int(limit)), + ).fetchall() + except sqlite3.OperationalError: + rows = conn.execute( + f""" + SELECT s.*, + COALESCE(sp.prompt, s.system_prompt) + AS _system_prompt_resolved, + COALESCE( + (SELECT {_PREVIEW_RAW_SELECT} + FROM messages m + WHERE m.session_id = s.id AND m.role = 'user' AND m.content IS NOT NULL + AND {_PREVIEW_ELIGIBLE_SQL} + ORDER BY m.timestamp, m.id LIMIT 1), + '' + ) AS _preview_raw, + {_sql_session_last_active("s")} AS last_active + FROM sessions s + LEFT JOIN system_prompts sp + ON sp.hash = s.system_prompt_hash + WHERE s.source = 'telegram' + AND s.user_id = ? + ORDER BY last_active DESC, s.started_at DESC + LIMIT ? + """, + (str(user_id), int(limit)), + ).fetchall() + + sessions: List[Dict[str, Any]] = [] + for row in rows: + session = self._session_row_dict(row) + session["preview"] = _shape_preview(session.pop("_preview_raw", "")) + sessions.append(session) + return sessions diff --git a/hermes_state_titles.py b/hermes_state_titles.py new file mode 100644 index 0000000000..3d6dbea0f5 --- /dev/null +++ b/hermes_state_titles.py @@ -0,0 +1,305 @@ +"""Session title mixin for SessionDB: sanitizing, auto/user provenance +ranking, and lineage-aware lookups.""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Dict, Optional + +from agent.message_sanitization import _sanitize_surrogates +from hermes_state_common import _COMPRESSION_CHILD_SQL, escape_like as _escape_like + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionTitlesMixin: + """Sanitizing, ranking auto/user titles, lineage-aware lookups.""" + + @classmethod + def _title_rank(cls, source: Optional[str]) -> int: + """Rank a stored title_source. + + NULL (pre-provenance rows) is indistinguishable from a manual ``/title`` + of that era, so it ranks as ``user``: auto-titling only ever fills + genuinely empty legacy titles. + """ + if source is None: + return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER] + return cls._TITLE_SOURCE_RANK.get(str(source), 0) + + @staticmethod + def sanitize_title(title: Optional[str]) -> Optional[str]: + """Strip control/zero-width/bidi chars, collapse whitespace, normalize + empty to None. Raises ValueError if longer than MAX_TITLE_LENGTH + after cleaning.""" + from hermes_state import SessionDB + if not title: + return None + + # Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError). + title = _sanitize_surrogates(title) + + # ASCII controls, keeping \t \n \r so the whitespace collapse below + # turns them into spaces. + cleaned = re.sub(r'[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]', '', title) + + # Zero-width, bidi override, object-replacement, interlinear annotation. + cleaned = re.sub( + r'[\u200b-\u200f\u2028-\u202e\u2060-\u2069\ufeff\ufffc\ufff9-\ufffb]', + '', cleaned, + ) + + cleaned = re.sub(r'\s+', ' ', cleaned).strip() + + if not cleaned: + return None + + if len(cleaned) > SessionDB.MAX_TITLE_LENGTH: + raise ValueError( + f"Title too long ({len(cleaned)} chars, max {SessionDB.MAX_TITLE_LENGTH})" + ) + + return cleaned + + def _is_compression_ancestor( + self, conn, *, ancestor_id: str, descendant_id: str + ) -> bool: + """True if *ancestor_id* is a compression predecessor of *descendant_id*. + + Uses the canonical continuation edge ``_COMPRESSION_CHILD_SQL`` (parent + ended with ``end_reason = 'compression'`` and child started at/after its + ``ended_at``), which excludes delegate/branch children that also carry + ``parent_session_id``. One recursive CTE so the edge is defined once. + """ + if not ancestor_id or not descendant_id or ancestor_id == descendant_id: + return False + edge = _COMPRESSION_CHILD_SQL.format(a="child") + row = conn.execute( + f""" + WITH RECURSIVE ancestors(id) AS ( + SELECT ? + UNION + SELECT parent.id + FROM ancestors a + JOIN sessions child ON child.id = a.id + JOIN sessions parent ON parent.id = child.parent_session_id + WHERE {edge} + ) + SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1 + """, + (descendant_id, ancestor_id, descendant_id), + ).fetchone() + return row is not None + + def _set_session_title( + self, + session_id: str, + title: str, + *, + source: str, + ) -> bool: + """Write a title, enforcing provenance precedence. + + A ``user`` write always lands. ``derived``/``llm`` land only when the + row is untitled or holds strictly lower authority, so derived upgrades + to llm exactly once, nothing overwrites a user name, and re-running the + titler on an llm row is a no-op (stops sessions renaming themselves). + No writer may move a hidden canonical Bot Chat off its title. + + Read and write are one compare-and-swap in a single transaction, so a + manual ``/title`` racing an in-flight generation is not clobbered. + """ + title = self.sanitize_title(title) + is_user = source == self.TITLE_SOURCE_USER + new_rank = self._title_rank(source) if not is_user else None + + def _do(conn): + current = conn.execute( + "SELECT title, title_source, hidden FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + if current is None: + return 0 + # The canonical Bot Chat's NAME is its identity: Bot Mode resolves it + # by exact-title lookup on every open, so a rename orphans the whole + # conversation (next open mints an empty replacement and UNIQUE(title) + # blocks renaming back). Refuse here, the single write path every + # surface funnels through. Hidden is the discriminator: canonical + # chats are born hidden; a visible session merely named "Bot Chat" + # stays renameable. Provenance-blind so the auto-titler no-ops too. + if ( + (current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE + and bool(current["hidden"]) + and title != self.CANONICAL_BOT_CHAT_TITLE + ): + if is_user: + raise ValueError( + "This is the bot's canonical Bot Chat — its name is its " + "identity, and renaming it would orphan the conversation. " + "To start fresh, create a new bot instead." + ) + return 0 + if not is_user and current["title"] is not None: + if self._title_rank(current["title_source"]) >= new_rank: + return 0 + + if title: + cursor = conn.execute( + "SELECT id FROM sessions WHERE title = ? AND id != ?", + (title, session_id), + ) + conflict = cursor.fetchone() + if conflict: + conflict_id = conflict["id"] + # If the conflicting holder is a hidden compressed ancestor + # of this continuation, the user cannot free the title, so + # transfer it onto the tip. Uniqueness and lineage are kept. + if self._is_compression_ancestor( + conn, ancestor_id=conflict_id, descendant_id=session_id + ): + conn.execute( + "UPDATE sessions SET title = NULL WHERE id = ?", + (conflict_id,), + ) + else: + raise ValueError( + f"Title '{title}' is already in use by session {conflict_id}" + ) + # CAS on the values just read (``IS`` is NULL-safe): a concurrent + # write between the SELECT and here loses instead of being overwritten. + cursor = conn.execute( + "UPDATE sessions SET title = ?, title_source = ? " + "WHERE id = ? AND title IS ? AND title_source IS ?", + ( + title, + source if title else None, + session_id, + current["title"], + current["title_source"], + ), + ) + return cursor.rowcount + + rowcount = self._execute_write(_do) + return rowcount > 0 + + def set_session_title(self, session_id: str, title: str) -> bool: + """Set a title on the user's behalf (``user`` provenance; auto-titling + never replaces it). Empty clears the title. Raises ValueError on a + title conflict or validation failure. Automatic callers must use + :meth:`set_auto_title`.""" + return self._set_session_title( + session_id, title, source=self.TITLE_SOURCE_USER + ) + + def set_auto_title(self, session_id: str, title: str, *, source: str) -> bool: + """Set an automatic title; False (untouched) when a higher-authority + title already holds the row.""" + if source not in (self.TITLE_SOURCE_DERIVED, self.TITLE_SOURCE_LLM): + raise ValueError(f"invalid automatic title source: {source!r}") + return self._set_session_title(session_id, title, source=source) + + def set_auto_title_if_empty(self, session_id: str, title: str) -> bool: + """Back-compat shim (third-party plugins reference it by name); new + code calls :meth:`set_auto_title` with an explicit source.""" + return self.set_auto_title( + session_id, title, source=self.TITLE_SOURCE_LLM + ) + + def get_session_title(self, session_id: str) -> Optional[str]: + """Get the title for a session, or None.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title FROM sessions WHERE id = ?", (session_id,) + ) + row = cursor.fetchone() + return row["title"] if row else None + + def get_session_title_source(self, session_id: str) -> Optional[str]: + """Get the provenance of a session's title, or None when untitled.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title, title_source FROM sessions WHERE id = ?", + (session_id,), + ) + row = cursor.fetchone() + if not row or row["title"] is None: + return None + return row["title_source"] + + def set_session_title_source(self, session_id: str, source: str) -> bool: + """Overwrite a title's provenance without touching the text: a title + copied across a compression rotation keeps the original's authority.""" + if source not in self._TITLE_SOURCE_RANK: + raise ValueError(f"invalid title source: {source!r}") + + return self._write_rowcount( + "UPDATE sessions SET title_source = ? " + "WHERE id = ? AND title IS NOT NULL", + (source, session_id), + ) > 0 + + def get_session_by_title(self, title: str) -> Optional[Dict[str, Any]]: + """Look up a session by exact title. Returns session dict or None.""" + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT s.*, " + "COALESCE(sp.prompt, s.system_prompt) AS _system_prompt_resolved " + "FROM sessions s " + "LEFT JOIN system_prompts sp ON sp.hash = s.system_prompt_hash " + "WHERE s.title = ?", + (title,), + ) + row = cursor.fetchone() + return self._session_row_dict(row) if row else None + + def resolve_session_by_title(self, title: str) -> Optional[str]: + """Resolve a title to a session ID, preferring the latest "title #N" + continuation over the exact match.""" + exact = self.get_session_by_title(title) + + # Escape LIKE wildcards so "%"/"_" in titles cannot false-match. + escaped = _escape_like(title) + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT id, title, started_at FROM sessions " + "WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC", + (f"{escaped} #%",), + ) + numbered = cursor.fetchall() + + if numbered: + return numbered[0]["id"] + elif exact: + return exact["id"] + return None + + def get_next_title_in_lineage(self, base_title: str) -> str: + """Next title in a lineage ("my session" → "my session #2"): strip any + " #N" suffix, then increment the highest existing number.""" + match = re.match(r'^(.*?) #(\d+)$', base_title) + if match: + base = match.group(1) + else: + base = base_title + + escaped = _escape_like(base) + with self._read_ctx() as conn: + cursor = conn.execute( + "SELECT title FROM sessions WHERE title = ? OR title LIKE ? ESCAPE '\\'", + (base, f"{escaped} #%"), + ) + existing = [row["title"] for row in cursor.fetchall()] + + if not existing: + return base + + max_num = 1 # The unnumbered original counts as #1 + for t in existing: + m = re.match(r'^.* #(\d+)$', t) + if m: + max_num = max(max_num, int(m.group(1))) + + return f"{base} #{max_num + 1}" diff --git a/hermes_state_usage.py b/hermes_state_usage.py new file mode 100644 index 0000000000..9abae52aa7 --- /dev/null +++ b/hermes_state_usage.py @@ -0,0 +1,597 @@ +"""Token/usage accounting mixin for SessionDB: the coalescing background +token writer, per-model usage rows, and billing-route columns. Writer thread +state lives on the SessionDB instance.""" + +from __future__ import annotations + +import atexit +import logging +import threading +import time +import weakref +from typing import Any, Dict, List, Optional, Tuple + +# caplog tests pin the "hermes_state" logger name. +logger = logging.getLogger("hermes_state") + + +class SessionUsageMixin: + """Coalesced token writer, per-model usage rows, billing route.""" + + def update_session_billing_route( + self, + session_id: str, + *, + provider: str, + base_url: str, + billing_mode: Optional[str] = None, + ) -> None: + """Unconditionally set the billing route (``update_token_counts`` only + COALESCE-fills NULLs) so the dashboard reflects the latest /model switch. + + Also nulls ``system_prompt`` so the cached snapshot (stale ``Model:`` / + ``Provider:`` header) is rebuilt, like ``update_session_model``. + """ + # Barrier against queued token deltas — see update_session_model. + self.flush_token_counts() + + def _do(conn): + conn.execute( + """UPDATE sessions SET + billing_provider = ?, + billing_base_url = ?, + billing_mode = COALESCE(?, billing_mode), + system_prompt = NULL, + system_prompt_hash = NULL + WHERE id = ?""", + (provider, base_url, billing_mode, session_id), + ) + self._delete_unreferenced_system_prompts(conn) + self._execute_write(_do) + + def queue_token_counts(self, session_id: str, **kwargs) -> None: + """Enqueue a token/cost delta for the background writer. + + Same kwargs and semantics as :meth:`update_token_counts`, applied + asynchronously; cheap enough for the turn thread. After close() has + stopped the writer, falls back to the synchronous path and may raise. + """ + with self._token_queue_cond: + thread = self._token_writer_thread + writer_stopped = self._token_writer_stop and ( + thread is None or not thread.is_alive() + ) + if not writer_stopped: + self._token_queue.append((session_id, kwargs)) + if thread is None or not thread.is_alive(): + # Daemon so exit never hangs on accounting; the atexit hook + # (registered once per instance) drains leftovers. Checking + # ``not is_alive()`` rather than ``is None`` respawns a writer + # that died from an unexpected escape, otherwise deltas + # would pile up until a reader's flush drained them. + thread = threading.Thread( + target=self._token_writer_loop, + name="session-db-token-writer", + daemon=True, + ) + self._token_writer_thread = thread + thread.start() + if self._token_atexit_hook is None: + self_ref = weakref.ref(self) + + def _drain_at_exit() -> None: + db = self_ref() + if db is not None: + db._drain_token_queue_at_exit() + + self._token_atexit_hook = _drain_at_exit + atexit.register(_drain_at_exit) + self._token_queue_cond.notify_all() + if writer_stopped: + # close() ran (a stop-flagged but live writer still accepts; its + # loop drains before exiting). Enqueueing now would drop the delta + # silently — no writer, atexit hook gone — so apply inline and let a + # closed-connection failure raise at the call site. + self.update_token_counts(session_id, **kwargs) + + def flush_token_counts(self, timeout: float = 5.0) -> bool: + """Block until every queued token delta has been applied. + + False on timeout (callers then read totals stale by the queued deltas). + Never raises: apply failures are logged by the writer. + """ + # Lock-free fast path: reads queue-then-busy (see ordering notes below). + if not self._token_queue and not self._token_writer_busy: + return True + batch = None + with self._token_queue_cond: + deadline = time.monotonic() + timeout + while self._token_queue or self._token_writer_busy: + # A live writer is authoritative even when stop-flagged: draining + # here would race its in-flight batch, and newer deltas committing + # before older ones breaks last-non-None-wins / first-accounted- + # route / COALESCE-backfill fields. Only a dead writer lets the + # caller take leftovers; re-checked each wakeup because the writer + # can exit mid-wait with deltas enqueued after its final check. + # busy is claimed while draining so a concurrent flush cannot + # report drained or pop a newer delta while this batch is + # unapplied: a claimed busy means "wait", never "drain alongside". + thread = self._token_writer_thread + if ( + (thread is None or not thread.is_alive()) + and not self._token_writer_busy + ): + self._token_writer_busy = True + batch = list(self._token_queue) + self._token_queue.clear() + break + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._token_queue_cond.wait(remaining) + if batch: + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + return True + + def _token_writer_loop(self) -> None: + while True: + with self._token_queue_cond: + idle_deadline = time.monotonic() + self._TOKEN_WRITER_IDLE_SECONDS + while not self._token_queue and not self._token_writer_stop: + remaining = idle_deadline - time.monotonic() + if remaining <= 0: + # Retire under the same lock queue_token_counts() uses to + # decide to spawn, so no delta strands behind an exiting worker. + self._token_writer_thread = None + return + self._token_queue_cond.wait(remaining) + if not self._token_queue: + self._token_writer_thread = None + return # stop requested and fully drained + # busy BEFORE clearing the queue: flush's lock-free fast path + # reads queue-then-busy and must never see "empty and idle" + # while a popped batch is unapplied. + self._token_writer_busy = True + batch = list(self._token_queue) + self._token_queue.clear() + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + + def _apply_token_batch(self, batch: List[Tuple[str, Dict[str, Any]]]) -> None: + """Apply queued deltas in order, coalescing where safe. Never raises.""" + try: + coalesced = self._coalesce_token_deltas(batch) + except Exception as exc: + # Coalescing must never kill the writer (callers cannot observe a + # dead one); the merge is only an optimization. + logger.warning( + "async token accounting: coalesce failed, applying raw " + "batch: %s", exc, + ) + coalesced = batch + for session_id, kwargs in coalesced: + try: + self.update_token_counts(session_id, **kwargs) + except Exception as exc: + # Accounting loss is logged, never raised into a turn. + logger.warning( + "async token accounting: apply failed (session=%s): %s", + session_id, exc, + ) + + def _coalesce_token_deltas( + self, batch: List[Tuple[str, Dict[str, Any]]] + ) -> List[Tuple[str, Dict[str, Any]]]: + """Merge adjacent incremental deltas with an identical route, so + ordering across sessions and /model switches is preserved exactly. + absolute=True deltas never merge.""" + groups: List[Tuple[Optional[tuple], str, Dict[str, Any]]] = [] + for session_id, kwargs in batch: + key = None + if not kwargs.get("absolute"): + key = (session_id,) + tuple( + kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS + ) + if groups and key is not None and groups[-1][0] == key: + merged = groups[-1][2] + for f in self._TOKEN_DELTA_SUM_FIELDS: + merged[f] = merged.get(f, 0) + kwargs.get(f, 0) + for f in self._TOKEN_DELTA_COST_FIELDS: + value = kwargs.get(f) + if value is not None: + # All-None runs stay None so COALESCE keeps the stored value. + merged[f] = (merged.get(f) or 0.0) + value + else: + groups.append((key, session_id, dict(kwargs))) + return [(sid, kw) for _, sid, kw in groups] + + def _stop_token_writer(self, join_timeout: float = 10.0) -> None: + """Stop the writer thread and drain remaining deltas. Never raises.""" + with self._token_queue_cond: + self._token_writer_stop = True + self._token_queue_cond.notify_all() + thread = self._token_writer_thread + if thread is not None and thread.is_alive(): + thread.join(timeout=join_timeout) + if thread.is_alive(): + # Writer stuck mid-apply: leave deltas unapplied rather than + # race it and misorder/double-count. + logger.warning( + "async token accounting: writer did not stop within %.0fs; " + "%d queued delta(s) not persisted", + join_timeout, len(self._token_queue), + ) + return + # Writer gone: apply leftovers synchronously under the same busy + # protocol. Wait out a flush caller-drain that already claimed busy — + # close() nulls the connection right after this returns and must not + # yank it mid-batch. + with self._token_queue_cond: + deadline = time.monotonic() + join_timeout + while self._token_writer_busy: + remaining = deadline - time.monotonic() + if remaining <= 0: + logger.warning( + "async token accounting: concurrent drain did not " + "finish within %.0fs; %d queued delta(s) not persisted", + join_timeout, len(self._token_queue), + ) + return + self._token_queue_cond.wait(remaining) + # busy BEFORE clearing the queue (same ordering as the writer loop), + # or flush's lock-free fast path could see "empty and idle". + batch = list(self._token_queue) + if batch: + self._token_writer_busy = True + self._token_queue.clear() + if batch: + try: + self._apply_token_batch(batch) + finally: + with self._token_queue_cond: + self._token_writer_busy = False + self._token_queue_cond.notify_all() + + def _drain_token_queue_at_exit(self) -> None: + try: + self._stop_token_writer() + except Exception: + pass # never fatal at interpreter shutdown + + def update_token_counts( + self, + session_id: str, + input_tokens: int = 0, + output_tokens: int = 0, + model: str = None, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, + estimated_cost_usd: Optional[float] = None, + actual_cost_usd: Optional[float] = None, + cost_status: Optional[str] = None, + cost_source: Optional[str] = None, + pricing_version: Optional[str] = None, + billing_provider: Optional[str] = None, + billing_base_url: Optional[str] = None, + billing_mode: Optional[str] = None, + api_call_count: int = 0, + absolute: bool = False, + ) -> None: + """Update token counters and backfill model if unset. + + *absolute*=False increments (per-API-call deltas, CLI path); + *absolute*=True sets directly (gateway path, where the cached agent + holds cumulative totals). + """ + # Ensure the row exists: under concurrent load the initial + # create_session() may have failed on SQLite locking, and the UPDATE + # would silently affect 0 rows. + self._insert_session_row(session_id, "unknown", model=model) + if absolute: + sql = """UPDATE sessions SET + input_tokens = ?, + output_tokens = ?, + cache_read_tokens = ?, + cache_write_tokens = ?, + reasoning_tokens = ?, + estimated_cost_usd = COALESCE(?, 0), + actual_cost_usd = CASE + WHEN ? IS NULL THEN actual_cost_usd + ELSE ? + END, + cost_status = COALESCE(?, cost_status), + cost_source = COALESCE(?, cost_source), + pricing_version = COALESCE(?, pricing_version), + billing_provider = COALESCE(billing_provider, ?), + billing_base_url = COALESCE(billing_base_url, ?), + billing_mode = COALESCE(billing_mode, ?), + model = COALESCE(model, ?), + api_call_count = ? + WHERE id = ?""" + else: + sql = """UPDATE sessions SET + input_tokens = input_tokens + ?, + output_tokens = output_tokens + ?, + cache_read_tokens = cache_read_tokens + ?, + cache_write_tokens = cache_write_tokens + ?, + reasoning_tokens = reasoning_tokens + ?, + estimated_cost_usd = COALESCE(estimated_cost_usd, 0) + COALESCE(?, 0), + actual_cost_usd = CASE + WHEN ? IS NULL THEN actual_cost_usd + ELSE COALESCE(actual_cost_usd, 0) + ? + END, + cost_status = COALESCE(?, cost_status), + cost_source = COALESCE(?, cost_source), + pricing_version = COALESCE(?, pricing_version), + billing_provider = COALESCE(billing_provider, ?), + billing_base_url = COALESCE(billing_base_url, ?), + billing_mode = COALESCE(billing_mode, ?), + model = COALESCE(model, ?), + api_call_count = COALESCE(api_call_count, 0) + ? + WHERE id = ?""" + has_accounted_usage = bool( + input_tokens or output_tokens or cache_read_tokens + or cache_write_tokens or reasoning_tokens or api_call_count + or estimated_cost_usd or actual_cost_usd + ) + params = ( + input_tokens, + output_tokens, + cache_read_tokens, + cache_write_tokens, + reasoning_tokens, + estimated_cost_usd, + actual_cost_usd, + actual_cost_usd, + cost_status, + cost_source, + pricing_version, + billing_provider if has_accounted_usage else None, + billing_base_url if has_accounted_usage else None, + billing_mode if has_accounted_usage else None, + model if has_accounted_usage else None, + api_call_count, + session_id, + ) + # Per-model attribution: the sessions row keeps one (model, provider) + # pair, so a mid-session /model switch would attribute every token to + # the initial model. Each delta carries the route active at call time + # and is recorded into session_model_usage keyed by it. Only the + # incremental path records here: absolute cumulative updates cannot be + # split back into routes; Insights reconciles the residual instead. + record_model_usage = (not absolute) and ( + input_tokens or output_tokens or cache_read_tokens + or cache_write_tokens or reasoning_tokens or api_call_count + or estimated_cost_usd + ) + + def _do(conn): + row = conn.execute( + "SELECT model, billing_provider, api_call_count FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + existing_model = row["model"] if row is not None else None + existing_provider = row["billing_provider"] if row is not None else None + existing_api_calls = int((row["api_call_count"] if row is not None else 0) or 0) + + # create_session records the requested route before any API call. + # If that fails and fallback succeeds, the first accounted usage is + # the authoritative route; after that keep the row as is (one row + # cannot represent mixed-provider usage). + first_accounted_route = ( + existing_api_calls == 0 + and has_accounted_usage + and bool(model) + and bool(billing_provider) + and (existing_model != model or existing_provider != billing_provider) + ) + if first_accounted_route: + conn.execute( + """UPDATE sessions + SET model = ?, billing_provider = ?, + billing_base_url = ?, billing_mode = ? + WHERE id = ?""", + (model, billing_provider, billing_base_url, billing_mode, session_id), + ) + conn.execute(sql, params) + if record_model_usage: + self._record_model_usage( + conn, + session_id, + model=model, + billing_provider=billing_provider, + billing_base_url=billing_base_url, + billing_mode=billing_mode, + input_tokens=input_tokens, + output_tokens=output_tokens, + cache_read_tokens=cache_read_tokens, + cache_write_tokens=cache_write_tokens, + reasoning_tokens=reasoning_tokens, + estimated_cost_usd=estimated_cost_usd, + actual_cost_usd=actual_cost_usd, + cost_status=cost_status, + cost_source=cost_source, + api_call_count=api_call_count, + ) + self._execute_write(_do) + + def _record_model_usage( + self, + conn, + session_id: str, + *, + model: Optional[str], + billing_provider: Optional[str], + billing_base_url: Optional[str], + billing_mode: Optional[str], + input_tokens: int, + output_tokens: int, + cache_read_tokens: int, + cache_write_tokens: int, + reasoning_tokens: int, + estimated_cost_usd: Optional[float], + actual_cost_usd: Optional[float], + cost_status: Optional[str], + cost_source: Optional[str], + api_call_count: int, + task: str = "", + ) -> None: + """Accumulate a per-API-call usage delta into session_model_usage. + + Runs inside the caller's write transaction, after the ``sessions`` + UPDATE, so per-model rows stay consistent with the summary row. A + missing model/provider falls back to the session row (same COALESCE + behaviour as the summary update). ``task`` is ``''`` for the main loop; + auxiliary calls record their task name via :meth:`record_auxiliary_usage`. + """ + row = conn.execute( + "SELECT model, billing_provider, billing_base_url, billing_mode " + "FROM sessions WHERE id = ?", + (session_id,), + ).fetchone() + sess_model = row["model"] if row is not None else None + sess_provider = row["billing_provider"] if row is not None else None + sess_base_url = row["billing_base_url"] if row is not None else None + sess_billing_mode = row["billing_mode"] if row is not None else None + + # Aux rows must NOT inherit the main-loop route (vision on gemini while + # the main loop runs anthropic); missing info stays 'unknown'/empty. + if task: + eff_model = model or "unknown" + eff_provider = billing_provider or "" + eff_base_url = billing_base_url or "" + eff_billing_mode = billing_mode or "" + else: + eff_model = model or sess_model or "unknown" + eff_provider = billing_provider or sess_provider or "" + eff_base_url = billing_base_url or sess_base_url or "" + eff_billing_mode = billing_mode or sess_billing_mode or "" + now = time.time() + conn.execute( + """INSERT INTO session_model_usage ( + session_id, model, billing_provider, billing_base_url, billing_mode, + task, api_call_count, input_tokens, output_tokens, + cache_read_tokens, cache_write_tokens, reasoning_tokens, + estimated_cost_usd, actual_cost_usd, cost_status, cost_source, + first_seen, last_seen + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, model, billing_provider, billing_base_url, billing_mode, task) + DO UPDATE SET + api_call_count = api_call_count + excluded.api_call_count, + input_tokens = input_tokens + excluded.input_tokens, + output_tokens = output_tokens + excluded.output_tokens, + cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens, + cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens, + reasoning_tokens = reasoning_tokens + excluded.reasoning_tokens, + estimated_cost_usd = estimated_cost_usd + excluded.estimated_cost_usd, + actual_cost_usd = actual_cost_usd + excluded.actual_cost_usd, + cost_status = COALESCE(excluded.cost_status, cost_status), + cost_source = COALESCE(excluded.cost_source, cost_source), + last_seen = excluded.last_seen""", + ( + session_id, + eff_model, + eff_provider, + eff_base_url, + eff_billing_mode, + task or "", + api_call_count or 0, + input_tokens or 0, + output_tokens or 0, + cache_read_tokens or 0, + cache_write_tokens or 0, + reasoning_tokens or 0, + float(estimated_cost_usd or 0.0), + float(actual_cost_usd or 0.0), + cost_status, + cost_source, + now, + now, + ), + ) + + def record_auxiliary_usage( + self, + session_id: str, + task: str, + *, + model: Optional[str] = None, + billing_provider: Optional[str] = None, + billing_base_url: Optional[str] = None, + input_tokens: int = 0, + output_tokens: int = 0, + cache_read_tokens: int = 0, + cache_write_tokens: int = 0, + reasoning_tokens: int = 0, + estimated_cost_usd: Optional[float] = None, + api_call_count: int = 1, + ) -> None: + """Record an auxiliary LLM call's usage (vision, compression, title + generation, ...) against *session_id*. + + Writes a per-(model, provider, task) delta into ``session_model_usage`` + WITHOUT touching the ``sessions`` summary row: the gateway overwrites + session counters with absolute main-loop totals, so aux tokens there + would be clobbered or double-counted. Insights read the union. + ``api_call_count`` may aggregate N calls (background-review forks). + Best-effort: callers must never fail an aux call over accounting. + """ + if not session_id or not task: + return + # FK to sessions.id: same INSERT OR IGNORE guard as update_token_counts. + self._insert_session_row(session_id, "unknown") + + def _do(conn): + self._record_model_usage( + conn, + session_id, + model=model, + billing_provider=billing_provider, + billing_base_url=billing_base_url, + billing_mode=None, + input_tokens=input_tokens or 0, + output_tokens=output_tokens or 0, + cache_read_tokens=cache_read_tokens or 0, + cache_write_tokens=cache_write_tokens or 0, + reasoning_tokens=reasoning_tokens or 0, + estimated_cost_usd=estimated_cost_usd, + actual_cost_usd=None, + cost_status=None, + cost_source=None, + api_call_count=( + 1 if api_call_count is None else int(api_call_count) + ), + task=task, + ) + self._execute_write(_do) + + def usage_totals(self, *, min_message_count: int = 1, include_archived: bool = False) -> Dict[str, float]: + """Tokens and spend across the whole store (one scan), so the sidebar + total does not shrink with paging. Spend prefers the billed figure over + the estimate, the same precedence a single row renders.""" + where = ["parent_session_id IS NULL", "message_count >= ?"] + params: List[Any] = [min_message_count] + if not include_archived: + where.append("COALESCE(archived, 0) = 0") + + row = self._read_one( + f""" + SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0), + COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0) + FROM sessions + WHERE {' AND '.join(where)} + """, + params, + ) + + return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)} diff --git a/hermes_state_wal.py b/hermes_state_wal.py new file mode 100644 index 0000000000..a6a1ce9171 --- /dev/null +++ b/hermes_state_wal.py @@ -0,0 +1,821 @@ +"""SQLite journal-mode and PRAGMA policy for state.db. + +Split out of ``hermes_state.py``. Every name is re-imported there so +``hermes_state.`` keeps resolving, and tests that monkeypatch it keep +intercepting because intra-module calls to patched helpers go through a lazy +``from hermes_state import ...`` at call time. +""" + +from __future__ import annotations + +import logging +import sqlite3 +import sys +import threading +import time +from typing import Any, Dict, Optional + +from hermes_cli.sqlite_runtime import ( + is_sqlite_wal_reset_vulnerable as _is_sqlite_wal_reset_vulnerable, +) + +# Log-record parity with the origin module (caplog tests pin "hermes_state"). +logger = logging.getLogger("hermes_state") + + +# --------------------------------------------------------------------------- +# WAL-compatibility fallback +# --------------------------------------------------------------------------- +# WAL needs mmap shared memory and fcntl byte-range locks, which network +# filesystems (NFS, SMB/CIFS, some FUSE, WSL1) don't provide reliably — there +# ``PRAGMA journal_mode=WAL`` raises ``locking protocol`` (SQLITE_PROTOCOL). +# ZFS instead corrupts the -shm file under concurrent connection bursts (COW + +# mmap), presenting as ``disk I/O error``. Propagating either would silently +# break everything backed by state.db/kanban.db, so we fall back to +# ``journal_mode=DELETE`` (pre-WAL default, works on NFS/ZFS): readers block +# during a write, but it works. The WAL-reset-bug gate and the +# never-live-downgrade invariant are documented on apply_wal_with_fallback. +_WAL_INCOMPAT_MARKERS = ( + "locking protocol", # SQLITE_PROTOCOL on NFS/SMB + "not authorized", # Some FUSE mounts block WAL pragma outright + "disk i/o error", # ZFS SHM corruption under concurrent connections +) + + +# SQLite's default is -1 (unlimited), so state.db-wal would keep the high-water +# mark of the largest-ever transaction forever. See _apply_wal_size_limit(). +_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB + + +# Dedup sets: kanban_db.connect() runs on every kanban operation, so an +# undeduped fallback log line would repeat per connection and fill errors.log. +_wal_fallback_warned_paths: set[str] = set() + + +_wal_fallback_warned_lock = threading.Lock() + + +_wal_reset_bug_warned_paths: set[str] = set() + + +_wal_reset_bug_warned_lock = threading.Lock() + + +# "configured delete overridden by on-disk WAL" ERROR. +_delete_overridden_warned_paths: set[str] = set() + + +_delete_overridden_warned_lock = threading.Lock() + + +def _on_disk_journal_mode(conn: sqlite3.Connection) -> Optional[str]: + """Read the journal mode from the DB header; ``None`` if undeterminable. + + ``None`` (new DB, or PRAGMA failed) sends callers down their fail-closed + "unknown → refuse to downgrade" branch. ``disk i/o error`` can be transient + on virtualized block devices (XFS on cloud hosts), so it is retried a few + times first: transient EIO clears, deterministic filesystem errors do not. + """ + last_exc: Optional[Exception] = None + for _ in range(4): + try: + row = conn.execute("PRAGMA journal_mode").fetchone() + except sqlite3.OperationalError as exc: + last_exc = exc + if "disk i/o error" not in str(exc).lower(): + return None + time.sleep(0.05) + continue + if row is None: + return None + mode = row[0] + if isinstance(mode, bytes): # defensive: sqlite3 occasionally returns bytes + try: + mode = mode.decode("ascii") + except UnicodeDecodeError: + return None + return str(mode).strip().lower() if mode is not None else None + if last_exc is not None: + logger.debug( + "_on_disk_journal_mode: retries exhausted on disk read (%s)", last_exc + ) + return None + + +def _apply_wal_size_limit(conn: sqlite3.Connection) -> None: + """Bound the WAL so it returns space to the OS after big transactions. + + SQLite's default ``journal_size_limit`` is -1: a checkpointed WAL is reused + in place, never truncated, so ``state.db-wal`` keeps the high-water mark + of the largest transaction ever run. One bulk op strands gigabytes — + ``hermes sessions optimize`` on a 3 GB state.db left a 3 GB WAL and filled + the disk, so the maintenance command was self-defeating on the largest + DBs. With a limit, each checkpoint truncates the WAL back to it; 64 MiB is + above normal transaction sizes (steady-state commits never pay a truncate) + while capping slack predictably. kanban_db uses ``wal_autocheckpoint=100``. + + Best-effort: never raises — failure only costs disk slack and must not + prevent the database from opening. + """ + try: + conn.execute(f"PRAGMA journal_size_limit={_WAL_SIZE_LIMIT_BYTES}") + except sqlite3.OperationalError as exc: # pragma: no cover - defensive + logger.debug("journal_size_limit not applied: %s", exc) + + +def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None: + """Enable ``PRAGMA checkpoint_fullfsync`` on macOS (no-op elsewhere). + + Apple's ``fsync(2)`` guarantees neither data-on-platter nor write ordering, + so WAL's corruption-safety assumption fails on Darwin without ``F_FULLFSYNC``. + A launchd shutdown drops the page cache (power-loss for in-flight pages), so + a checkpoint that "reported" durable can leave a malformed ``state.db``; + a plain in-session kill survives via the page cache. The barrier applies + only at checkpoint boundaries (~+0.1 ms/commit vs ~+4 ms for + ``fullfsync=1``). Best-effort: never raises. + """ + if sys.platform != "darwin": + return + try: + conn.execute("PRAGMA checkpoint_fullfsync=1") + except sqlite3.OperationalError: + pass + + +def _enforce_macos_synchronous_full(conn: sqlite3.Connection) -> None: + """Enforce ``PRAGMA synchronous=FULL`` on macOS to prevent btree corruption. + + With NORMAL, a WAL checkpoint racing process termination (launchd shutdown) + can leave half-written btree pages (``btreeInitPage error 11``) because + Darwin's ``fsync()`` guarantees neither ordering nor durability. Called + after every successful WAL activation so a prior connection's NORMAL never + sticks. Best-effort: never raises. + """ + if sys.platform != "darwin": + return + try: + conn.execute("PRAGMA synchronous=FULL") + except sqlite3.OperationalError: + pass + + +def is_sqlite_wal_reset_vulnerable( + version_info: Optional[tuple] = None, +) -> bool: + """True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2; + fixed 3.51.3+, backports 3.50.7 / 3.44.6). Pre-WAL libraries are safe. + https://sqlite.org/wal.html#walresetbug + """ + info = version_info if version_info is not None else sqlite3.sqlite_version_info + return _is_sqlite_wal_reset_vulnerable(info) + + +def sqlite_source_id() -> str: + """Return ``sqlite_source_id()``, or an empty string when unavailable.""" + try: + conn = sqlite3.connect(":memory:") + try: + row = conn.execute("SELECT sqlite_source_id()").fetchone() + finally: + conn.close() + except sqlite3.Error: + return "" + if not row or row[0] is None: + return "" + return str(row[0]) + + +def _database_has_content(conn: sqlite3.Connection) -> bool: + """Whether the file already holds pages (existing vs brand-new DB). + + ``PRAGMA page_count`` is a lock-free header read. Fail-quiet: any error + answers False, because the only caller gates a warning on this and an + unknown-answer warning would fire on every fresh database — exactly where + there is provably no operator choice being overwritten. + """ + try: + row = conn.execute("PRAGMA page_count").fetchone() + except sqlite3.Error: + return False + if not row or row[0] is None: + return False + try: + return int(row[0]) > 0 + except (TypeError, ValueError): + return False + + +def resolve_journal_mode() -> str: + """Return the configured journal mode (``wal`` or ``delete``). + + ``database.journal_mode`` in config.yaml is the canonical operator setting; + ``wal`` is the default, ``delete`` is for filesystems without WAL-safe + durability (macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``. + """ + try: + from hermes_cli.config import load_config_readonly + + config = load_config_readonly() or {} + database = config.get("database", {}) + if not isinstance(database, dict): + return "wal" + raw = database.get("journal_mode", "wal") + except Exception: + return "wal" + + if not isinstance(raw, str): + return "wal" + mode = raw.strip().lower() + return mode if mode in ("wal", "delete") else "wal" + + +class WalUnsupportedError(sqlite3.OperationalError): + """Raised by :func:`apply_wal_with_fallback` when ``require_wal=True`` and + the filesystem cannot provide WAL — whether SQLite *raised* + ``SQLITE_PROTOCOL`` or (macOS NFS) silently returned the still-effective + mode. Subclasses ``OperationalError`` so existing DB-init handlers still + catch it while WAL-mandating callers can catch the narrower type. + """ + + +def apply_wal_with_fallback( + conn: sqlite3.Connection, + *, + db_label: str = "state.db", + require_wal: bool = False, +) -> str: + """Set ``journal_mode=WAL`` on ``conn``, falling back to DELETE on failure. + + Returns the mode actually set (``"wal"`` or ``"delete"``). Shared by + :class:`SessionDB` and ``hermes_cli.kanban_db.connect`` for identical + fallback behavior. + + On WAL-incompatible filesystems (NFS, SMB, some FUSE, ZFS) SQLite either + raises ``OperationalError`` ("locking protocol" / "disk I/O error") or — + macOS NFS / SMB / AgentFS NFS overlay — silently refuses and leaves the DB + in DELETE. Either way we log at ERROR (a write now blocks readers — a real + concurrency loss) and fall back to DELETE so the feature keeps working. + ``require_wal=True`` raises :class:`WalUnsupportedError` instead; all + current callers keep the default so NFS-homed installs work. + + On SQLite builds with the WAL-reset bug (https://sqlite.org/wal.html#walresetbug, + fixed 3.51.3+, backports 3.50.7 / 3.44.6), refuse to enable WAL on + fresh / non-WAL databases; an already-WAL DB keeps WAL with a warning. + This gate is deliberately RETAINED: an attempt to revert it (theory: DELETE + was "the mode that corrupts") was confounded — its clean WAL result came + from SQLite 3.53.1, which also carries 3.51.0's close()-broken-POSIX-lock + defenses. Re-measured on the bundled 3.50.4 with the lock fix, WAL and + DELETE are both clean, so there is no evidence WAL is safer; keep new + databases out of WAL until a fixed runtime ships. + + Invariant on every path (NFS and WAL-reset alike): never downgrade to + DELETE if the on-disk header reports WAL or the mode cannot be read (see + _on_disk_journal_mode). Other gateway/cron/worker connections may hold the + DB open, and a live downgrade destroys their committed-but-uncheckpointed + transactions. + + The ERROR is deduplicated per ``db_label``: once per process per DB, so + state.db and kanban.db on one NFS mount each log once. + """ + from hermes_state import is_sqlite_wal_reset_vulnerable, resolve_journal_mode + configured = resolve_journal_mode() + + # Vulnerable SQLite: never enable WAL on new/non-WAL files. Resolve the + # operator setting first so an explicit DELETE request still verifies SQLite + # accepted DELETE rather than silently returning MEMORY or another mode. + if is_sqlite_wal_reset_vulnerable(): + return _apply_delete_for_wal_reset_bug( + conn, + db_label=db_label, + require_delete=configured == "delete", + ) + + # Read-only probe — no flock, no checkpoint, no WAL/SHM unlink — so + # WAL-init cannot unlink files other connections hold open. + current_mode = _on_disk_journal_mode(conn) + if current_mode == "wal": + if configured == "delete": + # Never-live-downgrade keeps WAL; tell the operator their delete did not apply. + _log_configured_delete_overridden_once(db_label) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + + # Honor the canonical database.journal_mode setting (on-disk WAL DBs were + # returned above and are never live-downgraded). + if configured == "delete": + if current_mode is None: + # Probe failed (locked/busy): another process may hold this DB open + # in WAL, so ownership is not provably exclusive and flipping modes + # could destroy a concurrent writer's committed-but-uncheckpointed + # transactions. Fail loudly — the operator asked for DELETE and we + # cannot verify it. + raise sqlite3.OperationalError( + "could not verify journal mode before applying configured " + "journal_mode=delete (database is locked — possible " + "concurrent openers); refusing to downgrade a database " + "this process does not exclusively own" + ) + actual = _set_journal_mode_no_wait(conn, "DELETE") + if actual != "delete": + raise sqlite3.OperationalError( + f"could not set configured journal_mode=delete (got {actual or 'no result'})" + ) + return actual + + # Decide BEFORE the flip whether it would overwrite a mode somebody chose: + # the probe and page_count are only readable while the file is untouched. + # A 0-page DB has no prior choice, and every caller reaches this before + # creating schema, so brand-new databases stay quiet. + _upgrading_existing_db = ( + current_mode is not None + and current_mode != "wal" + and _database_has_content(conn) + ) + + try: + # ``PRAGMA journal_mode=WAL`` RETURNS the resulting mode. Filesystems + # that refuse by *raising* SQLITE_PROTOCOL hit the except branch, but + # macOS NFS, SMB/CIFS and the AgentFS NFS overlay refuse WITHOUT raising + # and just return the still-effective mode. Trust the row, not the + # absence of an exception, or we report a false "wal", skip the + # fallback ERROR, and leave the DB silently in DELETE. + row = conn.execute("PRAGMA journal_mode=WAL").fetchone() + mode = str(row[0]).strip().lower() if row and row[0] is not None else "" + if mode == "wal": + if _upgrading_existing_db: + _log_journal_mode_upgrade_once(db_label, current_mode) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + # Silent refusal: WAL was not honored, but nothing raised. + silent_exc = WalUnsupportedError( + f"journal_mode=WAL refused without raising (still {mode!r})" + ) + if require_wal: + raise silent_exc + _log_wal_fallback_once(db_label, silent_exc) + return mode or "delete" + except sqlite3.OperationalError as exc: + # The require_wal silent-refusal raise above lands here (subclass of + # OperationalError) — propagate unchanged, skip the marker logic. + if isinstance(exc, WalUnsupportedError): + raise + msg = str(exc).lower() + if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS): + # Unrelated OperationalError — don't silently swallow. + raise + # ``disk i/o error`` is ambiguous: deterministic WAL-incompatibility on + # ZFS / APFS-CoW (SHM corruption under connection bursts), or a one-shot + # transient EIO (page-cache pressure, brief lock contention). Treating + # a transient EIO as a permanent downgrade signal produced mixed-mode + # corruption (process A downgrades to DELETE while siblings set WAL), + # so retry the pragma: transient EIO clears and we return "wal"; + # deterministic cases keep failing into the guarded DELETE fallback. + if "disk i/o error" in msg: + for _ in range(2): + time.sleep(0.05) + try: + row = conn.execute("PRAGMA journal_mode=WAL").fetchone() + except sqlite3.OperationalError as retry_exc: + if "disk i/o error" not in str(retry_exc).lower(): + raise + exc = retry_exc + continue + mode = ( + str(row[0]).strip().lower() + if row and row[0] is not None + else "" + ) + if mode == "wal": + # Transient EIO cleared and the switch went through; same + # header rewrite, so same upgrade signal. + if _upgrading_existing_db: + _log_journal_mode_upgrade_once(db_label, current_mode) + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + break + # Don't downgrade if another process already set WAL on disk, or if the + # mode cannot be read (probe blocked by a concurrent opener's locks) — + # ownership is not provably exclusive either way. + existing = _on_disk_journal_mode(conn) + if existing == "wal" or existing is None: + raise + if require_wal: + raise WalUnsupportedError(str(exc)) from exc + _log_wal_fallback_once(db_label, exc) + _set_journal_mode_no_wait(conn, "DELETE") + return "delete" + + +def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str: + """Execute ``PRAGMA journal_mode=`` without waiting on other openers. + + The ONLY place a journal-mode switch may be issued for a non-WAL target. + Forces ``busy_timeout=0`` so SQLite's exclusivity requirement becomes a + concurrent-opener detector: leaving WAL needs exclusive access, so if ANY + other connection (this process or another) holds the DB the pragma fails + immediately with ``database is locked`` instead of waiting out a busy + timeout and sneaking the flip between a concurrent writer's transactions — + exactly how committed-but-uncheckpointed WAL transactions get destroyed. + + Callers must treat a raised ``OperationalError`` as "not exclusively + owned: leave the journal mode alone", never as retryable. Returns SQLite's + reported mode (lowercase), or ``""`` if no row. + """ + previous_timeout = 0 + try: + row = conn.execute("PRAGMA busy_timeout").fetchone() + if row and row[0] is not None: + previous_timeout = int(row[0]) + except (sqlite3.OperationalError, TypeError, ValueError): + previous_timeout = 0 + conn.execute("PRAGMA busy_timeout=0") + try: + row = conn.execute(f"PRAGMA journal_mode={mode}").fetchone() + return str(row[0]).strip().lower() if row and row[0] is not None else "" + finally: + try: + conn.execute(f"PRAGMA busy_timeout={previous_timeout}") + except sqlite3.OperationalError: + pass + + +def _apply_delete_for_wal_reset_bug( + conn: sqlite3.Connection, + *, + db_label: str, + require_delete: bool = False, +) -> str: + """Avoid enabling WAL when the linked SQLite has the WAL-reset bug. + + - Already-WAL on disk: leave WAL alone (no live downgrade) and warn. + - Mode unreadable (probe blocked by a concurrent opener's locks): not + provably exclusive — leave the mode alone and warn. Never treat "could + not read the mode" as "not WAL": that confusion once flipped a live WAL + state.db to DELETE under a concurrent writer, destroying its + committed-but-uncheckpointed transactions. + - Otherwise: set DELETE (refusing to wait out concurrent openers) and warn. + - For an explicit operator request, verify SQLite accepted DELETE. + """ + current = _on_disk_journal_mode(conn) + + if current == "wal": + _log_wal_reset_bug_once(db_label, kept_wal=True) + if require_delete: + # Upgrading SQLite (the warning above) doesn't help on a + # WAL-incompatible filesystem; emit the actionable message last. + _log_configured_delete_overridden_once(db_label) + # No TRUNCATE / journal_mode=DELETE while other processes may still + # hold this WAL DB open; same safety rule as the NFS path. + _apply_wal_size_limit(conn) + _apply_macos_checkpoint_barrier(conn) + _enforce_macos_synchronous_full(conn) + return "wal" + + if current is None: + # Probe failed — likely another opener's locks, and the DB may be in + # WAL under a live writer. Never flip a mode we cannot even read. + if require_delete: + raise sqlite3.OperationalError( + "could not verify journal mode before applying configured " + "journal_mode=delete (database is locked — possible " + "concurrent openers); refusing to downgrade a database " + "this process does not exclusively own" + ) + _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) + return "wal" + + actual = "" + try: + actual = _set_journal_mode_no_wait(conn, "DELETE") + except sqlite3.OperationalError as exc: + if require_delete: + raise + lowered = str(exc).lower() + if "locked" in lowered or "busy" in lowered: + # A concurrent opener appeared between probe and flip (or already + # held the DB): SQLite refused the exclusive lock. Leave the mode as is. + _log_wal_reset_bug_once(db_label, kept_wal=True, indeterminate=True) + return current or "delete" + # Best-effort for the automatic fallback: DELETE is normally already + # the default for new file-backed databases. + if require_delete and actual != "delete": + raise sqlite3.OperationalError( + "could not set configured journal_mode=delete " + f"(got {actual or 'no result'})" + ) + _log_wal_reset_bug_once(db_label, kept_wal=False) + return "delete" + + +def _wal_reset_repair_hint() -> str: + """Repair hint matching what ``hermes update`` can actually do for this + install type (uv-managed venv vs git/pip/docker/nix).""" + try: + from hermes_cli.config import ( + detect_install_method, + recommended_update_command_for_method, + get_project_root, + ) + method = detect_install_method(get_project_root()) + cmd = recommended_update_command_for_method(method) + if method in {"git", "unknown"}: + return f"Hermes-managed installs can repair the embedded runtime with `{cmd}`" + if method == "docker": + return f"update the container image with `{cmd}`" + # nix/nixos + return cmd + except Exception: + pass + return ( + "install a Python build bundled with SQLite 3.51.3+ " + "(or backports 3.50.7 / 3.44.6) and restart Hermes" + ) + + +# Dedup state for _log_journal_mode_upgrade_once. +_journal_upgrade_warned_paths: set = set() + + +_journal_upgrade_warned_lock = threading.Lock() + + +def _log_wal_reset_bug_once( + db_label: str, + *, + kept_wal: bool, + indeterminate: bool = False, +) -> None: + """Log once per (process, db_label) about the WAL-reset vulnerability path.""" + from hermes_state import _wal_reset_bug_warned_paths + with _wal_reset_bug_warned_lock: + if db_label in _wal_reset_bug_warned_paths: + return + _wal_reset_bug_warned_paths.add(db_label) + if indeterminate: + action = ( + "journal mode could not be verified or exclusively switched " + "(database is locked — possible concurrent openers); leaving the " + "journal mode untouched (no live downgrade under concurrent " + "openers)" + ) + elif kept_wal: + action = ( + "is already in WAL mode — leaving WAL in place (no live " + "downgrade under concurrent openers)" + ) + else: + action = "using journal_mode=DELETE instead of enabling WAL" + # Install-type-aware so the warning never promises a repair path that + # doesn't exist for git/pip/system Python installs. + repair_hint = _wal_reset_repair_hint() + logger.warning( + "%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset " + "corruption bug (https://sqlite.org/wal.html#walresetbug) — %s. " + "Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); " + "%s. See `hermes doctor`. This warning fires once per " + "process per database.", + db_label, + sqlite3.sqlite_version, + sys.executable, + action, + repair_hint, + ) + + +def _log_journal_mode_upgrade_once(db_label: str, previous_mode: str) -> None: + """Log a single WARNING per (process, db_label) about a non-WAL -> WAL flip. + + ``PRAGMA journal_mode`` is a property of the FILE: switching an existing DB + to WAL rewrites its header and outlives the process. Operators do set + DELETE on the file directly (the documented WAL-reset-bug mitigation), and + nothing told them the next open would silently put WAL back. + + WARNING, not ERROR: the reverse move is ERROR in ``_log_wal_fallback_once`` + because dropping to DELETE loses concurrency, whereas this direction is + normally desirable (managed_uv repairs DELETE-stuck DBs on update). The + only problem was invisibility, so this names the durable setting without + claiming a degradation. Deduped per process per ``db_label`` because + kanban opens a fresh connection per operation. + """ + from hermes_state import _journal_upgrade_warned_paths + with _journal_upgrade_warned_lock: + if db_label in _journal_upgrade_warned_paths: + return + _journal_upgrade_warned_paths.add(db_label) + logger.warning( + "%s: on-disk journal_mode was %s and has been switched to WAL. This " + "rewrites the database header and persists after this process exits. " + "If %s was a deliberate choice (for example the mitigation for the " + "SQLite WAL-reset bug, or a WAL-unsafe filesystem), setting it with " + "PRAGMA on the file will not survive -- every open re-applies the " + "configured mode. Set `database.journal_mode: delete` in config.yaml " + "to make it stick. This message fires once per process per database.", + db_label, + previous_mode, + previous_mode, + ) + + +def _log_wal_fallback_once(db_label: str, exc: Exception) -> None: + """Log a single ERROR per (process, db_label) about WAL fallback. + + ERROR, not WARNING: silently dropping to DELETE is a real concurrency loss + (under kanban dispatcher + workers a write blocks readers as SQLITE_BUSY). + Deduped because kanban opens a fresh connection per operation. + """ + from hermes_state import _wal_fallback_warned_paths + with _wal_fallback_warned_lock: + if db_label in _wal_fallback_warned_paths: + return + _wal_fallback_warned_paths.add(db_label) + logger.error( + "%s: WAL journal_mode unsupported on this filesystem (%s) — " + "falling back to journal_mode=DELETE (slower rollback-journal " + "mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See " + "https://www.sqlite.org/wal.html for details. This message " + "fires once per process per database.", + db_label, + exc, + ) + + +def _log_configured_delete_overridden_once(db_label: str) -> None: + """Log a single ERROR per (process, db_label) when the operator configured + ``journal_mode=delete`` but the on-disk DB is already WAL. + + Never-live-downgrade keeps WAL (a live downgrade causes mixed-mode + corruption); without this the operator would never learn that + ``database.journal_mode: delete`` had no effect and that a one-time + offline ``PRAGMA journal_mode=DELETE`` (no open connections) is required. + """ + from hermes_state import _delete_overridden_warned_paths + with _delete_overridden_warned_lock: + if db_label in _delete_overridden_warned_paths: + return + _delete_overridden_warned_paths.add(db_label) + logger.error( + "%s: database.journal_mode=delete is configured but the on-disk " + "database is already WAL; keeping WAL (a live downgrade under open " + "connections can corrupt the DB). To apply journal_mode=DELETE, stop " + "all connections to this DB and run a one-time offline " + "'PRAGMA journal_mode=DELETE' on the file. This message fires once " + "per process per database.", + db_label, + ) + + +# --------------------------------------------------------------------------- +# Config-driven database pragmas +# --------------------------------------------------------------------------- +# Operators write synchronous as a name; mapped here rather than passed through +# so a typo becomes a warning instead of a silently different durability level. +_SYNCHRONOUS_LEVELS: Dict[str, int] = { + "OFF": 0, + "NORMAL": 1, + "FULL": 2, + "EXTRA": 3, +} + + +_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()} + + +_SYNCHRONOUS_FULL = 2 + + +def resolve_synchronous_level(raw_value: Any) -> Optional[int]: + """Map a configured ``database.synchronous`` value to its PRAGMA integer. + + Accepts SQLite's names (``OFF``/``NORMAL``/``FULL``/``EXTRA``, any case) or + ``0``-``3``. Anything else returns None so the caller warns and leaves the + level untouched — guessing at a malformed durability setting is worse. + """ + if isinstance(raw_value, bool): + # bool is an int subclass and YAML turns bare `on`/`off` into one. + # "off" is a real durability choice; True is meaningless. + return 0 if raw_value is False else None + if isinstance(raw_value, int): + return raw_value if raw_value in _SYNCHRONOUS_NAMES else None + text = str(raw_value).strip() + if not text: + return None + upper = text.upper() + if upper in _SYNCHRONOUS_LEVELS: + return _SYNCHRONOUS_LEVELS[upper] + try: + value = int(text) + except (TypeError, ValueError): + return None + return value if value in _SYNCHRONOUS_NAMES else None + + +def _apply_synchronous_pragma( + conn: sqlite3.Connection, + raw_value: Any, + *, + db_label: str, +) -> None: + """Set ``PRAGMA synchronous`` from config, never below FULL on macOS. + + Kept out of the integer loop in :func:`apply_database_pragmas`: this PRAGMA + decides whether a commit is on the platter, so an unrecognised value must + not fall through to "SQLite default" the way a bad ``cache_size`` can. + + Darwin floor: :func:`_enforce_macos_synchronous_full` runs during + ``apply_wal_with_fallback()`` and this runs after it, so a configured + ``NORMAL`` would otherwise silently undo the macOS btree protection. + Raising the level on macOS is allowed; lowering it is refused out loud. + """ + level = resolve_synchronous_level(raw_value) + if level is None: + logger.warning( + "%s: ignoring unrecognized database.synchronous=%r " + "(expected OFF, NORMAL, FULL, EXTRA, or 0-3)", + db_label, + raw_value, + ) + return + if sys.platform == "darwin" and level < _SYNCHRONOUS_FULL: + logger.warning( + "%s: refusing database.synchronous=%s on macOS; keeping FULL. " + "Darwin's fsync() does not guarantee write ordering, so a lower " + "level readmits the half-written btree pages FULL exists to " + "prevent.", + db_label, + _SYNCHRONOUS_NAMES[level], + ) + return + try: + conn.execute(f"PRAGMA synchronous={level}") + except sqlite3.OperationalError: + pass + + +def apply_database_pragmas( + conn: sqlite3.Connection, + *, + db_label: str = "state.db", +) -> None: + """Apply optional performance and WAL-sizing PRAGMAs from ``config.yaml``. + + Journal mode is NOT handled here — ``database.journal_mode`` is owned by + :func:`resolve_journal_mode` inside :func:`apply_wal_with_fallback`, under + all the safety guards. + + Keys under ``database:``: ``cache_size`` (negative = KiB, positive = + pages), ``mmap_size`` (bytes, 0 = disabled), ``temp_store`` (0-3), + ``wal_autocheckpoint`` (pages), ``journal_size_limit`` (bytes), and + ``synchronous`` (``OFF``/``NORMAL``/``FULL``/``EXTRA`` or ``0``-``3``). + Unset ``synchronous`` leaves SQLite's default, a *compile-time* constant + (``SQLITE_DEFAULT_WAL_SYNCHRONOUS``) that differs between bundled, distro + and Homebrew builds; setting it explicitly is the only way to know. + + Best-effort: config load or pragma failures are ignored so DB init never + breaks on a malformed ``database:`` section. + """ + try: + # Local import avoids a circular import with hermes_cli.config. + from hermes_cli.config import cfg_get, load_config_readonly + + cfg = load_config_readonly() + except Exception: + return + + # Applied to ALL connection types: writer, read_only, WAL per-thread readers. + for pragma_name in ( + "cache_size", + "mmap_size", + "temp_store", + "wal_autocheckpoint", + "journal_size_limit", + ): + raw_value = cfg_get(cfg, "database", pragma_name, default=None) + if raw_value is None: + continue + try: + value = int(str(raw_value).strip()) + except (TypeError, ValueError): + logger.warning( + "%s: ignoring non-integer database.%s=%r", + db_label, + pragma_name, + raw_value, + ) + continue + try: + conn.execute(f"PRAGMA {pragma_name}={value}") + except sqlite3.OperationalError: + pass + + # Last: the sizing pragmas above cannot change durability, and the macOS + # enforcement ran earlier during WAL activation (see _apply_synchronous_pragma + # for why that ordering needs an explicit floor rather than an override). + raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None) + if raw_synchronous is not None: + _apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label) diff --git a/tests/test_state_db_write_durability.py b/tests/test_state_db_write_durability.py index 4ba56e49f5..04ab8c88f1 100644 --- a/tests/test_state_db_write_durability.py +++ b/tests/test_state_db_write_durability.py @@ -102,8 +102,12 @@ def test_repair_path_has_no_bare_connects() -> None: Source-level guard: the bare form is exactly what regressed, and a unit test on the helper alone would not notice a sixth site being added. """ - source = Path(hermes_state.__file__).read_text(encoding="utf-8") - tree = ast.parse(source, filename=str(hermes_state.__file__)) + # The repair/probe helpers live in hermes_state_repair; hermes_state only + # re-imports them. + import hermes_state_repair + + source = Path(hermes_state_repair.__file__).read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(hermes_state_repair.__file__)) def is_db_path_connect(node: ast.AST) -> bool: if not isinstance(node, ast.Call):