From acfda3759871e50a6899151069a564458604a721 Mon Sep 17 00:00:00 2001 From: Michael Steuer Date: Fri, 4 Sep 2026 06:48:45 +0200 Subject: [PATCH] fix(sessions): chunk IN-list ids in bulk delete to avoid 'too many SQL variables' `hermes sessions prune --source cron --older-than 14` on a store with ~60K cron sessions (~50K matches) died with sqlite3.OperationalError: too many SQL variables. SessionDB.delete_sessions, _collect_delegate_child_ids and _delete_delegate_children each bound the full id list into a single IN (?,?,...). SQLite caps bound parameters at SQLITE_MAX_VARIABLE_NUMBER (999 on < 3.32, 32766 after), so any bulk delete above that failed outright. Chunk every IN list (`_id_chunks` / `_SQL_IN_CHUNK` in hermes_state_common, 900 ids; the delegate walk binds each id twice so it chunks at half). Same transaction, same cascade/orphan contract; only the parameter binding is split. Hand-ported from PR #102679 (targeted the pre-decomposition hermes_state.py god file; the functions now live in hermes_state_sessions.py). Authored by @mssteuer; ported under --author. --- hermes_state_common.py | 13 +++++++++++ hermes_state_sessions.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/hermes_state_common.py b/hermes_state_common.py index 103920c8ee..a5a824ed08 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -279,6 +279,19 @@ def _placeholders(items) -> str: return ",".join("?" for _ in range(items if isinstance(items, int) else len(items))) +# Ids per ``IN (?,...)`` list: SQLite caps bound parameters at SQLITE_MAX_VARIABLE_NUMBER (999 on builds +# < 3.32, 32766 after); a bulk prune of a cron-heavy store bound tens of thousands of ids into one list and +# died with "too many SQL variables". Every IN-list over session ids goes through ``_id_chunks``. +_SQL_IN_CHUNK = 900 + + +def _id_chunks(ids, size: int = _SQL_IN_CHUNK): + """Yield *ids* (any iterable) as lists of at most *size* elements.""" + ids = list(ids) + for start in range(0, len(ids), size): + yield ids[start:start + size] + + _FTS_TRIGGERS = ("messages_fts_insert", "messages_fts_delete", "messages_fts_update", "messages_fts_trigram_insert", "messages_fts_trigram_delete", "messages_fts_trigram_update") diff --git a/hermes_state_sessions.py b/hermes_state_sessions.py index 1e33f8d5a5..5f9e3f85b8 100644 --- a/hermes_state_sessions.py +++ b/hermes_state_sessions.py @@ -17,7 +17,7 @@ from hermes_state_common import ( _LISTABLE_CHILD_SQL, _PREVIEW_ELIGIBLE_SQL, _PREVIEW_RAW_SELECT, _RECOVERABLE_END_REASONS, _RECOVERABLE_END_REASONS_SQL, _RESET_END_REASONS, _legacy_reset_child_sql, _shape_preview, _sql_json_extract, _sql_session_last_active, _sql_session_last_active_by_id, escape_like as _escape_like, - _placeholders as _session_ids_placeholders, + _SQL_IN_CHUNK, _id_chunks, _placeholders as _session_ids_placeholders, ) # caplog tests pin the "hermes_state" logger name. @@ -141,24 +141,29 @@ def _collect_delegate_child_ids(conn, parent_ids: List[str]) -> List[str]: found: set[str] = set(seeds) frontier = list(seeds) while frontier: - ph = _session_ids_placeholders(frontier) - cursor = conn.execute( - f"SELECT id FROM sessions WHERE {df} IN ({ph}) " - f"OR (parent_session_id IN ({ph}) AND {df} IS NOT NULL)", frontier + frontier, - ) - frontier = [row["id"] for row in cursor.fetchall() if row["id"] not in found] - found.update(frontier) + next_frontier: List[str] = [] + for chunk in _id_chunks(frontier, _SQL_IN_CHUNK // 2): # each id is bound twice below + ph = _session_ids_placeholders(chunk) + cursor = conn.execute( + f"SELECT id FROM sessions WHERE {df} IN ({ph}) " + f"OR (parent_session_id IN ({ph}) AND {df} IS NOT NULL)", chunk + chunk, + ) + for row in cursor.fetchall(): + if row["id"] not in found: + found.add(row["id"]) + next_frontier.append(row["id"]) + frontier = next_frontier return [sid for sid in found if sid not in seeds] def _delete_delegate_children(conn, parent_ids: List[str]) -> List[str]: ids = _collect_delegate_child_ids(conn, parent_ids) - if ids: - ph = _session_ids_placeholders(ids) - conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", ids) + for chunk in _id_chunks(ids): + ph = _session_ids_placeholders(chunk) + conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", chunk) # FK safety: orphan any untagged stragglers pointing at a doomed row. - conn.execute(f"UPDATE sessions SET parent_session_id = NULL WHERE parent_session_id IN ({ph})", ids) - conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", ids) + conn.execute(f"UPDATE sessions SET parent_session_id = NULL WHERE parent_session_id IN ({ph})", chunk) + conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", chunk) return ids @@ -1524,19 +1529,19 @@ class SessionSessionsMixin: return 0 removed_ids: list[str] = [] def _do(conn): - existing = [row["id"] for row in conn.execute( - f"SELECT id FROM sessions WHERE id IN ({_session_ids_placeholders(unique_ids)})", - unique_ids, + existing = [row["id"] for chunk in _id_chunks(unique_ids) for row in conn.execute( + f"SELECT id FROM sessions WHERE id IN ({_session_ids_placeholders(chunk)})", chunk, ).fetchall()] if not existing: return 0 - ph = _session_ids_placeholders(existing) removed_ids.extend(_delete_delegate_children(conn, existing)) - conn.execute( # orphan children whose parent is in the kill list (FK) - f"UPDATE sessions SET parent_session_id = NULL WHERE parent_session_id IN ({ph})", existing, - ) - conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", existing) - conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", existing) + for chunk in _id_chunks(existing): + ph = _session_ids_placeholders(chunk) + conn.execute( # orphan children whose parent is in the kill list (FK) + f"UPDATE sessions SET parent_session_id = NULL WHERE parent_session_id IN ({ph})", chunk, + ) + conn.execute(f"DELETE FROM messages WHERE session_id IN ({ph})", chunk) + conn.execute(f"DELETE FROM sessions WHERE id IN ({ph})", chunk) self._delete_unreferenced_system_prompts(conn) removed_ids.extend(existing) return len(existing)