22a1b20ed4
The page-level salvage lane infers a salvaged store's physical column order from SCHEMA_HISTORY; a column added to SCHEMA_SQL without an event here makes the newest rows of every upgraded store map to nothing and take the positional fallback.
260 lines
12 KiB
Python
260 lines
12 KiB
Python
"""Physical column history of the state.db tables the page-level salvage lane maps.
|
|
|
|
``_reconcile_columns()`` (hermes_state_schema.py) upgrades a live store with
|
|
``ALTER TABLE ADD COLUMN``, which APPENDS each newly declared column. A store
|
|
therefore keeps its columns in the order they were *added*, which diverges
|
|
from the order ``SCHEMA_SQL`` declares them whenever a later release inserts a
|
|
column mid-definition (#101409). SQLite never rewrites existing rows on ADD
|
|
COLUMN either, so a row written when the table had ``k`` columns carries
|
|
exactly the first ``k`` physical names.
|
|
|
|
A ``lost_and_found`` record carries no schema, so the salvage lane has to
|
|
recognise the physical layout that produced it. This module is the single
|
|
source of truth for that: the declared order of every table as first shipped
|
|
(``base``) and, per release that changed it, the edits made to the
|
|
declaration (``events``). Replaying the events reproduces every historical
|
|
declared order, and every physical layout a real store can have is some
|
|
prefix chain over those snapshots (created at snapshot ``i``, then each later
|
|
upgrade appending the columns it had not seen yet).
|
|
|
|
``tests/hermes_cli/test_session_schema_history.py`` asserts the replay ends
|
|
at the current ``SCHEMA_SQL``. When that test fails you added or moved a
|
|
column: append an event at the END, labelled with the next sequence number,
|
|
describing the edit (``("+", column, after)`` / ``("-", column)``) — do NOT
|
|
rewrite or reorder older events, real stores were shaped by them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Callable, Iterator, Optional, Sequence
|
|
|
|
|
|
Edit = tuple # ("+", column, previous_column_or_None) | ("-", column)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _TableHistory:
|
|
base: tuple[str, ...]
|
|
# (label, edits) in replay order. Labels are "<seq> <commit time UTC>
|
|
# <sha>" of the main commit that shipped the edit; the sequence number
|
|
# is what the ordering test checks, so an event appended out of order
|
|
# (or inserted mid-list "to keep dates sorted") fails loudly.
|
|
events: tuple[tuple[str, tuple[Edit, ...]], ...]
|
|
|
|
|
|
def _apply(columns: list[str], edits: Sequence[Edit]) -> None:
|
|
for edit in edits:
|
|
if edit[0] == "-":
|
|
columns.remove(edit[1])
|
|
continue
|
|
_, column, after = edit
|
|
columns.insert(0 if after is None else columns.index(after) + 1, column)
|
|
|
|
|
|
def declared_snapshots(table: str) -> list[tuple[str, ...]]:
|
|
"""Every declared column order the table has shipped with, oldest first."""
|
|
|
|
history = SCHEMA_HISTORY[table]
|
|
columns = list(history.base)
|
|
snapshots = [tuple(columns)]
|
|
for _label, edits in history.events:
|
|
_apply(columns, edits)
|
|
snapshots.append(tuple(columns))
|
|
return snapshots
|
|
|
|
|
|
def current_declared_columns(table: str) -> tuple[str, ...]:
|
|
return declared_snapshots(table)[-1]
|
|
|
|
|
|
def reachable_physical_layouts(
|
|
table: str,
|
|
accept: Optional[Callable[[tuple[str, ...], int], bool]] = None,
|
|
) -> Iterator[tuple[str, ...]]:
|
|
"""Yield every physical layout an upgraded store of ``table`` can have.
|
|
|
|
A store created at snapshot ``i`` and later opened by releases
|
|
``j1 < j2 < ...`` appends, at each step, the snapshot's columns it does
|
|
not have yet. Columns declared and later removed stay physically present
|
|
(``_reconcile_columns`` never drops), so they appear in layouts too.
|
|
|
|
``accept(layout, first_new_index)`` lets the caller prune: it is called
|
|
for every new layout with the index where it starts to differ from its
|
|
parent, and a False return stops extending that branch. The graph is
|
|
large without pruning (~230k sessions layouts) but salvage always
|
|
prunes against real cells, which keeps it in the low thousands.
|
|
"""
|
|
|
|
snapshots = declared_snapshots(table)
|
|
seen: dict[tuple[str, ...], int] = {}
|
|
frontier: list[tuple[tuple[str, ...], int]] = []
|
|
for index, snapshot in enumerate(snapshots):
|
|
if snapshot in seen:
|
|
continue
|
|
if accept is not None and not accept(snapshot, 0):
|
|
continue
|
|
seen[snapshot] = index
|
|
frontier.append((snapshot, index))
|
|
yield snapshot
|
|
while frontier:
|
|
next_frontier: list[tuple[tuple[str, ...], int]] = []
|
|
for layout, index in frontier:
|
|
have = set(layout)
|
|
for later in range(index + 1, len(snapshots)):
|
|
added = tuple(c for c in snapshots[later] if c not in have)
|
|
if not added:
|
|
continue
|
|
candidate = layout + added
|
|
if candidate in seen and seen[candidate] <= later:
|
|
continue
|
|
if accept is not None and not accept(candidate, len(layout)):
|
|
continue
|
|
seen[candidate] = later
|
|
next_frontier.append((candidate, later))
|
|
yield candidate
|
|
frontier = next_frontier
|
|
|
|
|
|
SCHEMA_HISTORY: dict[str, _TableHistory] = {
|
|
"sessions": _TableHistory(
|
|
base=(
|
|
'id', 'source', 'user_id', 'model', 'model_config',
|
|
'system_prompt', 'parent_session_id', 'started_at', 'ended_at',
|
|
'end_reason', 'message_count', 'tool_call_count', 'input_tokens',
|
|
'output_tokens',
|
|
),
|
|
events=(
|
|
('01 2026-03-08T23:32Z c5e8166c8b', (('+', 'title', 'output_tokens'),)),
|
|
('02 2026-03-17T10:44Z d417ba2a48', (
|
|
('+', 'cache_read_tokens', 'output_tokens'),
|
|
('+', 'cache_write_tokens', 'cache_read_tokens'),
|
|
('+', 'reasoning_tokens', 'cache_write_tokens'),
|
|
('+', 'billing_provider', 'reasoning_tokens'),
|
|
('+', 'billing_base_url', 'billing_provider'),
|
|
('+', 'billing_mode', 'billing_base_url'),
|
|
('+', 'estimated_cost_usd', 'billing_mode'),
|
|
('+', 'actual_cost_usd', 'estimated_cost_usd'),
|
|
('+', 'cost_status', 'actual_cost_usd'),
|
|
('+', 'cost_source', 'cost_status'),
|
|
('+', 'pricing_version', 'cost_source'),
|
|
)),
|
|
('03 2026-04-22T12:51Z 5fb143169b', (('+', 'api_call_count', 'title'),)),
|
|
('04 2026-05-10T20:06Z 878611a79d', (
|
|
('+', 'handoff_pending', 'api_call_count'),
|
|
('+', 'handoff_platform', 'handoff_pending'),
|
|
)),
|
|
('05 2026-05-10T20:06Z 00ce5f04d9', (
|
|
('-', 'handoff_pending'),
|
|
('+', 'handoff_state', 'api_call_count'),
|
|
('+', 'handoff_error', 'handoff_platform'),
|
|
)),
|
|
('06 2026-05-31T22:46Z 51c68d4ab1', (('+', 'cwd', 'reasoning_tokens'),)),
|
|
('07 2026-06-01T08:22Z 3e59be0c41', (('+', 'rewind_count', 'handoff_error'),)),
|
|
('08 2026-06-02T01:41Z 85b65e29f0', (('+', 'archived', 'rewind_count'),)),
|
|
('09 2026-06-25T22:49Z ffa3d3c811', (
|
|
('+', 'git_branch', 'cwd'),
|
|
('+', 'git_repo_root', 'git_branch'),
|
|
)),
|
|
('10 2026-06-28T22:10Z 86e64900b9', (
|
|
('+', 'session_key', 'user_id'),
|
|
('+', 'chat_id', 'session_key'),
|
|
('+', 'chat_type', 'chat_id'),
|
|
('+', 'thread_id', 'chat_type'),
|
|
)),
|
|
('11 2026-06-30T08:06Z f2ccb2859f', (
|
|
('+', 'compression_failure_cooldown_until', 'handoff_error'),
|
|
('+', 'compression_failure_error', 'compression_failure_cooldown_until'),
|
|
)),
|
|
('12 2026-07-05T21:01Z 747386ecfa', (
|
|
('+', 'display_name', 'thread_id'),
|
|
('+', 'origin_json', 'display_name'),
|
|
('+', 'expiry_finalized', 'origin_json'),
|
|
)),
|
|
('13 2026-07-13T20:49Z af7dceaf77', (
|
|
('+', 'compression_fallback_streak', 'compression_failure_error'),
|
|
)),
|
|
('14 2026-07-15T16:50Z e8b7ce8c19', (('+', 'profile_name', 'compression_fallback_streak'),)),
|
|
('15 2026-07-23T15:08Z ec5835ab8b', (
|
|
('+', 'compression_ineffective_count', 'compression_fallback_streak'),
|
|
)),
|
|
('16 2026-07-24T16:15Z 951d606730', (('+', 'pinned', 'archived'),)),
|
|
('17 2026-07-27T19:14Z cfb206fe2e', (
|
|
('+', 'last_activity_at', 'title'),
|
|
('+', 'last_activity_description', 'last_activity_at'),
|
|
('+', 'last_activity_provenance', 'last_activity_description'),
|
|
)),
|
|
('18 2026-07-27T19:22Z 5646fed97e', (
|
|
('-', 'last_activity_at'),
|
|
('-', 'last_activity_description'),
|
|
('-', 'last_activity_provenance'),
|
|
)),
|
|
('19 2026-08-02T23:16Z c2088efe9e', (
|
|
('+', 'last_activity_at', 'title'),
|
|
('+', 'last_activity_description', 'last_activity_at'),
|
|
('+', 'last_activity_provenance', 'last_activity_description'),
|
|
)),
|
|
('20 2026-08-03T15:07Z 7d066c3c56', (('+', 'system_prompt_hash', 'system_prompt'),)),
|
|
('21 2026-08-04T19:20Z d98287fe3c', (('+', 'last_read_at', 'pinned'),)),
|
|
('22 2026-08-08T22:17Z fe42097865', (('+', 'title_source', 'title'),)),
|
|
('23 2026-08-15T07:31Z fbaea9bddc', (('+', 'hidden', 'pinned'),)),
|
|
('24 2026-08-15T07:33Z e89532d97e', (('+', 'git_metadata_generation', 'git_repo_root'),)),
|
|
('25 2026-09-02T11:14Z 238b6c1ab9', (
|
|
('+', 'compression_recovery_deadline', 'compression_ineffective_count'),
|
|
)),
|
|
('26 2026-09-02T14:22Z 8e4366d358', (('+', 'tool_names', 'last_read_at'),)),
|
|
('27 2026-09-19T00:10Z 922a0c3c87', (('+', 'transport_profile', 'profile_name'),)),
|
|
),
|
|
),
|
|
"messages": _TableHistory(
|
|
base=(
|
|
'id', 'session_id', 'role', 'content', 'tool_call_id',
|
|
'tool_calls', 'tool_name', 'timestamp', 'token_count',
|
|
),
|
|
events=(
|
|
('01 2026-02-21T08:05Z b33ed9176f', (('+', 'finish_reason', 'token_count'),)),
|
|
('02 2026-03-25T16:47Z 42fec19151', (
|
|
('+', 'reasoning', 'finish_reason'),
|
|
('+', 'reasoning_details', 'reasoning'),
|
|
('+', 'codex_reasoning_items', 'reasoning_details'),
|
|
)),
|
|
('03 2026-04-22T11:31Z a7d78d3bfd', (('+', 'reasoning_content', 'reasoning'),)),
|
|
('04 2026-04-26T01:22Z 81e01f6ee9', (('+', 'codex_message_items', 'codex_reasoning_items'),)),
|
|
('05 2026-05-20T20:00Z 31a0100104', (('+', 'platform_message_id', 'codex_message_items'),)),
|
|
('06 2026-05-23T08:33Z 4a91e36495', (('+', 'observed', 'platform_message_id'),)),
|
|
('07 2026-06-01T08:22Z 3e59be0c41', (('+', 'active', 'observed'),)),
|
|
('08 2026-06-11T03:45Z aaccaada28', (('+', 'anthropic_content_blocks', 'reasoning_details'),)),
|
|
('09 2026-06-11T03:45Z efcbbde48c', (('-', 'anthropic_content_blocks'),)),
|
|
('10 2026-06-20T17:57Z 854d75723f', (('+', 'compacted', 'active'),)),
|
|
('11 2026-07-11T12:41Z a0a6cd80f5', (('+', 'effect_disposition', 'tool_name'),)),
|
|
('12 2026-07-19T02:55Z 7b3dcee928', (('+', 'api_content', 'compacted'),)),
|
|
('13 2026-07-23T18:46Z a4bc1ca502', (
|
|
('+', 'display_kind', 'api_content'),
|
|
('+', 'display_metadata', 'display_kind'),
|
|
)),
|
|
('14 2026-08-25T10:55Z 1104ffe0b9', (('+', '_compressed_summary', 'observed'),)),
|
|
('15 2026-09-09T17:05Z 1c6683e8e0', (
|
|
('+', 'display_identity', 'display_metadata'),
|
|
('+', 'display_order', 'display_identity'),
|
|
)),
|
|
),
|
|
),
|
|
"session_model_usage": _TableHistory(
|
|
base=(
|
|
'session_id', 'model', 'billing_provider', 'billing_base_url',
|
|
'api_call_count', 'input_tokens', 'output_tokens',
|
|
'cache_read_tokens', 'cache_write_tokens', 'reasoning_tokens',
|
|
'estimated_cost_usd', 'first_seen', 'last_seen',
|
|
),
|
|
events=(
|
|
('01 2026-07-11T12:59Z 0d63c23f36', (
|
|
('+', 'billing_mode', 'billing_base_url'),
|
|
('+', 'actual_cost_usd', 'estimated_cost_usd'),
|
|
('+', 'cost_status', 'actual_cost_usd'),
|
|
('+', 'cost_source', 'cost_status'),
|
|
)),
|
|
('02 2026-07-16T11:23Z eb6aa03609', (('+', 'task', 'billing_mode'),)),
|
|
),
|
|
),
|
|
}
|