fix(state): one corrupt timestamp row no longer kills sessions list, export or insights
SQLite dynamic typing lets a TEXT cell ('not-a-timestamp'), inf/nan or a
garbage double (8.4e252 salvaged from a damaged page) sit in a REAL
timestamp column. Every reader called datetime.fromtimestamp()/float
arithmetic on the raw cell, so ONE bad row raised TypeError/OverflowError
out of the row loop and took down the whole `hermes sessions list`/browse
table (#102399), all three exporters — JSONL/MD, QMD, HTML (#102352) —
and `hermes insights` (#99959).
Fix the class with ONE helper, hermes_cli.timefmt.coerce_epoch(): a
stored cell becomes float epoch seconds inside a sane 1970..2103 window
or None after a WARNING that names the session id. Every reader routes
through it — relative_time (list/browse/resume picker), format_epoch
(prune/candidates tables), the three exporters' timestamp formatters,
insights' _get_sessions/_day/period range — so a bad row renders as
'?'/'N/A'/raw text for that one cell and the command completes.
Write side: hermes_state_messages._coerce_timestamp (append_message,
append_messages_batch, import) and the import path's started_at now use
the same window, so a new out-of-range timestamp falls back to now()
instead of being persisted — new bad rows cannot be written by Hermes.
Reported-by: #102399, #102352, #99959 reporters; kokhlo's insights
analysis pointed at every reporting site, not just line 860.
This commit is contained in:
@@ -10,6 +10,7 @@ from decimal import Decimal
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.usage_pricing import CanonicalUsage, estimate_usage_cost, format_cost_label, format_duration_compact, has_known_pricing
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
|
||||
_TOKEN_KEYS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens")
|
||||
_SKILL_TOOLS = {"skill_view", "skill_manage"}
|
||||
@@ -71,7 +72,7 @@ def _hour12(hr: int) -> str:
|
||||
|
||||
|
||||
def _day(ts: Any) -> str:
|
||||
return datetime.fromtimestamp(ts).strftime("%b %d") if ts else "?"
|
||||
return datetime.fromtimestamp(ts).strftime("%b %d") if ts and (ts := coerce_epoch(ts)) else "?"
|
||||
|
||||
|
||||
def _scoped(before: str, after: str = "", *, src: str = " AND s.source = ?") -> tuple[str, str]:
|
||||
@@ -206,7 +207,13 @@ class InsightsEngine:
|
||||
# ------------------------------------------------------------------ SQL
|
||||
|
||||
def _get_sessions(self, cutoff: float, source: str = None) -> List[Dict]:
|
||||
return [dict(row) for row in self._query("_GET_SESSIONS", cutoff, source)]
|
||||
# Coerce the two epoch columns once at load: one corrupt/TEXT cell must degrade to "unknown"
|
||||
# for that session, never abort the whole report (#99959).
|
||||
rows = [dict(row) for row in self._query("_GET_SESSIONS", cutoff, source)]
|
||||
for row in rows:
|
||||
for col in ("started_at", "ended_at"):
|
||||
row[col] = coerce_epoch(row.get(col), session_id=row.get("id"), field=col)
|
||||
return rows
|
||||
|
||||
def _get_tool_usage(self, cutoff: float, source: str = None) -> List[Dict]:
|
||||
"""Tool call counts from two sources: ``tool_name`` on 'tool' rows (set
|
||||
@@ -479,9 +486,9 @@ class InsightsEngine:
|
||||
" ╚══════════════════════════════════════════════════════════╝",
|
||||
"",
|
||||
]
|
||||
if o.get("date_range_start") and o.get("date_range_end"):
|
||||
start_str = datetime.fromtimestamp(o["date_range_start"]).strftime("%b %d, %Y")
|
||||
end_str = datetime.fromtimestamp(o["date_range_end"]).strftime("%b %d, %Y")
|
||||
if (start := coerce_epoch(o.get("date_range_start"))) is not None and (end := coerce_epoch(o.get("date_range_end"))) is not None:
|
||||
start_str = datetime.fromtimestamp(start).strftime("%b %d, %Y")
|
||||
end_str = datetime.fromtimestamp(end).strftime("%b %d, %Y")
|
||||
lines += [f" Period: {start_str} — {end_str}", ""]
|
||||
lines += self._section("📋 Overview") + [
|
||||
f" Sessions: {o['total_sessions']:<12} Messages: {o['total_messages']:,}",
|
||||
|
||||
@@ -364,7 +364,7 @@ class CLISessionMixin:
|
||||
for idx, session in enumerate(sessions, start=1):
|
||||
title = session.get("title") or "—"
|
||||
preview = (session.get("preview") or "")[:38]
|
||||
last_active = _relative_time(session.get("last_active"))
|
||||
last_active = _relative_time(session.get("last_active"), session_id=session.get("id"))
|
||||
_cli_visible_print(f" {idx:<3} {title:<32} {preview:<40} {last_active:<13} {session['id']}")
|
||||
_cli_visible_print()
|
||||
_cli_visible_print(" Use /resume <number>, /resume <session id>, or /resume <session title> to continue.")
|
||||
|
||||
@@ -12,6 +12,8 @@ from html import escape as html_escape
|
||||
import json
|
||||
from typing import Any, Dict, Iterable, Iterator, List, Literal, Optional, Tuple
|
||||
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
|
||||
|
||||
ExportFormat = Literal["jsonl", "markdown"]
|
||||
ExportOnly = Literal["user-prompts"]
|
||||
@@ -91,7 +93,7 @@ def iter_user_prompt_records(sessions: Iterable[Dict[str, Any]]) -> Iterator[Dic
|
||||
record: Dict[str, Any] = {
|
||||
"session_id": session_id,
|
||||
"index": index,
|
||||
"created_at": _format_timestamp(message.get("timestamp")),
|
||||
"created_at": _format_timestamp(message.get("timestamp"), session_id),
|
||||
"role": "user",
|
||||
"text": _message_text(message.get("content")),
|
||||
}
|
||||
@@ -123,7 +125,7 @@ def _append_session_messages(lines: List[str], session: Dict[str, Any], *, headi
|
||||
return
|
||||
for message in visible_messages:
|
||||
role = str(message.get("role") or "unknown")
|
||||
timestamp = _format_timestamp(message.get("timestamp"))
|
||||
timestamp = _format_timestamp(message.get("timestamp"), _session_id(session))
|
||||
suffix = f" - {timestamp}" if timestamp else ""
|
||||
text = _message_text(message.get("content"))
|
||||
if role == "tool":
|
||||
@@ -157,15 +159,15 @@ def _content_part_text(part: Any) -> str:
|
||||
return json.dumps(part, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def _format_timestamp(value: Any) -> Optional[str]:
|
||||
def _format_timestamp(value: Any, session_id: Optional[str] = None) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
dt = datetime.fromtimestamp(float(value), tz=timezone.utc)
|
||||
elif isinstance(value, datetime):
|
||||
if isinstance(value, datetime):
|
||||
dt = (value if value.tzinfo else value.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
|
||||
elif (ts := coerce_epoch(value, session_id=session_id)) is None:
|
||||
return str(value) # corrupt cell: odd-looking date, not an aborted export
|
||||
else:
|
||||
return str(value)
|
||||
dt = datetime.fromtimestamp(ts, tz=timezone.utc)
|
||||
return dt.isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
@@ -176,7 +178,7 @@ def _session_metadata_lines(session: Dict[str, Any]) -> List[str]:
|
||||
lines.append(f"- {label}: `{session[key]}`")
|
||||
if title := session.get("title"):
|
||||
lines.append(f"- Title: {' '.join(str(title).splitlines()).strip()}")
|
||||
if started := _format_timestamp(session.get("started_at")):
|
||||
if started := _format_timestamp(session.get("started_at"), _session_id(session)):
|
||||
lines.append(f"- Started: {started}")
|
||||
if (message_count := session.get("message_count")) is not None:
|
||||
lines.append(f"- Messages: {message_count}")
|
||||
|
||||
@@ -5,6 +5,8 @@ import secrets
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import quote
|
||||
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
|
||||
# --- Icons (Lucide-style SVGs) ---
|
||||
ICON_USER = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-user"><path d="M19 21v-2a4 4 0 0 0-4-4H9a4 4 0 0 0-4 4v2"/><circle cx="12" cy="7" r="4"/></svg>'
|
||||
ICON_BOT = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-bot"><path d="M12 8V4H8"/><rect width="16" height="12" x="4" y="8" rx="2"/><path d="M2 14h2"/><path d="M20 14h2"/><path d="M15 13v2"/><path d="M9 13v2"/></svg>'
|
||||
@@ -650,8 +652,9 @@ def _escape_html(text: Any) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _format_timestamp(ts: float) -> str:
|
||||
return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") if ts else "N/A"
|
||||
def _format_timestamp(ts: Any) -> str:
|
||||
# A corrupt cell renders as N/A; never raw text (a TEXT timestamp would otherwise reach an HTML sink).
|
||||
return datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") if ts and (ts := coerce_epoch(ts)) else "N/A"
|
||||
|
||||
|
||||
_ROLE_ICONS = {"user": ICON_USER, "assistant": ICON_BOT, "system": ICON_SHIELD}
|
||||
|
||||
@@ -14,6 +14,8 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
|
||||
EXPORTER_VERSION = "hermes sessions export (md/qmd) v1"
|
||||
_SHA_LINE_RE = re.compile(r"- SHA256 of exported body: `([0-9a-f]{64})`")
|
||||
_SHA_PLACEHOLDER = "__SHA256_PLACEHOLDER__"
|
||||
@@ -23,10 +25,8 @@ _VERIFICATION_HEADING = "## Export verification"
|
||||
def _iso_timestamp(value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
try:
|
||||
ts = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
if (ts := coerce_epoch(value)) is None:
|
||||
return str(value) # corrupt cell: odd-looking date, not an aborted export
|
||||
return datetime.fromtimestamp(ts, tz=timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
|
||||
_DURATION_RE = re.compile(
|
||||
r"^(\d+(?:\.\d+)?)\s*"
|
||||
r"(s|sec|secs|second|seconds|"
|
||||
@@ -51,8 +53,8 @@ def parse_point_in_time(value: str, flag: str) -> float:
|
||||
|
||||
|
||||
def format_epoch(ts: Optional[float]) -> str:
|
||||
"""Render an epoch timestamp as a short local-time string."""
|
||||
return "-" if ts is None else datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")
|
||||
"""Render an epoch timestamp as a short local-time string; ``-`` when unset or corrupt."""
|
||||
return "-" if (ts := coerce_epoch(ts)) is None else datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
|
||||
# (filter key, argparse attr, CLI flag, description template) for the four epoch bounds.
|
||||
|
||||
@@ -273,7 +273,7 @@ def _cmd_list(db, args):
|
||||
return ((os.path.basename(key.rstrip("/\\")) or key) if key else "—")[:16]
|
||||
_title = lambda s, n: (s.get("title") or "—")[:n] # noqa: E731
|
||||
_preview = lambda s, n: s.get("preview", "")[:n] # noqa: E731
|
||||
_ago = lambda s: _relative_time(s.get("last_active")) # noqa: E731
|
||||
_ago = lambda s: _relative_time(s.get("last_active"), session_id=s["id"]) # noqa: E731
|
||||
layouts = { # (has_ws, has_titles): header, rule width, row formatter
|
||||
(True, True): (f"{'Title':<28} {'Workspace':<18} {'Last Active':<13} {'ID'}", 110,
|
||||
lambda s: f"{_title(s, 26):<28} {_ws(s):<18} {_ago(s):<13} {s['id']}"),
|
||||
@@ -727,7 +727,7 @@ def _cmd_pinned(db, args):
|
||||
print(f"{'Title':<32} {'Last Active':<13} {'Src':<9} {'ID'}\n" + "─" * 100)
|
||||
for s in pinned_rows:
|
||||
title = (s.get("title") or s.get("preview", "") or "—")[:30]
|
||||
print(f"{title:<32} {_relative_time(s.get('last_active')):<13} {(s.get('source') or '-'):<9} {s['id']}")
|
||||
print(f"{title:<32} {_relative_time(s.get('last_active'), session_id=s['id']):<13} {(s.get('source') or '-'):<9} {s['id']}")
|
||||
|
||||
|
||||
def _cmd_retitle_skills(db, args):
|
||||
|
||||
@@ -58,7 +58,7 @@ def _format_row(s: dict, max_x: int) -> str:
|
||||
name = ((s.get("title") or "").strip() or (s.get("preview") or "").strip())[:name_width] or sid
|
||||
return (
|
||||
f"{name:<{name_width}} {_session_status_tag(s.get('_status')):<5} "
|
||||
f"{_msgs_str(s):>5} {_relative_time(s.get('last_active')):<10} "
|
||||
f"{_msgs_str(s):>5} {_relative_time(s.get('last_active'), session_id=s['id']):<10} "
|
||||
f"{s.get('source', '')[:6]:<5} {sid}"
|
||||
)
|
||||
|
||||
@@ -218,7 +218,7 @@ def _fallback_picker(sessions: list) -> Optional[str]:
|
||||
for i, s in enumerate(sessions):
|
||||
print(
|
||||
f" {i + 1:>3}. {_clip(_label(s), 50):<50} {_session_status_tag(s.get('_status')):<5} "
|
||||
f"{_msgs_str(s):>5} {_relative_time(s.get('last_active')):<10} {s.get('source', '')[:6]}"
|
||||
f"{_msgs_str(s):>5} {_relative_time(s.get('last_active'), session_id=s['id']):<10} {s.get('source', '')[:6]}"
|
||||
)
|
||||
while True:
|
||||
try:
|
||||
|
||||
@@ -2,13 +2,47 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import time as _time
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Epoch-seconds window a stored timestamp must fall in to be trusted: 1970 .. ~2103 (inside 32-bit
|
||||
# ``time_t`` so ``fromtimestamp`` accepts it on every platform). SQLite dynamic typing lets a TEXT
|
||||
# cell, ``inf``/``nan`` or a garbage double (``8.4e252`` salvaged from a damaged page) sit in a REAL
|
||||
# column; ``datetime.fromtimestamp`` then raises and one bad row killed the whole listing, export
|
||||
# or report (#102399, #102352, #99959).
|
||||
EPOCH_MIN = 0.0
|
||||
EPOCH_MAX = 4_200_000_000.0
|
||||
|
||||
|
||||
def relative_time(ts) -> str:
|
||||
"""Format a timestamp as relative time (e.g., '2h ago', 'yesterday')."""
|
||||
if not ts:
|
||||
def coerce_epoch(value: Any, *, session_id: Optional[str] = None, field: str = "timestamp") -> Optional[float]:
|
||||
"""A stored timestamp cell as float epoch seconds, or ``None`` when it cannot be trusted.
|
||||
|
||||
Numbers, numeric strings and ``datetime`` are accepted; anything else, non-finite values and
|
||||
values outside ``EPOCH_MIN..EPOCH_MAX`` return ``None`` after a WARNING naming the session so
|
||||
the corrupt row can be found. ``None``/``""`` mean "unset" and stay silent. Every reader that
|
||||
renders a row timestamp goes through here (a bad row degrades to one ``?`` cell, never a dead
|
||||
command) and every writer uses it to refuse persisting a new bad row.
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
ts = float(value.timestamp()) if isinstance(value, datetime) else float(value)
|
||||
except (TypeError, ValueError):
|
||||
ts = math.nan
|
||||
if not (EPOCH_MIN <= ts <= EPOCH_MAX): # also False for nan
|
||||
logger.warning("Ignoring corrupt %s %r%s", field, value, f" on session {session_id}" if session_id else "")
|
||||
return None
|
||||
return ts
|
||||
|
||||
|
||||
def relative_time(ts, *, session_id: Optional[str] = None) -> str:
|
||||
"""Format a timestamp as relative time (e.g., '2h ago', 'yesterday'); ``?`` when unset or corrupt."""
|
||||
if not ts or (ts := coerce_epoch(ts, session_id=session_id, field="last_active")) is None:
|
||||
return "?"
|
||||
delta = _time.time() - ts
|
||||
if delta < 60:
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
from agent.context_compressor import _DB_PERSISTED_MARKER as _DB_PERSISTED_MARKER_KEY, split_user_originated_turn
|
||||
from agent.memory_manager import sanitize_context
|
||||
from agent.message_sanitization import _sanitize_surrogates
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
from hermes_state_common import (
|
||||
_COMPRESSION_LOCK_ROW_SQL, _ENDED_ROW_SQL, _RESET_END_REASONS, _RESET_END_REASONS_SQL, _ended_by_compression,
|
||||
_legacy_reset_child_sql, _placeholders, _sql_json_extract)
|
||||
@@ -54,18 +55,15 @@ def _json_or(raw: Any, fallback: Any, warning: str) -> Any:
|
||||
|
||||
|
||||
def _coerce_timestamp(value: Any, default: float) -> float:
|
||||
"""Explicit message timestamp (datetime or number) or *default* when invalid."""
|
||||
if value is None:
|
||||
"""Explicit message timestamp (datetime or number) or *default* when invalid or outside the sane
|
||||
epoch window — the write-side twin of the readers' ``coerce_epoch``: a bad row is never persisted."""
|
||||
result = coerce_epoch(value, field="message timestamp")
|
||||
if result is None:
|
||||
return default
|
||||
try:
|
||||
result = float(value.timestamp()) if hasattr(value, "timestamp") else float(value)
|
||||
# SQLite reads both signed zero spellings back as 0.0. Hash the value
|
||||
# in that stored form so -0.0 and 0.0 retain the historical SQL/Python
|
||||
# identity equality.
|
||||
return 0.0 if result == 0.0 else result
|
||||
except (TypeError, ValueError):
|
||||
logger.debug("Ignoring invalid explicit message timestamp: %r", value)
|
||||
return default
|
||||
|
||||
|
||||
def _parse_tool_calls(tool_calls: Any) -> Any:
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from agent.skill_commands import SKILL_SCAFFOLD_SQL_LIKE
|
||||
from utils import safe_json_loads
|
||||
from hermes_cli.timefmt import coerce_epoch
|
||||
from hermes_state_common import SCHEMA_SQL, _PREVIEW_RAW_SUBQUERY_SQL, _shape_preview, _sql_session_last_active
|
||||
|
||||
# Pre-split logger identity so log filtering/capture is unchanged.
|
||||
@@ -465,7 +466,7 @@ class SessionPortabilityMixin:
|
||||
|
||||
def _import_session_row(self, conn, raw: Dict[str, Any], messages: List[Dict[str, Any]], session_id: str) -> None:
|
||||
"""INSERT one normalized session + its messages; counts fixed up after."""
|
||||
started_at = self._coerce_or(raw.get("started_at"), float, None)
|
||||
started_at = coerce_epoch(raw.get("started_at"), session_id=session_id, field="started_at")
|
||||
params = {
|
||||
"id": session_id, "source": str(raw.get("source") or "import"),
|
||||
"system_prompt_hash": self._store_system_prompt(conn, raw.get("system_prompt")),
|
||||
|
||||
Reference in New Issue
Block a user