1572836184
Same class as the config/manifest loader swap earlier in this stack. The /ready probe re-read and pure-Python-parsed config.yaml on every poll, inside the aiohttp handler (110 KB seeded config: 2332 -> 487 ms per probe on a loaded runner). It now uses utils.load_yaml_file_readonly: the C loader, and repeat probes reuse the parse until the file signature changes. Parse errors are not cached, so an edit that breaks or fixes the file shows on the next probe. The bundled-platform manifest reader was the one startup manifest reader left on yaml.safe_load. managed_scope imports fast_safe_load at module level next to file_signature instead of lazily.
123 lines
5.5 KiB
Python
123 lines
5.5 KiB
Python
"""Bounded, non-destructive readiness probes for authenticated health surfaces."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import sqlite3
|
|
from contextlib import closing
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from hermes_constants import get_hermes_home
|
|
from utils import load_yaml_file_readonly
|
|
|
|
|
|
_DISK_DEGRADED_PERCENT = 90.0
|
|
_CONNECTED_STATES = {"connected", "running", "ok"}
|
|
|
|
|
|
def _check(status: str, detail: str | None = None, **extra: Any) -> dict[str, Any]:
|
|
return {"status": status, **({"detail": detail} if detail else {}), **extra}
|
|
|
|
|
|
def _probe_state_db(home: Path) -> dict[str, Any]:
|
|
"""Read-only schema probe plus the process-wide corruption latch (``hermes_state_health``).
|
|
|
|
The schema read only catches an unreadable header or schema; damage deeper in the file
|
|
surfaces when a reader or writer touches it, and those publish into the latch. Reporting
|
|
the latch here is what makes readiness and ``/api/status`` agree with the session list
|
|
(#72046). ``detail="corrupt"`` is the one reason string consumers key off."""
|
|
from hermes_state_health import STORAGE_CORRUPT, note_storage_error, storage_state
|
|
|
|
path = home / "state.db"
|
|
if not path.exists():
|
|
return _check("ok", "not initialized")
|
|
if storage_state(path) == STORAGE_CORRUPT:
|
|
return _check("degraded", STORAGE_CORRUPT)
|
|
try:
|
|
# Read-only schema query: catches unreadable/corrupt DBs without competing with
|
|
# writers. ``closing`` is required — sqlite3's context manager only commits/rolls
|
|
# back, never closes, so a bare ``with connect()`` leaks a connection per poll.
|
|
with closing(sqlite3.connect(f"file:{path.as_posix()}?mode=ro", uri=True, timeout=1.0)) as conn:
|
|
# A readiness probe must never compete with normal state writers. See #69567, #69678.
|
|
conn.execute("PRAGMA query_only = ON")
|
|
conn.execute("SELECT name FROM sqlite_master LIMIT 1").fetchone()
|
|
return _check("ok")
|
|
except Exception as exc:
|
|
if note_storage_error(path, exc):
|
|
return _check("degraded", STORAGE_CORRUPT)
|
|
return _check("degraded", type(exc).__name__)
|
|
|
|
|
|
def _probe_config(home: Path) -> dict[str, Any]:
|
|
path = home / "config.yaml"
|
|
if not path.exists():
|
|
return _check("ok", "using defaults")
|
|
try:
|
|
raw = load_yaml_file_readonly(path)
|
|
except Exception as exc:
|
|
return _check("degraded", f"invalid config ({type(exc).__name__})")
|
|
return _check("ok") if raw is None or isinstance(raw, dict) else _check("degraded", "top level is not a mapping")
|
|
|
|
|
|
def _probe_disk(home: Path) -> dict[str, Any]:
|
|
try:
|
|
usage = shutil.disk_usage(home)
|
|
except Exception as exc:
|
|
return _check("degraded", type(exc).__name__)
|
|
used_pct = round((usage.used / usage.total) * 100, 1) if usage.total else 0.0
|
|
return _check("degraded" if used_pct >= _DISK_DEGRADED_PERCENT else "ok", used_percent=used_pct, free_bytes=usage.free)
|
|
|
|
|
|
def _probe_gateway(runtime_status: dict[str, Any]) -> dict[str, Any]:
|
|
state = str(runtime_status.get("gateway_state") or "unknown")
|
|
platforms = runtime_status.get("platforms")
|
|
platforms = platforms if isinstance(platforms, dict) else {}
|
|
connected = sum(
|
|
isinstance(v, dict) and str(v.get("state") or v.get("status") or "").lower() in _CONNECTED_STATES
|
|
for v in platforms.values()
|
|
)
|
|
return _check("ok" if state in {"running", "draining"} else "degraded", state=state,
|
|
connected_platforms=connected, platforms=len(platforms))
|
|
|
|
|
|
def _probe_session_store(runtime_status: dict[str, Any], state_db_probe: dict[str, Any]) -> dict[str, Any]:
|
|
"""Report the running gateway cache state, not an independent reopen. A corrupt store is
|
|
unavailable whatever the cache says: an open handle on a damaged file is not a working one."""
|
|
if state_db_probe.get("detail") == "corrupt":
|
|
return _check("unavailable", "corrupt")
|
|
runtime_store = runtime_status.get("session_store")
|
|
state = str(runtime_store.get("status") or "unknown") if isinstance(runtime_store, dict) else ""
|
|
if state in {"ok", "unavailable", "retrying"}:
|
|
return _check(state)
|
|
# Older gateways publish no cache state: fall back to the state_db probe.
|
|
return _check("ok" if state_db_probe.get("status") == "ok" else "unavailable")
|
|
|
|
|
|
def collect_runtime_readiness(
|
|
*, configured_model: str, runtime_status: dict[str, Any] | None, active_api_runs: int = 0,
|
|
process_completion_queue_depth: int = 0, active_delegations: int = 0,
|
|
) -> dict[str, Any]:
|
|
"""Bounded readiness diagnostics, no runtime mutation. Even authenticated, probes
|
|
expose status and counts only: never config values, credentials, paths, payloads."""
|
|
home = get_hermes_home()
|
|
runtime = runtime_status if isinstance(runtime_status, dict) else {}
|
|
state_db_probe = _probe_state_db(home)
|
|
checks = {
|
|
"state_db": state_db_probe,
|
|
"session_store": _probe_session_store(runtime, state_db_probe),
|
|
"config": _probe_config(home),
|
|
"model": _check("ok" if str(configured_model or "").strip() else "degraded"),
|
|
"disk": _probe_disk(home),
|
|
"gateway": _probe_gateway(runtime),
|
|
"background_queues": _check(
|
|
"ok", active_api_runs=max(0, int(active_api_runs)),
|
|
process_completions=max(0, int(process_completion_queue_depth)),
|
|
active_delegations=max(0, int(active_delegations)),
|
|
),
|
|
}
|
|
return {"status": "ok" if all(c.get("status") == "ok" for c in checks.values()) else "degraded", "checks": checks}
|
|
|
|
|
|
__all__ = ["collect_runtime_readiness"]
|