1498 lines
74 KiB
Python
1498 lines
74 KiB
Python
"""Hindsight memory plugin — MemoryProvider interface.
|
|
|
|
Long-term memory with knowledge graph, entity resolution, and multi-strategy
|
|
retrieval. Supports cloud (API key) and local modes.
|
|
|
|
Config via environment variables:
|
|
HINDSIGHT_API_KEY — API key for Hindsight Cloud
|
|
HINDSIGHT_BANK_ID — memory bank identifier (default: hermes)
|
|
HINDSIGHT_BUDGET — recall budget: low/mid/high (default: mid)
|
|
HINDSIGHT_API_URL — API endpoint
|
|
HINDSIGHT_MODE — cloud or local (default: cloud)
|
|
HINDSIGHT_TIMEOUT — API request timeout in seconds (default: 120)
|
|
HINDSIGHT_IDLE_TIMEOUT — embedded daemon idle timeout seconds; 0 disables shutdown (default: 300)
|
|
HINDSIGHT_EMBED_PORT_HEALTH_GRACE_TIMEOUT — seconds to wait for a slow embedded daemon /health before treating it as stale (default: 30; set via config.json port_health_grace_timeout)
|
|
HINDSIGHT_RETAIN_TAGS — comma-separated tags attached to retained memories
|
|
HINDSIGHT_RETAIN_OBSERVATION_SCOPES — observation scoping for retained memories: per_tag/combined/all_combinations, or a JSON list of tag-lists for custom scopes
|
|
HINDSIGHT_RETAIN_SOURCE — metadata source value attached to retained memories (default: hermes)
|
|
HINDSIGHT_RETAIN_USER_PREFIX — label used before user turns in retained transcripts
|
|
HINDSIGHT_RETAIN_ASSISTANT_PREFIX — label used before assistant turns in retained transcripts
|
|
|
|
Or via $HERMES_HOME/hindsight/config.json (profile-scoped), falling back to
|
|
~/.hindsight/config.json (legacy, shared) for backward compatibility.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import atexit
|
|
import contextvars
|
|
import json
|
|
import logging
|
|
import os
|
|
import queue
|
|
import sys
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict, List, Optional
|
|
|
|
from agent.memory_provider import MemoryProvider, RecallStatus
|
|
from agent.secret_scope import get_secret
|
|
from hermes_cli.config import cfg_get
|
|
from hermes_constants import get_hermes_home
|
|
from hermes_time import now as _hermes_now
|
|
from tools.registry import tool_error
|
|
|
|
from .embedded import ( # noqa: F401 (re-exported; tests patch/import via this module)
|
|
_PORT_HEALTH_GRACE_ENV,
|
|
_RETRIABLE_CONNECTION_MARKERS,
|
|
_build_embedded_profile_env,
|
|
_check_local_runtime,
|
|
_embedded_profile_env_path,
|
|
_export_port_health_grace_timeout,
|
|
_load_simple_env,
|
|
_local_runtime_hint,
|
|
_materialize_embedded_profile_env,
|
|
_secure_write_profile_env,
|
|
_validate_profile_env_permissions,
|
|
)
|
|
from .settings import ( # noqa: F401
|
|
_DEFAULT_API_URL,
|
|
_DEFAULT_IDLE_TIMEOUT,
|
|
_DEFAULT_LOCAL_URL,
|
|
_DEFAULT_RETAIN_SOURCE,
|
|
_DEFAULT_TIMEOUT,
|
|
_HINDSIGHT_GLYPH,
|
|
_MIN_CLIENT_VERSION,
|
|
_MIN_VERSION_FOR_UPDATE_MODE_APPEND,
|
|
_OBSERVATION_SCOPE_KEYWORDS,
|
|
_PROVIDER_DEFAULT_MODELS,
|
|
_VALID_BUDGETS,
|
|
_daemon_llm_provider,
|
|
_normalize_observation_scopes,
|
|
_normalize_retain_tags,
|
|
_parse_int_setting,
|
|
_resolve_bank_id_template,
|
|
_sanitize_bank_segment,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_LOCAL_MODES = {"local", "local_embedded"}
|
|
_RETAIN_CONTEXT_DEFAULT = "conversation between Hermes Agent and the User"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _RecallResult:
|
|
"""Text + memory count from one recall, so the deterministic recall indicator
|
|
can report "recalled N memories" without re-parsing the formatted text.
|
|
``count`` is 0 for a reflect synthesis or on error."""
|
|
|
|
text: str
|
|
count: int
|
|
|
|
|
|
def _ensure_client_dependency() -> None:
|
|
"""Lazily install the Hindsight client (``tools.lazy_deps``) before importing it."""
|
|
try:
|
|
from tools.lazy_deps import ensure as _lazy_ensure
|
|
_lazy_ensure("memory.hindsight", prompt=False)
|
|
except ImportError:
|
|
pass
|
|
except Exception as exc:
|
|
raise ImportError(str(exc)) from exc
|
|
|
|
|
|
def _cloud_api_key(config: dict) -> str:
|
|
return config.get("apiKey") or config.get("api_key") or get_secret("HINDSIGHT_API_KEY", "")
|
|
|
|
|
|
def _maybe_upgrade_client() -> None:
|
|
"""Auto-upgrade an outdated hindsight-client through the environment-aware
|
|
lazy_deps installer (sealed hosted venvs redirect to the durable target)."""
|
|
try:
|
|
from importlib.metadata import version as pkg_version
|
|
from packaging.version import Version
|
|
installed = pkg_version("hindsight-client")
|
|
if Version(installed) < Version(_MIN_CLIENT_VERSION):
|
|
logger.warning("hindsight-client %s is outdated (need >=%s), attempting upgrade...",
|
|
installed, _MIN_CLIENT_VERSION)
|
|
from tools.lazy_deps import install_specs
|
|
outcome = install_specs([f"hindsight-client>={_MIN_CLIENT_VERSION}"], timeout=120)
|
|
if outcome.ok:
|
|
logger.info("hindsight-client upgraded to >=%s", _MIN_CLIENT_VERSION)
|
|
elif outcome.blocked:
|
|
logger.warning("Auto-upgrade unavailable: %s. Run: uv pip install 'hindsight-client>=%s'",
|
|
outcome.reason, _MIN_CLIENT_VERSION)
|
|
else:
|
|
logger.warning("Auto-upgrade failed: %s. Run: uv pip install 'hindsight-client>=%s'",
|
|
(outcome.stderr or "").strip() or "install error", _MIN_CLIENT_VERSION)
|
|
except Exception:
|
|
pass # packaging not available or other issue — proceed anyway
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Hindsight API capability probe (update_mode='append', Hindsight >= 0.5.0).
|
|
# Cached per API URL per process so every provider on the same API shares one
|
|
# /version round trip.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_append_capability_cache: Dict[str, bool] = {}
|
|
_append_capability_lock = threading.Lock()
|
|
|
|
|
|
def _meets_minimum_version(actual: str | None, required: str) -> bool:
|
|
"""True if *actual* >= *required* (semver). False on missing/invalid."""
|
|
if not actual:
|
|
return False
|
|
try:
|
|
from packaging.version import Version
|
|
return Version(actual) >= Version(required)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _fetch_hindsight_api_version(api_url: str, api_key: str | None = None,
|
|
timeout: float = 5.0) -> str | None:
|
|
"""GET ``<api_url>/version`` -> version string, or None on any failure
|
|
(the caller treats None as "legacy API, no update_mode support")."""
|
|
import urllib.request
|
|
if not api_url:
|
|
return None
|
|
url = api_url.rstrip("/") + "/version"
|
|
req = urllib.request.Request(url)
|
|
if api_key:
|
|
req.add_header("Authorization", f"Bearer {api_key}")
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310
|
|
data = json.loads(resp.read().decode("utf-8", errors="replace"))
|
|
except Exception as exc:
|
|
logger.debug("Hindsight /version probe failed for %s: %s", url, exc)
|
|
return None
|
|
if not isinstance(data, dict):
|
|
return None
|
|
version = data.get("version") or data.get("api_version")
|
|
return str(version) if version else None
|
|
|
|
|
|
def _check_api_supports_update_mode_append(api_url: str,
|
|
api_key: str | None = None) -> bool:
|
|
"""Cached capability check for ``update_mode='append'`` on *api_url*.
|
|
|
|
False on any probe failure — the safe default: a per-process unique
|
|
``document_id`` and no ``update_mode`` keeps the resume-overwrite fix intact.
|
|
"""
|
|
if not api_url:
|
|
return False
|
|
with _append_capability_lock:
|
|
if api_url in _append_capability_cache:
|
|
return _append_capability_cache[api_url]
|
|
version = _fetch_hindsight_api_version(api_url, api_key)
|
|
supported = _meets_minimum_version(version, _MIN_VERSION_FOR_UPDATE_MODE_APPEND)
|
|
with _append_capability_lock:
|
|
# A concurrent probe may have filled the cache meanwhile; its answer wins.
|
|
supported = _append_capability_cache.setdefault(api_url, supported)
|
|
if not supported:
|
|
logger.warning(
|
|
"Hindsight API at %s reports version %r, older than %s. "
|
|
"Falling back to per-process document_id — retains across "
|
|
"processes/sessions create separate documents instead of "
|
|
"appending to a session-scoped one. Upgrade Hindsight to "
|
|
"%s+ to enable update_mode='append' deduplication.",
|
|
api_url, version, _MIN_VERSION_FOR_UPDATE_MODE_APPEND,
|
|
_MIN_VERSION_FOR_UPDATE_MODE_APPEND,
|
|
)
|
|
else:
|
|
logger.debug("Hindsight API %s version %s supports update_mode='append'",
|
|
api_url, version)
|
|
return supported
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dedicated event loop for Hindsight async calls (one per process, reused).
|
|
# Avoids creating ephemeral loops that leak aiohttp sessions.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_loop: asyncio.AbstractEventLoop | None = None
|
|
_loop_thread: threading.Thread | None = None
|
|
_loop_lock = threading.Lock()
|
|
|
|
# Pushed to the per-provider retain queue to wake the writer for a clean exit.
|
|
_WRITER_SENTINEL = object()
|
|
|
|
|
|
def _get_loop() -> asyncio.AbstractEventLoop:
|
|
"""Return a long-lived event loop running on a background thread."""
|
|
global _loop, _loop_thread
|
|
with _loop_lock:
|
|
if _loop is not None and _loop.is_running():
|
|
return _loop
|
|
_loop = asyncio.new_event_loop()
|
|
|
|
def _run():
|
|
asyncio.set_event_loop(_loop)
|
|
_loop.run_forever()
|
|
|
|
_loop_thread = threading.Thread(target=_run, daemon=True, name="hindsight-loop")
|
|
_loop_thread.start()
|
|
return _loop
|
|
|
|
|
|
def _run_sync(coro, timeout: float = _DEFAULT_TIMEOUT):
|
|
"""Schedule *coro* on the shared loop and block until done."""
|
|
from agent.async_utils import safe_schedule_threadsafe
|
|
future = safe_schedule_threadsafe(coro, _get_loop())
|
|
if future is None:
|
|
raise RuntimeError("Hindsight loop unavailable")
|
|
return future.result(timeout=timeout)
|
|
|
|
|
|
def _context_thread(target, name: str) -> threading.Thread:
|
|
"""Daemon thread running *target* in a snapshot of the spawner's contextvars.
|
|
|
|
Background threads start with an EMPTY Context; under multiplex_profiles the
|
|
spawning thread carries the profile's secret scope + HERMES_HOME override, and
|
|
get_secret fails closed without it. (The shared ``hindsight-loop`` needs no
|
|
wrap: coroutines scheduled via run_coroutine_threadsafe inherit the
|
|
submitter's context per call.)
|
|
"""
|
|
return threading.Thread(
|
|
target=contextvars.copy_context().run, args=(target,), daemon=True, name=name,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Tool schemas
|
|
# ---------------------------------------------------------------------------
|
|
|
|
RETAIN_SCHEMA = {
|
|
"name": "hindsight_retain",
|
|
"description": (
|
|
"Store information to long-term memory. Hindsight automatically "
|
|
"extracts structured facts, resolves entities, and indexes for retrieval."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string", "description": "The information to store."},
|
|
"context": {"type": "string", "description": "Short label (e.g. 'user preference', 'project decision')."},
|
|
"tags": {
|
|
"type": "array",
|
|
"items": {"type": "string"},
|
|
"description": "Optional per-call tags to merge with configured default retain tags.",
|
|
},
|
|
"occurred_at": {
|
|
"type": "string",
|
|
"description": (
|
|
"When the remembered event actually happened, as an ISO-8601 date "
|
|
"or datetime (e.g. '2026-08-20' or '2026-08-20T14:30:00+02:00'). "
|
|
"Pass this whenever the memory references a specific event time "
|
|
"('yesterday', 'last Tuesday', 'on March 3rd') so Hindsight can "
|
|
"anchor it on the timeline. Omit for timeless facts/preferences."
|
|
),
|
|
},
|
|
},
|
|
"required": ["content"],
|
|
},
|
|
}
|
|
|
|
RECALL_SCHEMA = {
|
|
"name": "hindsight_recall",
|
|
"description": (
|
|
"Search long-term memory. Returns memories ranked by relevance using "
|
|
"semantic search, keyword matching, entity graph traversal, and reranking."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {"type": "string", "description": "What to search for."},
|
|
},
|
|
"required": ["query"],
|
|
},
|
|
}
|
|
|
|
REFLECT_SCHEMA = {
|
|
"name": "hindsight_reflect",
|
|
"description": (
|
|
"Synthesize a reasoned answer from long-term memories. Unlike recall, "
|
|
"this reasons across all stored memories to produce a coherent response."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {"type": "string", "description": "The question to reflect on."},
|
|
},
|
|
"required": ["query"],
|
|
},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_config() -> dict:
|
|
"""Load config: $HERMES_HOME/hindsight/config.json (profile-scoped), then
|
|
~/.hindsight/config.json (legacy, shared), then environment variables."""
|
|
for path in (get_hermes_home() / "hindsight" / "config.json",
|
|
Path.home() / ".hindsight" / "config.json"):
|
|
if path.exists():
|
|
try:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
pass
|
|
return {
|
|
"mode": os.environ.get("HINDSIGHT_MODE", "cloud"),
|
|
"apiKey": get_secret("HINDSIGHT_API_KEY", ""),
|
|
"timeout": _parse_int_setting(os.environ.get("HINDSIGHT_TIMEOUT"), _DEFAULT_TIMEOUT),
|
|
"idle_timeout": _parse_int_setting(os.environ.get("HINDSIGHT_IDLE_TIMEOUT"), _DEFAULT_IDLE_TIMEOUT),
|
|
"retain_tags": os.environ.get("HINDSIGHT_RETAIN_TAGS", ""),
|
|
"observation_scopes": os.environ.get("HINDSIGHT_RETAIN_OBSERVATION_SCOPES", ""),
|
|
"retain_source": os.environ.get("HINDSIGHT_RETAIN_SOURCE", _DEFAULT_RETAIN_SOURCE),
|
|
"retain_user_prefix": os.environ.get("HINDSIGHT_RETAIN_USER_PREFIX", "User"),
|
|
"retain_assistant_prefix": os.environ.get("HINDSIGHT_RETAIN_ASSISTANT_PREFIX", "Assistant"),
|
|
"banks": {
|
|
"hermes": {
|
|
"bankId": os.environ.get("HINDSIGHT_BANK_ID", "hermes"),
|
|
"budget": os.environ.get("HINDSIGHT_BUDGET", "mid"),
|
|
"enabled": True,
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _utc_timestamp() -> str:
|
|
"""UTC write/audit time for retain metadata."""
|
|
return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
|
|
|
|
def _event_timestamp() -> str:
|
|
"""Configured Hermes event time with an explicit UTC offset."""
|
|
event_time = _hermes_now()
|
|
# hermes_time.now() guarantees an aware datetime; the fallback keeps a
|
|
# replacement clock from silently emitting an offset-less Event Date.
|
|
if event_time.tzinfo is None or event_time.utcoffset() is None:
|
|
event_time = event_time.astimezone()
|
|
return event_time.isoformat(timespec="seconds")
|
|
|
|
|
|
def _mint_document_id(session_id: str) -> str:
|
|
"""Per-process-lifecycle document id. Reusing session_id alone caused
|
|
overwrites on /resume (the reloaded session starts with empty
|
|
_session_turns, so its next retain replaced the stored content)."""
|
|
return f"{session_id}-{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MemoryProvider implementation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# initialize() kwargs copied verbatim (str, stripped) onto ``self._<name>``.
|
|
_SESSION_KWARGS = (
|
|
"platform", "user_id", "user_name", "chat_id", "chat_name", "chat_type",
|
|
"thread_id", "agent_identity", "agent_workspace",
|
|
)
|
|
# Retain metadata keys, each stamped from the attribute of the same name when set.
|
|
_METADATA_ATTRS = (
|
|
"session_id", "platform", "user_id", "user_name", "chat_id", "chat_name",
|
|
"chat_type", "thread_id", "agent_identity",
|
|
)
|
|
_SYSTEM_PROMPT_TAILS = {
|
|
"context": "Relevant memories are automatically injected into context.",
|
|
"tools": ("Use hindsight_recall to search, hindsight_reflect for synthesis, "
|
|
"hindsight_retain to store facts."),
|
|
"hybrid": ("Relevant memories are automatically injected into context. "
|
|
"Use hindsight_recall to search, hindsight_reflect for synthesis, "
|
|
"hindsight_retain to store facts."),
|
|
}
|
|
_TOOL_ERRORS = {
|
|
"hindsight_retain": "Failed to store memory",
|
|
"hindsight_recall": "Failed to search memory",
|
|
"hindsight_reflect": "Failed to reflect",
|
|
}
|
|
|
|
|
|
class HindsightMemoryProvider(MemoryProvider):
|
|
"""Hindsight long-term memory with knowledge graph and multi-strategy retrieval."""
|
|
|
|
def backup_paths(self) -> List[str]:
|
|
"""Legacy shared config + embedded-mode profile env files live under ~/.hindsight."""
|
|
try:
|
|
return [str(Path.home() / ".hindsight")]
|
|
except Exception:
|
|
return []
|
|
|
|
def __init__(self):
|
|
self._config = None
|
|
self._api_key = None
|
|
self._api_url = _DEFAULT_API_URL
|
|
self._bank_id = "hermes"
|
|
self._budget = "mid"
|
|
self._mode = "cloud"
|
|
self._llm_base_url = ""
|
|
self._memory_mode = "hybrid" # "context", "tools", or "hybrid"
|
|
self._prefetch_method = "recall" # "recall" or "reflect"
|
|
self._retain_tags: List[str] = []
|
|
self._retain_source = _DEFAULT_RETAIN_SOURCE
|
|
self._retain_user_prefix = "User"
|
|
self._retain_assistant_prefix = "Assistant"
|
|
for name in _SESSION_KWARGS:
|
|
setattr(self, f"_{name}", "")
|
|
self._turn_index = 0
|
|
self._client = None
|
|
self._timeout = _DEFAULT_TIMEOUT
|
|
self._idle_timeout = _DEFAULT_IDLE_TIMEOUT
|
|
# Pending prefetch block + its memory count (for the recall indicator).
|
|
self._prefetch_result = ""
|
|
self._prefetch_count = 0
|
|
self._prefetch_lock = threading.Lock()
|
|
self._prefetch_thread = None
|
|
# Model-independent recall indicator state (see recall_status()).
|
|
self._last_recall_returned = False
|
|
self._last_recall_count = 0
|
|
self._recall_indicator = True
|
|
# Deterministic retain indicator, emitted via the agent's status channel
|
|
# (injected through initialize(status_callback=)).
|
|
self._retain_indicator = True
|
|
self._status_callback: Optional[Callable[[str], None]] = None
|
|
# Single-writer model for retain: sync_turn() enqueues, one writer
|
|
# thread drains sequentially. Ad-hoc threads raced interpreter shutdown
|
|
# ("cannot schedule new futures" / "Unclosed client session").
|
|
self._retain_queue: queue.Queue = queue.Queue()
|
|
self._writer_thread: threading.Thread | None = None
|
|
self._shutting_down = threading.Event()
|
|
self._atexit_registered = False
|
|
# Server-side async retain ops still in flight. With retain_async=True,
|
|
# aretain_batch returns on *acceptance*, not durability, so the
|
|
# background prefetch gates on these via get_operation_status (draining
|
|
# the local queue alone is not a read-after-write signal).
|
|
self._pending_retain_ops: set[str] = set()
|
|
self._pending_retain_ops_lock = threading.Lock()
|
|
self._retain_ops_bank_id = ""
|
|
# Seconds between get_operation_status polls — each is a server round
|
|
# trip, so deliberately coarser than the 0.05s local queue-drain poll.
|
|
self._RETAIN_OP_POLL_INTERVAL_S = 0.5
|
|
# Legacy alias — external callers may join _sync_thread; points at the writer.
|
|
self._sync_thread = None
|
|
self._session_id = ""
|
|
self._parent_session_id = ""
|
|
self._document_id = ""
|
|
|
|
self._tags: list[str] | None = None
|
|
self._recall_tags: list[str] | None = None
|
|
self._recall_tags_match = "any"
|
|
|
|
self._auto_retain = True
|
|
self._retain_every_n_turns = 1
|
|
self._retain_async = True
|
|
# Async retain never blocks the reply, but the next turn's warm prefetch
|
|
# could read BEFORE the retain is recall-visible. When True the prefetch
|
|
# first waits (bounded) for the writer queue to drain AND the server-side
|
|
# op(s) to complete — closing the race off the reply path.
|
|
self._prefetch_waits_for_retain = True
|
|
self._prefetch_retain_drain_timeout = 10.0
|
|
self._retain_context = _RETAIN_CONTEXT_DEFAULT
|
|
self._turn_counter = 0
|
|
self._session_turns: list[str] = [] # ALL turns for the session
|
|
# Turns already shipped by the last append-mode retain (delta watermark).
|
|
self._last_retained_turn_count = 0
|
|
|
|
self._auto_recall = True
|
|
self._recall_sync = False
|
|
self._recall_max_tokens = 4096
|
|
# Observation-only by default: observations are Hindsight's consolidated,
|
|
# deduplicated knowledge layer; raw world/experience facts re-ship the
|
|
# evidence they summarize and burn the recall_max_tokens budget.
|
|
self._recall_types: list[str] = ["observation"]
|
|
self._recall_prompt_preamble = ""
|
|
self._recall_max_input_chars = 800
|
|
|
|
self._bank_mission = ""
|
|
self._bank_retain_mission: str | None = None
|
|
self._bank_id_template = ""
|
|
|
|
@property
|
|
def name(self) -> str:
|
|
return "hindsight"
|
|
|
|
def is_available(self) -> bool:
|
|
try:
|
|
cfg = _load_config()
|
|
mode = cfg.get("mode", "cloud")
|
|
if mode in _LOCAL_MODES:
|
|
return _check_local_runtime()[0]
|
|
if mode == "local_external":
|
|
return True
|
|
return bool(_cloud_api_key(cfg)) or bool(cfg.get("api_url") or os.environ.get("HINDSIGHT_API_URL", ""))
|
|
except Exception:
|
|
return False
|
|
|
|
def unavailable_reason(self) -> str:
|
|
"""Install guidance for an unavailable local_embedded runtime. is_available()
|
|
gates initialize() out, so the hint it would log is never reached; agent_init
|
|
surfaces this instead."""
|
|
try:
|
|
mode = _load_config().get("mode", "cloud")
|
|
except Exception:
|
|
return ""
|
|
if mode not in _LOCAL_MODES:
|
|
return ""
|
|
available, reason = _check_local_runtime()
|
|
return "" if available else _local_runtime_hint(reason).strip()
|
|
|
|
def save_config(self, values, hermes_home):
|
|
"""Merge *values* into $HERMES_HOME/hindsight/config.json."""
|
|
from utils import atomic_json_write
|
|
config_path = Path(hermes_home) / "hindsight" / "config.json"
|
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
|
existing = {}
|
|
if config_path.exists():
|
|
try:
|
|
existing = json.loads(config_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
pass
|
|
existing.update(values)
|
|
atomic_json_write(config_path, existing, mode=0o600)
|
|
|
|
def post_setup(self, hermes_home: str, config: dict) -> None:
|
|
"""Custom setup wizard — installs only the deps needed for the selected mode."""
|
|
from .setup import run_setup
|
|
run_setup(self, hermes_home, config)
|
|
|
|
def get_config_schema(self):
|
|
return [
|
|
{"key": "mode", "description": "Connection mode", "default": "cloud", "choices": ["cloud", "local_embedded", "local_external"]},
|
|
# Cloud mode
|
|
{"key": "api_url", "description": "Hindsight Cloud API URL", "default": _DEFAULT_API_URL, "when": {"mode": "cloud"}},
|
|
{"key": "api_key", "description": "Hindsight Cloud API key", "secret": True, "env_var": "HINDSIGHT_API_KEY", "url": "https://ui.hindsight.vectorize.io", "when": {"mode": "cloud"}},
|
|
# Local external mode
|
|
{"key": "api_url", "description": "Hindsight API URL", "default": _DEFAULT_LOCAL_URL, "when": {"mode": "local_external"}},
|
|
{"key": "api_key", "description": "API key (optional)", "secret": True, "env_var": "HINDSIGHT_API_KEY", "when": {"mode": "local_external"}},
|
|
# Local embedded mode
|
|
{"key": "llm_provider", "description": "LLM provider", "default": "openai", "choices": ["openai", "anthropic", "gemini", "groq", "openrouter", "minimax", "ollama", "lmstudio", "openai_compatible"], "when": {"mode": "local_embedded"}},
|
|
{"key": "llm_base_url", "description": "Endpoint URL (e.g. http://192.168.1.10:8080/v1)", "default": "", "when": {"mode": "local_embedded", "llm_provider": "openai_compatible"}},
|
|
{"key": "llm_api_key", "description": "LLM API key (optional for openai_compatible)", "secret": True, "env_var": "HINDSIGHT_LLM_API_KEY", "when": {"mode": "local_embedded"}},
|
|
{"key": "llm_model", "description": "LLM model", "default": "gpt-4o-mini", "default_from": {"field": "llm_provider", "map": _PROVIDER_DEFAULT_MODELS}, "when": {"mode": "local_embedded"}},
|
|
{"key": "bank_id", "description": "Memory bank name (static fallback when bank_id_template is unset)", "default": "hermes"},
|
|
{"key": "bank_id_template", "description": "Optional template to derive bank_id dynamically. Placeholders: {profile}, {workspace}, {platform}, {user}, {session}. Example: hermes-{profile}", "default": ""},
|
|
{"key": "bank_mission", "description": "Mission/purpose description for the memory bank"},
|
|
{"key": "bank_retain_mission", "description": "Custom extraction prompt for memory retention"},
|
|
{"key": "recall_budget", "description": "Recall thoroughness", "default": "mid", "choices": ["low", "mid", "high"]},
|
|
{"key": "memory_mode", "description": "Memory integration mode", "default": "hybrid", "choices": ["hybrid", "context", "tools"]},
|
|
{"key": "recall_prefetch_method", "description": "Auto-recall method", "default": "recall", "choices": ["recall", "reflect"]},
|
|
{"key": "retain_tags", "description": "Default tags applied to retained memories (comma-separated)", "default": ""},
|
|
{"key": "observation_scopes", "description": "How observations are scoped during consolidation: 'combined' (default — one pass over all tags), 'per_tag' (one isolated observation per tag), 'all_combinations' (every tag subset — expensive), or a JSON list of tag-lists for explicit custom scopes. Empty uses Hindsight's 'combined' default.", "default": ""},
|
|
{"key": "retain_source", "description": "Metadata source value attached to retained memories (identifies the client that stored them)", "default": _DEFAULT_RETAIN_SOURCE},
|
|
{"key": "retain_user_prefix", "description": "Label used before user turns in retained transcripts", "default": "User"},
|
|
{"key": "retain_assistant_prefix", "description": "Label used before assistant turns in retained transcripts", "default": "Assistant"},
|
|
{"key": "recall_tags", "description": "Tags to filter when searching memories (comma-separated)", "default": ""},
|
|
{"key": "recall_tags_match", "description": "Tag matching mode for recall", "default": "any", "choices": ["any", "all", "any_strict", "all_strict"]},
|
|
{"key": "recall_types", "description": "Fact types to surface on recall — applies to both auto-recall and the hindsight_recall tool (comma-separated or list). Defaults to observation-only — observations are Hindsight's consolidated, deduplicated, evidence-grounded knowledge layer; raw world/experience facts are the supporting evidence observations already summarize. Set to e.g. 'observation,world,experience' to also include raw facts.", "default": "observation"},
|
|
{"key": "auto_recall", "description": "Automatically recall memories before each turn", "default": True},
|
|
{"key": "recall_sync", "description": "Recall synchronously against the current message before each turn (higher relevance, adds recall latency to the turn). Default off: recall runs in the background and is injected on the next turn.", "default": False},
|
|
{"key": "recall_indicator", "description": "Show a '👁️ Hindsight — recalled N memories' status line when auto-recall injects memory (turn off for customer-facing agents)", "default": True},
|
|
{"key": "retain_indicator", "description": "Show a '👁️ Hindsight — saving to memory…' status line when a turn is saved to memory (turn off for customer-facing agents)", "default": True},
|
|
{"key": "auto_retain", "description": "Automatically retain conversation turns", "default": True},
|
|
{"key": "retain_every_n_turns", "description": "Retain every N turns (1 = every turn)", "default": 1},
|
|
{"key": "retain_async","description": "Process retain asynchronously on the Hindsight server", "default": True},
|
|
{"key": "prefetch_waits_for_retain", "description": "Have the background next-turn prefetch wait for the just-completed retain to become recall-visible on the server (local queue drain + async operation completion) before recalling, so recall includes the just-completed turn (runs off the reply path, adds no response latency)", "default": True},
|
|
{"key": "prefetch_retain_drain_timeout", "description": "Max seconds the background prefetch waits for the retain to become recall-visible (queue drain + server-side completion) before recalling anyway", "default": 10.0},
|
|
{"key": "retain_context", "description": "Context label for retained memories", "default": "conversation between Hermes Agent and the User"},
|
|
{"key": "recall_max_tokens", "description": "Maximum tokens for recall results", "default": 4096},
|
|
{"key": "recall_max_input_chars", "description": "Maximum input query length for auto-recall", "default": 800},
|
|
{"key": "recall_prompt_preamble", "description": "Custom preamble for recalled memories in context"},
|
|
{"key": "timeout", "description": "API request timeout in seconds", "default": _DEFAULT_TIMEOUT},
|
|
{"key": "idle_timeout", "description": "Embedded daemon idle timeout in seconds (0 disables auto-shutdown)", "default": _DEFAULT_IDLE_TIMEOUT, "when": {"mode": "local_embedded"}},
|
|
{"key": "port_health_grace_timeout", "description": "Seconds to wait for a slow daemon /health before treating it as stale (raise on busy/low-resource hosts; blank uses the 30s default)", "default": "", "when": {"mode": "local_embedded"}},
|
|
]
|
|
|
|
# -- client -------------------------------------------------------------
|
|
|
|
def _int_setting(self, key: str, env_var: str, default: int, env_default=None) -> int:
|
|
"""Config value if set (explicit 0 preserved), else env var, else default."""
|
|
value = self._config.get(key)
|
|
if value is None:
|
|
value = os.environ.get(env_var, env_default)
|
|
return _parse_int_setting(value, default)
|
|
|
|
def _new_embedded_client(self):
|
|
available, reason = _check_local_runtime()
|
|
if not available:
|
|
raise RuntimeError("Hindsight local runtime is unavailable" + (f": {reason}" if reason else ""))
|
|
_ensure_client_dependency()
|
|
from hindsight import HindsightEmbedded
|
|
HindsightEmbedded.__del__ = lambda self: None
|
|
cfg = self._config
|
|
llm_provider = _daemon_llm_provider(cfg.get("llm_provider", ""))
|
|
logger.debug("Creating HindsightEmbedded client (profile=%s, provider=%s)",
|
|
cfg.get("profile", "hermes"), llm_provider)
|
|
kwargs = dict(
|
|
profile=cfg.get("profile", "hermes"),
|
|
llm_provider=llm_provider,
|
|
llm_api_key=cfg.get("llmApiKey") or cfg.get("llm_api_key") or get_secret("HINDSIGHT_LLM_API_KEY", ""),
|
|
llm_model=cfg.get("llm_model", ""),
|
|
)
|
|
if self._llm_base_url:
|
|
kwargs["llm_base_url"] = self._llm_base_url
|
|
self._idle_timeout = self._int_setting(
|
|
"idle_timeout", "HINDSIGHT_IDLE_TIMEOUT", _DEFAULT_IDLE_TIMEOUT, env_default=self._idle_timeout,
|
|
)
|
|
kwargs["idle_timeout"] = self._idle_timeout
|
|
return HindsightEmbedded(**kwargs)
|
|
|
|
def _new_cloud_client(self):
|
|
_ensure_client_dependency()
|
|
from hindsight_client import Hindsight
|
|
kwargs = {"base_url": self._api_url, "timeout": float(self._timeout or _DEFAULT_TIMEOUT)}
|
|
if self._api_key:
|
|
kwargs["api_key"] = self._api_key
|
|
logger.debug("Creating Hindsight cloud client (url=%s, has_key=%s, timeout=%s)",
|
|
self._api_url, bool(self._api_key), kwargs["timeout"])
|
|
return Hindsight(**kwargs)
|
|
|
|
def _get_client(self):
|
|
"""Return the cached Hindsight client (created once, reused)."""
|
|
if self._client is None:
|
|
self._client = (
|
|
self._new_embedded_client() if self._mode == "local_embedded" else self._new_cloud_client()
|
|
)
|
|
return self._client
|
|
|
|
def _run_sync(self, coro):
|
|
"""Schedule *coro* on the shared loop using the configured timeout."""
|
|
return _run_sync(coro, timeout=self._timeout)
|
|
|
|
def _run_hindsight_operation(self, operation):
|
|
"""Run an async client operation; for local_embedded, a stale-daemon
|
|
connection failure recreates the client and retries once."""
|
|
client = self._get_client()
|
|
try:
|
|
return self._run_sync(operation(client))
|
|
except Exception as exc:
|
|
text = f"{type(exc).__name__}: {exc}".lower()
|
|
if self._mode != "local_embedded" or not any(m in text for m in _RETRIABLE_CONNECTION_MARKERS):
|
|
raise
|
|
logger.info(
|
|
"Hindsight embedded daemon appears unreachable; recreating client and retrying once: %s",
|
|
exc,
|
|
)
|
|
self._client = None
|
|
client = self._get_client()
|
|
self._client = client
|
|
return self._run_sync(operation(client))
|
|
|
|
# -- retain writer thread + server-side visibility -------------------------
|
|
|
|
def _ensure_writer(self) -> None:
|
|
"""Lazy-start the single retain-writer thread (providers that never
|
|
retain, e.g. tools-only mode, don't pay for an idle thread)."""
|
|
thread = self._writer_thread
|
|
if thread is not None and thread.is_alive():
|
|
return
|
|
# A previous writer may have exited after shutdown(); allow the fresh one to drain.
|
|
self._shutting_down.clear()
|
|
thread = _context_thread(self._writer_loop, "hindsight-writer")
|
|
self._writer_thread = self._sync_thread = thread
|
|
thread.start()
|
|
|
|
def _register_atexit(self) -> None:
|
|
"""Idempotent atexit drain so a CLI exit that skips
|
|
MemoryManager.shutdown_all() can't race interpreter teardown."""
|
|
if not self._atexit_registered:
|
|
self._atexit_registered = True
|
|
atexit.register(self._atexit_shutdown)
|
|
|
|
def _writer_loop(self) -> None:
|
|
"""Drain the retain queue serially; exits on the sentinel. A failing job
|
|
can't kill the writer, and task_done() always fires so queue.join() works."""
|
|
while True:
|
|
try:
|
|
job = self._retain_queue.get(timeout=1.0)
|
|
except queue.Empty:
|
|
if self._shutting_down.is_set():
|
|
return
|
|
continue
|
|
try:
|
|
if job is _WRITER_SENTINEL:
|
|
return
|
|
try:
|
|
job()
|
|
except Exception as exc:
|
|
logger.warning("Hindsight retain failed: %s", exc, exc_info=True)
|
|
finally:
|
|
self._retain_queue.task_done()
|
|
|
|
def _atexit_shutdown(self) -> None:
|
|
if self._shutting_down.is_set():
|
|
return
|
|
try:
|
|
self.shutdown()
|
|
except Exception as exc:
|
|
logger.debug("Hindsight atexit shutdown failed: %s", exc)
|
|
|
|
def _track_retain_ops(self, retain_response, bank_id: str) -> None:
|
|
"""Record the async ``operation_id``/``operation_ids`` from an
|
|
aretain_batch reply (pending server-side until recall-visible), with the
|
|
bank they were written to. No id (older API / synchronous completion)
|
|
means only the local queue drain is available as a signal."""
|
|
ids: list[str] = []
|
|
single = getattr(retain_response, "operation_id", None)
|
|
if single:
|
|
ids.append(str(single))
|
|
ids.extend(str(op) for op in (getattr(retain_response, "operation_ids", None) or []) if op)
|
|
if not ids:
|
|
return
|
|
self._retain_ops_bank_id = bank_id
|
|
with self._pending_retain_ops_lock:
|
|
self._pending_retain_ops.update(ids)
|
|
|
|
def _is_retain_op_complete(self, bank_id: str, op_id: str) -> bool:
|
|
"""True when a server-side retain op is done or gone. Completed ops are
|
|
evicted server-side, so NotFound (404) also means "no longer pending".
|
|
Transient errors return False so the caller keeps waiting."""
|
|
from hindsight_client_api.exceptions import NotFoundException
|
|
|
|
try:
|
|
resp = self._run_hindsight_operation(
|
|
lambda client: client.operations.get_operation_status(bank_id=bank_id, operation_id=op_id)
|
|
)
|
|
except NotFoundException:
|
|
return True
|
|
except Exception as exc:
|
|
logger.debug("Prefetch: operation status check failed for %s: %s", op_id, exc)
|
|
return False
|
|
return str(getattr(resp, "status", "") or "").lower() in {"completed", "failed"}
|
|
|
|
def _wait_for_retains_drained(self, timeout: float) -> bool:
|
|
"""Block up to *timeout* seconds for the just-completed turn's retain to
|
|
become recall-visible. Background prefetch thread only — never the reply path.
|
|
|
|
Two ordered barriers on one budget: (1) the local writer queue drains
|
|
(retain *dispatched*) — polls ``unfinished_tasks`` rather than
|
|
``queue.join()`` so a wedged write can't hang the prefetch; (2) the
|
|
server-side async ops complete (an explicit read-after-write signal,
|
|
since async retain returns on acceptance, not durability).
|
|
Returns False on timeout/shutdown.
|
|
"""
|
|
deadline = None if timeout <= 0 else time.monotonic() + timeout
|
|
while self._retain_queue.unfinished_tasks > 0:
|
|
if self._shutting_down.is_set():
|
|
return False
|
|
if deadline is not None and time.monotonic() >= deadline:
|
|
logger.debug("Prefetch: retain drain timed out after %.1fs (%d pending)",
|
|
timeout, self._retain_queue.unfinished_tasks)
|
|
return False
|
|
time.sleep(0.05)
|
|
return self._wait_for_server_retain_ops(deadline, timeout)
|
|
|
|
def _wait_for_server_retain_ops(self, deadline: float | None, timeout: float) -> bool:
|
|
"""Poll tracked async retain ops until complete or *deadline* (monotonic;
|
|
None = unbounded). Completed ops leave the pending set as they finish.
|
|
|
|
Ops still pending at the deadline are DROPPED: keeping them would let a
|
|
permanently failing status endpoint grow the pending set forever and burn
|
|
the full timeout on EVERY later prefetch (and, via prefetch()'s bounded
|
|
join, a per-turn reply-latency penalty). Dropping trades a possibly-stale
|
|
recall now for guaranteed liveness; logged at WARNING once per prefetch.
|
|
"""
|
|
def _expired() -> bool:
|
|
return deadline is not None and time.monotonic() >= deadline
|
|
|
|
while True:
|
|
with self._pending_retain_ops_lock:
|
|
bank_id = self._retain_ops_bank_id or self._bank_id
|
|
pending = list(self._pending_retain_ops)
|
|
if not pending:
|
|
return True
|
|
if self._shutting_down.is_set():
|
|
return False
|
|
done: set[str] = set()
|
|
for op_id in pending:
|
|
if self._shutting_down.is_set():
|
|
return False
|
|
if _expired():
|
|
break
|
|
if self._is_retain_op_complete(bank_id, op_id):
|
|
done.add(op_id)
|
|
with self._pending_retain_ops_lock:
|
|
self._pending_retain_ops.difference_update(done)
|
|
if not self._pending_retain_ops:
|
|
return True
|
|
dropped = len(self._pending_retain_ops) if _expired() else 0
|
|
if dropped:
|
|
self._pending_retain_ops.clear()
|
|
if dropped:
|
|
logger.warning(
|
|
"Prefetch: server retain visibility timed out after %.1fs; "
|
|
"dropping %d unresolved op(s) so later prefetches stay "
|
|
"bounded (recall may miss the just-completed turn)",
|
|
timeout, dropped,
|
|
)
|
|
return False
|
|
time.sleep(self._RETAIN_OP_POLL_INTERVAL_S)
|
|
|
|
# -- retain target -----------------------------------------------------------
|
|
|
|
def _probe_url(self) -> str:
|
|
"""URL to probe /version on: the running embedded client's dynamic
|
|
per-profile port when available, else the configured api_url."""
|
|
if self._mode == "local_embedded" and self._client is not None:
|
|
url = getattr(self._client, "url", None)
|
|
if url:
|
|
return str(url)
|
|
return self._api_url or ""
|
|
|
|
def _resolve_retain_target(self, fallback_document_id: str) -> tuple[str, str | None]:
|
|
"""Pick (document_id, update_mode) from live API capability.
|
|
|
|
Hindsight >= 0.5.0 supports ``update_mode='append'``, so the stable
|
|
session-scoped ``document_id`` can be reused across process lifecycles
|
|
without overwriting prior turns. Older APIs get *fallback_document_id*
|
|
(per-process unique, minted at initialize/switch time) and no
|
|
``update_mode`` — the only way the resume-overwrite fix works there.
|
|
The probe is cached per (process, api_url).
|
|
"""
|
|
if self._session_id and _check_api_supports_update_mode_append(self._probe_url(), self._api_key):
|
|
return self._session_id, "append"
|
|
return fallback_document_id, None
|
|
|
|
# -- lifecycle ---------------------------------------------------------------
|
|
|
|
def initialize(self, session_id: str, **kwargs) -> None:
|
|
self._session_id = str(session_id or "").strip()
|
|
self._parent_session_id = str(kwargs.get("parent_session_id", "") or "").strip()
|
|
# Status channel for the retain indicator (recall reports via recall_status()).
|
|
status_cb = kwargs.get("status_callback")
|
|
if callable(status_cb):
|
|
self._status_callback = status_cb
|
|
# session_id stays in tags so processes for one session remain filterable together.
|
|
self._document_id = _mint_document_id(self._session_id)
|
|
_maybe_upgrade_client()
|
|
|
|
self._config = cfg = _load_config()
|
|
for name in _SESSION_KWARGS:
|
|
setattr(self, f"_{name}", str(kwargs.get(name) or "").strip())
|
|
self._turn_index = 0
|
|
self._session_turns = []
|
|
self._last_retained_turn_count = 0
|
|
self._mode = cfg.get("mode", "cloud")
|
|
self._timeout = self._int_setting("timeout", "HINDSIGHT_TIMEOUT", _DEFAULT_TIMEOUT)
|
|
self._idle_timeout = self._int_setting("idle_timeout", "HINDSIGHT_IDLE_TIMEOUT", _DEFAULT_IDLE_TIMEOUT)
|
|
if self._mode == "local": # legacy alias
|
|
self._mode = "local_embedded"
|
|
if self._mode == "local_embedded":
|
|
# Must precede the daemon_embed_manager import, which reads it at import time.
|
|
_export_port_health_grace_timeout(cfg)
|
|
available, reason = _check_local_runtime()
|
|
if not available:
|
|
logger.warning(
|
|
"Hindsight local mode disabled because its runtime could not be imported: %s.%s",
|
|
reason, _local_runtime_hint(reason),
|
|
)
|
|
self._mode = "disabled"
|
|
return
|
|
self._api_key = _cloud_api_key(cfg)
|
|
default_url = _DEFAULT_LOCAL_URL if self._mode in {"local_embedded", "local_external"} else _DEFAULT_API_URL
|
|
self._api_url = cfg.get("api_url") or os.environ.get("HINDSIGHT_API_URL", default_url)
|
|
self._llm_base_url = cfg.get("llm_base_url", "")
|
|
|
|
banks = cfg_get(cfg, "banks", "hermes", default={})
|
|
self._bank_id_template = cfg.get("bank_id_template", "") or ""
|
|
self._bank_id = _resolve_bank_id_template(
|
|
self._bank_id_template,
|
|
fallback=cfg.get("bank_id") or banks.get("bankId", "hermes"),
|
|
profile=self._agent_identity,
|
|
workspace=self._agent_workspace,
|
|
platform=self._platform,
|
|
user=self._user_id,
|
|
session=self._session_id,
|
|
)
|
|
budget = cfg.get("recall_budget") or cfg.get("budget") or banks.get("budget", "mid")
|
|
self._budget = budget if budget in _VALID_BUDGETS else "mid"
|
|
memory_mode = cfg.get("memory_mode", "hybrid")
|
|
self._memory_mode = memory_mode if memory_mode in _SYSTEM_PROMPT_TAILS else "hybrid"
|
|
prefetch_method = cfg.get("recall_prefetch_method") or cfg.get("prefetch_method", "recall")
|
|
self._prefetch_method = prefetch_method if prefetch_method in {"recall", "reflect"} else "recall"
|
|
|
|
self._bank_mission = cfg.get("bank_mission", "")
|
|
self._bank_retain_mission = cfg.get("bank_retain_mission") or None
|
|
|
|
self._retain_tags = _normalize_retain_tags(
|
|
cfg.get("retain_tags") or os.environ.get("HINDSIGHT_RETAIN_TAGS", "")
|
|
)
|
|
self._tags = self._retain_tags or None
|
|
self._observation_scopes = _normalize_observation_scopes(
|
|
cfg.get("observation_scopes") or os.environ.get("HINDSIGHT_RETAIN_OBSERVATION_SCOPES", "")
|
|
)
|
|
self._recall_tags = cfg.get("recall_tags") or None
|
|
self._recall_tags_match = cfg.get("recall_tags_match", "any")
|
|
self._retain_source = str(
|
|
cfg.get("retain_source") or os.environ.get("HINDSIGHT_RETAIN_SOURCE", _DEFAULT_RETAIN_SOURCE)
|
|
).strip()
|
|
self._retain_user_prefix = str(
|
|
cfg.get("retain_user_prefix") or os.environ.get("HINDSIGHT_RETAIN_USER_PREFIX", "User")
|
|
).strip() or "User"
|
|
self._retain_assistant_prefix = str(
|
|
cfg.get("retain_assistant_prefix") or os.environ.get("HINDSIGHT_RETAIN_ASSISTANT_PREFIX", "Assistant")
|
|
).strip() or "Assistant"
|
|
|
|
self._auto_retain = cfg.get("auto_retain", True)
|
|
self._retain_every_n_turns = max(1, int(cfg.get("retain_every_n_turns", 1)))
|
|
self._retain_context = cfg.get("retain_context", _RETAIN_CONTEXT_DEFAULT)
|
|
|
|
self._auto_recall = cfg.get("auto_recall", True)
|
|
self._recall_sync = bool(cfg.get("recall_sync", False))
|
|
self._recall_max_tokens = int(cfg.get("recall_max_tokens", 4096))
|
|
# None -> observation-only; a comma-separated string is accepted for
|
|
# parity with recall_tags; an explicit list broadens or disables the filter.
|
|
configured_types = cfg.get("recall_types")
|
|
if configured_types is None:
|
|
self._recall_types = ["observation"]
|
|
elif isinstance(configured_types, str):
|
|
self._recall_types = [t.strip() for t in configured_types.split(",") if t.strip()]
|
|
else:
|
|
self._recall_types = list(configured_types) or ["observation"]
|
|
self._recall_prompt_preamble = cfg.get("recall_prompt_preamble", "")
|
|
# On by default so the user SEES memory working regardless of whether the
|
|
# model mentions it; off switch for customer-facing agents.
|
|
self._recall_indicator = bool(cfg.get("recall_indicator", True))
|
|
self._retain_indicator = bool(cfg.get("retain_indicator", True))
|
|
self._recall_max_input_chars = int(cfg.get("recall_max_input_chars", 800))
|
|
self._retain_async = cfg.get("retain_async", True)
|
|
self._prefetch_waits_for_retain = cfg.get("prefetch_waits_for_retain", True)
|
|
self._prefetch_retain_drain_timeout = float(cfg.get("prefetch_retain_drain_timeout", 10.0))
|
|
|
|
client_version = "unknown"
|
|
try:
|
|
from importlib.metadata import version as pkg_version
|
|
client_version = pkg_version("hindsight-client")
|
|
except Exception:
|
|
pass
|
|
logger.info("Hindsight initialized: mode=%s, api_url=%s, bank=%s, budget=%s, memory_mode=%s, prefetch_method=%s, client=%s",
|
|
self._mode, self._api_url, self._bank_id, self._budget, self._memory_mode, self._prefetch_method, client_version)
|
|
if self._bank_id_template:
|
|
logger.debug("Hindsight bank resolved from template %r: profile=%s workspace=%s platform=%s user=%s -> bank=%s",
|
|
self._bank_id_template, self._agent_identity, self._agent_workspace,
|
|
self._platform, self._user_id, self._bank_id)
|
|
logger.debug("Hindsight config: auto_retain=%s, auto_recall=%s, retain_every_n=%d, "
|
|
"retain_async=%s, retain_context=%s, recall_max_tokens=%d, recall_max_input_chars=%d, tags=%s, recall_tags=%s",
|
|
self._auto_retain, self._auto_recall, self._retain_every_n_turns,
|
|
self._retain_async, self._retain_context, self._recall_max_tokens, self._recall_max_input_chars,
|
|
self._tags, self._recall_tags)
|
|
|
|
if self._mode == "local_embedded":
|
|
self._start_embedded_daemon()
|
|
|
|
def _start_embedded_daemon(self) -> None:
|
|
"""Start the embedded daemon on a background thread so it doesn't block
|
|
the chat; its Rich output goes to a log file, not the terminal."""
|
|
# PostgreSQL's initdb refuses root, so the daemon can never initialize
|
|
# its data directory under root. Without this guard the start thread
|
|
# fails, retries and loops forever, reloading embedding models (~958MB
|
|
# RAM, ~33% CPU) with no user-visible error.
|
|
if hasattr(os, "geteuid") and os.geteuid() == 0:
|
|
msg = (
|
|
"Hindsight local_embedded mode cannot run as root "
|
|
"(PostgreSQL initdb refuses root). Skipping the embedded "
|
|
"memory daemon. Run Hermes as a non-root user, or switch "
|
|
"to cloud / local_external mode via 'hermes memory setup'."
|
|
)
|
|
logger.warning(msg)
|
|
# Also print: a daemon that never starts would otherwise fail
|
|
# silently and the user would only see Hermes get sluggish.
|
|
try:
|
|
print(f" ⚠ {msg}", file=sys.stderr, flush=True)
|
|
except Exception:
|
|
pass
|
|
self._mode = "disabled"
|
|
return
|
|
_context_thread(self._daemon_start_worker, "hindsight-daemon-start").start()
|
|
|
|
def _daemon_start_worker(self) -> None:
|
|
import traceback
|
|
log_dir = get_hermes_home() / "logs"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
log_path = log_dir / "hindsight-embed.log"
|
|
|
|
def _log(text: str, exc: bool = False) -> None:
|
|
with open(log_path, "a", encoding="utf-8") as f:
|
|
f.write(text)
|
|
if exc:
|
|
traceback.print_exc(file=f)
|
|
|
|
try:
|
|
# Point the daemon manager's Rich console at our log file rather
|
|
# than redirecting global fds (which would capture other threads).
|
|
import hindsight_embed.daemon_embed_manager as dem
|
|
from rich.console import Console
|
|
dem.console = Console(file=open(log_path, "a", encoding="utf-8"), force_terminal=False)
|
|
|
|
client = self._get_client()
|
|
profile = self._config.get("profile", "hermes")
|
|
# Keep the profile .env in sync with our config; if it changed and
|
|
# the daemon is running, restart it with the new settings.
|
|
if _load_simple_env(_embedded_profile_env_path(self._config)) != _build_embedded_profile_env(self._config):
|
|
_materialize_embedded_profile_env(self._config)
|
|
if client._manager.is_running(profile):
|
|
_log("\n=== Config changed, restarting daemon ===\n")
|
|
client._manager.stop(profile)
|
|
client._ensure_started()
|
|
_log("\n=== Daemon started successfully ===\n")
|
|
except Exception as e:
|
|
_log(f"\n=== Daemon startup failed: {e} ===\n", exc=True)
|
|
|
|
def system_prompt_block(self) -> str:
|
|
mode = self._memory_mode if self._memory_mode in ("context", "tools") else "hybrid"
|
|
label = "" if mode == "hybrid" else f" ({mode} mode)"
|
|
return (
|
|
f"# Hindsight Memory\n"
|
|
f"Active{label}. Bank: {self._bank_id}, budget: {self._budget}.\n"
|
|
f"{_SYSTEM_PROMPT_TAILS[mode]}"
|
|
)
|
|
|
|
# -- recall ------------------------------------------------------------------
|
|
|
|
def _recall_disabled(self) -> bool:
|
|
"""Guards shared by the async and synchronous recall paths."""
|
|
for skip, why in (
|
|
(self._memory_mode == "tools", "tools-only mode"),
|
|
(not self._auto_recall, "auto_recall disabled"),
|
|
(self._shutting_down.is_set(), "shutting down"),
|
|
):
|
|
if skip:
|
|
logger.debug("Prefetch: skipped (%s)", why)
|
|
return True
|
|
return False
|
|
|
|
def _recall_kwargs(self, query: str) -> dict:
|
|
kwargs: dict = {
|
|
"bank_id": self._bank_id, "query": query,
|
|
"budget": self._budget, "max_tokens": self._recall_max_tokens,
|
|
}
|
|
if self._recall_tags:
|
|
kwargs["tags"] = self._recall_tags
|
|
kwargs["tags_match"] = self._recall_tags_match
|
|
if self._recall_types:
|
|
kwargs["types"] = self._recall_types
|
|
return kwargs
|
|
|
|
def _do_recall(self, query: str) -> _RecallResult:
|
|
"""One recall/reflect for *query*. Shared by the background prefetch
|
|
worker and the opt-in synchronous path (``recall_sync``)."""
|
|
if self._recall_max_input_chars and len(query) > self._recall_max_input_chars:
|
|
query = query[:self._recall_max_input_chars]
|
|
try:
|
|
if self._prefetch_method == "reflect":
|
|
logger.debug("Recall: calling reflect (bank=%s, query_len=%d)", self._bank_id, len(query))
|
|
resp = self._run_hindsight_operation(lambda client: client.areflect(bank_id=self._bank_id, query=query, budget=self._budget))
|
|
return _RecallResult(resp.text or "", 0) # synthesis -> no discrete count
|
|
recall_kwargs = self._recall_kwargs(query)
|
|
logger.debug("Recall: calling recall (bank=%s, query_len=%d, budget=%s)",
|
|
self._bank_id, len(query), self._budget)
|
|
resp = self._run_hindsight_operation(lambda client: client.arecall(**recall_kwargs))
|
|
results = resp.results or []
|
|
logger.debug("Recall: returned %d results", len(results))
|
|
return _RecallResult("\n".join(f"- {r.text}" for r in results if r.text), len(results))
|
|
except Exception as e:
|
|
logger.debug("Hindsight recall failed: %s", e, exc_info=True)
|
|
return _RecallResult("", 0)
|
|
|
|
def _finish_prefetch(self, result: str, count: int) -> str:
|
|
"""Record indicator state (cleared on empty turns so it never reports a
|
|
stale count) and format the injected block."""
|
|
self._last_recall_returned = bool(result)
|
|
self._last_recall_count = count if result else 0
|
|
if not result:
|
|
logger.debug("Prefetch: no results available")
|
|
return ""
|
|
logger.debug("Prefetch: returning %d chars of context", len(result))
|
|
header = self._recall_prompt_preamble or (
|
|
"# Hindsight Memory (persistent cross-session context)\n"
|
|
"Use this to answer questions about the user and prior sessions. "
|
|
"Do not call tools to look up information that is already present here."
|
|
)
|
|
return f"{header}\n\n{result}"
|
|
|
|
def prefetch(self, query: str, *, session_id: str = "") -> str:
|
|
# Opt-in: recall synchronously against the *current* message so the
|
|
# injected memories match this turn's query, not the previous turn's.
|
|
if self._recall_sync:
|
|
if self._recall_disabled():
|
|
return self._finish_prefetch("", 0)
|
|
recalled = self._do_recall(query)
|
|
return self._finish_prefetch(recalled.text, recalled.count)
|
|
# Default: the background worker's result for the previous turn (capped join).
|
|
if self._prefetch_thread and self._prefetch_thread.is_alive():
|
|
logger.debug("Prefetch: waiting for background thread to complete")
|
|
self._prefetch_thread.join(timeout=3.0)
|
|
with self._prefetch_lock:
|
|
result, count = self._prefetch_result, self._prefetch_count
|
|
self._prefetch_result, self._prefetch_count = "", 0
|
|
return self._finish_prefetch(result, count)
|
|
|
|
def recall_status(self) -> Optional[RecallStatus]:
|
|
"""Count injected by the last prefetch, or None when nothing was injected
|
|
or ``recall_indicator=false`` (customer-facing agents)."""
|
|
if not self._recall_indicator or not self._last_recall_returned:
|
|
return None
|
|
return RecallStatus(provider_label="Hindsight", count=self._last_recall_count, glyph=_HINDSIGHT_GLYPH)
|
|
|
|
def queue_prefetch(self, query: str, *, session_id: str = "") -> None:
|
|
# Sync mode recalls live each turn — nothing to prime in the background.
|
|
if self._recall_sync or self._recall_disabled():
|
|
return
|
|
|
|
def _run():
|
|
# Wait (bounded, off the reply path) for the just-completed turn's
|
|
# retain to be recall-visible so the warmed context includes it.
|
|
if self._prefetch_waits_for_retain:
|
|
self._wait_for_retains_drained(self._prefetch_retain_drain_timeout)
|
|
recalled = self._do_recall(query)
|
|
if recalled.text:
|
|
with self._prefetch_lock:
|
|
self._prefetch_result = recalled.text
|
|
self._prefetch_count = recalled.count
|
|
|
|
self._prefetch_thread = _context_thread(_run, "hindsight-prefetch")
|
|
self._prefetch_thread.start()
|
|
|
|
# -- retain ------------------------------------------------------------------
|
|
|
|
def _build_turn_messages(self, user_content: str, assistant_content: str) -> List[Dict[str, str]]:
|
|
# One conversation turn -> both messages share the turn-level event timestamp.
|
|
now = _event_timestamp()
|
|
return [
|
|
{"role": "user", "content": f"{self._retain_user_prefix}: {user_content}", "timestamp": now},
|
|
{"role": "assistant", "content": f"{self._retain_assistant_prefix}: {assistant_content}", "timestamp": now},
|
|
]
|
|
|
|
def _build_metadata(self, *, message_count: int, turn_index: int) -> Dict[str, str]:
|
|
metadata: Dict[str, str] = {
|
|
"retained_at": _utc_timestamp(),
|
|
"message_count": str(message_count),
|
|
"turn_index": str(turn_index),
|
|
}
|
|
if self._retain_source:
|
|
metadata["source"] = self._retain_source
|
|
for name in _METADATA_ATTRS:
|
|
value = getattr(self, f"_{name}")
|
|
if value:
|
|
metadata[name] = value
|
|
return metadata
|
|
|
|
def _build_retain_kwargs(
|
|
self,
|
|
content: str,
|
|
*,
|
|
context: str | None = None,
|
|
metadata: Dict[str, str] | None = None,
|
|
tags: List[str] | None = None,
|
|
occurred_at: str | None = None,
|
|
update_mode: str | None = None,
|
|
) -> Dict[str, Any]:
|
|
"""Build one aretain_batch item. The item timestamp is what the server
|
|
uses to resolve occurred_start/occurred_end (incl. relative phrases in
|
|
content): an explicit occurred_at wins, else the configured event clock."""
|
|
item: Dict[str, Any] = {
|
|
"content": content,
|
|
"metadata": metadata or self._build_metadata(message_count=1, turn_index=self._turn_index),
|
|
"timestamp": (occurred_at or "").strip() or _event_timestamp(),
|
|
}
|
|
if context is not None:
|
|
item["context"] = context
|
|
if update_mode is not None:
|
|
item["update_mode"] = update_mode
|
|
merged_tags = _normalize_retain_tags(list(self._retain_tags) + _normalize_retain_tags(tags))
|
|
if merged_tags:
|
|
item["tags"] = merged_tags
|
|
if self._observation_scopes:
|
|
item["observation_scopes"] = self._observation_scopes
|
|
return item
|
|
|
|
def _lineage_tags(self) -> list[str]:
|
|
tags = []
|
|
if self._session_id:
|
|
tags.append(f"session:{self._session_id}")
|
|
if self._parent_session_id:
|
|
tags.append(f"parent:{self._parent_session_id}")
|
|
return tags
|
|
|
|
def _retain_batch(self, item: dict, *, bank_id: str, document_id: str | None = None,
|
|
retain_async: bool | None = None):
|
|
"""Dispatch one item via aretain_batch (bank_id/document_id/retain_async are
|
|
call-level args, never item keys)."""
|
|
kwargs: Dict[str, Any] = {"bank_id": bank_id, "items": [item]}
|
|
if document_id is not None:
|
|
kwargs["document_id"] = document_id
|
|
if retain_async is not None:
|
|
kwargs["retain_async"] = retain_async
|
|
return self._run_hindsight_operation(lambda client: client.aretain_batch(**kwargs))
|
|
|
|
def _make_turn_retain_job(self, turns: list[str], *, document_id: str, update_mode: str | None,
|
|
label: str, track_ops: bool = True) -> Callable[[], None]:
|
|
"""Build the writer job that ships *turns* as one document. Every input is
|
|
snapshotted NOW because the writer runs after later sync_turn() calls have
|
|
mutated _session_turns / _turn_index / _session_id."""
|
|
content = "[" + ",".join(turns) + "]"
|
|
metadata = self._build_metadata(message_count=len(turns) * 2, turn_index=self._turn_index)
|
|
tags = self._lineage_tags() or None
|
|
bank_id, retain_async, retain_context = self._bank_id, self._retain_async, self._retain_context
|
|
|
|
def _job() -> None:
|
|
item = self._build_retain_kwargs(content, context=retain_context, metadata=metadata,
|
|
tags=tags, update_mode=update_mode)
|
|
logger.debug("Hindsight %s: bank=%s, doc=%s, mode=%s, async=%s, content_len=%d, num_turns=%d",
|
|
label, bank_id, document_id, update_mode, retain_async, len(content), len(turns))
|
|
resp = self._retain_batch(item, bank_id=bank_id, document_id=document_id, retain_async=retain_async)
|
|
# Async retains are only *accepted* here; track the op id(s) so the
|
|
# next-turn prefetch can wait for true server-side completion.
|
|
if retain_async and track_ops:
|
|
self._track_retain_ops(resp, bank_id)
|
|
logger.debug("Hindsight %s succeeded", label)
|
|
|
|
return _job
|
|
|
|
def sync_turn(self, user_content: str, assistant_content: str, *, session_id: str = "") -> None:
|
|
"""Enqueue a retain for the current turn (non-blocking; runs on the writer
|
|
thread). Dropped once shutdown() has fired so post-exit retains never
|
|
reach aiohttp during interpreter teardown."""
|
|
if not self._auto_retain:
|
|
logger.debug("sync_turn: skipped (auto_retain disabled)")
|
|
return
|
|
if self._shutting_down.is_set():
|
|
logger.debug("sync_turn: skipped (shutting down)")
|
|
return
|
|
if session_id:
|
|
self._session_id = str(session_id).strip()
|
|
|
|
self._session_turns.append(json.dumps(self._build_turn_messages(user_content, assistant_content), ensure_ascii=False))
|
|
self._turn_counter += 1
|
|
self._turn_index = self._turn_counter
|
|
remainder = self._turn_counter % self._retain_every_n_turns
|
|
if remainder:
|
|
logger.debug("sync_turn: buffered turn %d (will retain at turn %d)",
|
|
self._turn_counter, self._turn_counter + (self._retain_every_n_turns - remainder))
|
|
return
|
|
|
|
document_id, update_mode = self._resolve_retain_target(self._document_id)
|
|
# Append-capable APIs get only the delta since the last retain; legacy /
|
|
# overwrite APIs need the whole session because each retain replaces the document.
|
|
if update_mode == "append":
|
|
turns_to_retain = self._session_turns[self._last_retained_turn_count:]
|
|
if not turns_to_retain:
|
|
logger.debug("sync_turn: skipped append retain; no new turns since last retain")
|
|
return
|
|
else:
|
|
turns_to_retain = list(self._session_turns)
|
|
logger.debug("sync_turn: retaining %d/%d turns, payload %d chars",
|
|
len(turns_to_retain), len(self._session_turns), sum(len(t) for t in turns_to_retain))
|
|
|
|
job = self._make_turn_retain_job(turns_to_retain, document_id=document_id,
|
|
update_mode=update_mode, label="retain")
|
|
self._ensure_writer()
|
|
self._register_atexit()
|
|
# Emitted only once every skip/buffer gate above has passed, so it fires
|
|
# solely on turns that actually persist.
|
|
self._emit_saving_indicator()
|
|
self._retain_queue.put(job)
|
|
# Advance the watermark only after the delta is queued so a later retain
|
|
# doesn't re-ship turns already handed to the writer.
|
|
if update_mode == "append":
|
|
self._last_retained_turn_count = len(self._session_turns)
|
|
|
|
def _emit_saving_indicator(self) -> None:
|
|
"""Model-independent "saving to memory" status line. No-op when
|
|
``retain_indicator=false`` or no status channel; never raises."""
|
|
if not self._retain_indicator or self._status_callback is None:
|
|
return
|
|
try:
|
|
self._status_callback(f"{_HINDSIGHT_GLYPH} Hindsight — saving to memory…")
|
|
except Exception:
|
|
logger.debug("Retain indicator emit failed (non-fatal)", exc_info=True)
|
|
|
|
# -- tools -------------------------------------------------------------------
|
|
|
|
def get_tool_schemas(self) -> List[Dict[str, Any]]:
|
|
if self._memory_mode == "context":
|
|
return []
|
|
return [RETAIN_SCHEMA, RECALL_SCHEMA, REFLECT_SCHEMA]
|
|
|
|
def _tool_retain(self, args: dict) -> str:
|
|
content = args["content"]
|
|
context = args.get("context")
|
|
item = self._build_retain_kwargs(content, context=context, tags=args.get("tags"),
|
|
occurred_at=args.get("occurred_at"))
|
|
logger.debug("Tool hindsight_retain: bank=%s, content_len=%d, context=%s",
|
|
self._bank_id, len(content), context)
|
|
self._retain_batch(item, bank_id=self._bank_id)
|
|
logger.debug("Tool hindsight_retain: success")
|
|
return "Memory stored successfully."
|
|
|
|
def _tool_recall(self, args: dict) -> str:
|
|
query = args["query"]
|
|
recall_kwargs = self._recall_kwargs(query)
|
|
logger.debug("Tool hindsight_recall: bank=%s, query_len=%d, budget=%s",
|
|
self._bank_id, len(query), self._budget)
|
|
resp = self._run_hindsight_operation(lambda client: client.arecall(**recall_kwargs))
|
|
results = resp.results or []
|
|
logger.debug("Tool hindsight_recall: %d results", len(results))
|
|
if not results:
|
|
return "No relevant memories found."
|
|
return "\n".join(f"{i}. {r.text}" for i, r in enumerate(results, 1))
|
|
|
|
def _tool_reflect(self, args: dict) -> str:
|
|
query = args["query"]
|
|
logger.debug("Tool hindsight_reflect: bank=%s, query_len=%d, budget=%s",
|
|
self._bank_id, len(query), self._budget)
|
|
resp = self._run_hindsight_operation(
|
|
lambda client: client.areflect(bank_id=self._bank_id, query=query, budget=self._budget)
|
|
)
|
|
logger.debug("Tool hindsight_reflect: response_len=%d", len(resp.text or ""))
|
|
return resp.text or "No relevant memories found."
|
|
|
|
# tool name -> (required arg, handler)
|
|
_TOOL_HANDLERS = {
|
|
"hindsight_retain": ("content", _tool_retain),
|
|
"hindsight_recall": ("query", _tool_recall),
|
|
"hindsight_reflect": ("query", _tool_reflect),
|
|
}
|
|
|
|
def handle_tool_call(self, tool_name: str, args: dict, **kwargs) -> str:
|
|
entry = self._TOOL_HANDLERS.get(tool_name)
|
|
if entry is None:
|
|
return tool_error(f"Unknown tool: {tool_name}")
|
|
required, handler = entry
|
|
if not args.get(required, ""):
|
|
return tool_error(f"Missing required parameter: {required}")
|
|
try:
|
|
return json.dumps({"result": handler(self, args)})
|
|
except Exception as e:
|
|
logger.warning("%s failed: %s", tool_name, e, exc_info=True)
|
|
return tool_error(f"{_TOOL_ERRORS[tool_name]}: {e}")
|
|
|
|
# -- session lifecycle -------------------------------------------------------
|
|
|
|
def on_session_switch(
|
|
self,
|
|
new_session_id: str,
|
|
*,
|
|
parent_session_id: str = "",
|
|
reset: bool = False,
|
|
**kwargs,
|
|
) -> None:
|
|
"""Refresh per-session state when the agent rotates session_id (/resume,
|
|
/branch, /reset, /new, context compression); otherwise writes land in
|
|
the previous session's document.
|
|
|
|
Always: update ``_session_id`` (metadata/tags), mint a fresh
|
|
``_document_id`` (so the new session can't overwrite the old document),
|
|
and clear the batch buffers — even for /resume and /branch, batching must
|
|
restart from zero so an in-flight retain doesn't flush under the wrong
|
|
document. Before clearing, flush buffered turns under the OLD ids
|
|
(``retain_every_n_turns > 1`` would otherwise silently lose them), and
|
|
join any in-flight prefetch and drop its cached result so the new
|
|
session's first ``prefetch()`` can't read stale recall. ``reset`` is
|
|
accepted but unneeded: buffer clearing is correct for every switch.
|
|
"""
|
|
new_id = str(new_session_id or "").strip()
|
|
if not new_id:
|
|
return
|
|
|
|
# 1. Flush buffered turns under the OLD identifiers, resolved BEFORE the
|
|
# rotation so the flush lands in the old session's document either way
|
|
# (legacy: per-process unique; >=0.5.0: stable session-scoped + append).
|
|
if self._session_turns:
|
|
old_document_id, old_update_mode = self._resolve_retain_target(self._document_id)
|
|
job = self._make_turn_retain_job(list(self._session_turns), document_id=old_document_id,
|
|
update_mode=old_update_mode, label="flush-on-switch",
|
|
track_ops=False)
|
|
|
|
def _flush():
|
|
try:
|
|
job()
|
|
except Exception as e:
|
|
logger.warning("Hindsight flush-on-switch failed: %s", e, exc_info=True)
|
|
|
|
# Same writer queue as sync_turn: FIFO behind still-queued old-session
|
|
# retains, no two threads racing aretain_batch on one document, and
|
|
# shutdown's drain semantics intact. Skip once shutdown has fired.
|
|
if not self._shutting_down.is_set():
|
|
self._ensure_writer()
|
|
self._register_atexit()
|
|
self._retain_queue.put(_flush)
|
|
|
|
# 2. Drain the old session's in-flight prefetch and drop its result.
|
|
if self._prefetch_thread and self._prefetch_thread.is_alive():
|
|
self._prefetch_thread.join(timeout=3.0)
|
|
with self._prefetch_lock:
|
|
self._prefetch_result = ""
|
|
|
|
# 3. Rotate to the new session.
|
|
if parent_session_id:
|
|
self._parent_session_id = str(parent_session_id).strip()
|
|
self._session_id = new_id
|
|
self._document_id = _mint_document_id(new_id)
|
|
self._session_turns = []
|
|
self._turn_counter = 0
|
|
self._turn_index = 0
|
|
self._last_retained_turn_count = 0
|
|
logger.debug(
|
|
"Hindsight on_session_switch: new_session=%s parent=%s reset=%s doc=%s",
|
|
self._session_id, self._parent_session_id, reset, self._document_id,
|
|
)
|
|
|
|
def shutdown(self) -> None:
|
|
logger.debug("Hindsight shutdown: stopping writer + waiting for background threads")
|
|
# Stop accepting retain jobs first so late sync_turn() calls are dropped.
|
|
self._shutting_down.set()
|
|
# The writer finishes in-flight work then exits on the sentinel; the
|
|
# bounded join keeps shutdown predictable even if the daemon is wedged.
|
|
writer = self._writer_thread
|
|
if writer is not None and writer.is_alive():
|
|
try:
|
|
self._retain_queue.put(_WRITER_SENTINEL)
|
|
except Exception:
|
|
pass
|
|
writer.join(timeout=10.0)
|
|
if writer.is_alive():
|
|
logger.warning("Hindsight writer did not stop within 10s; abandoning %d pending retain(s)",
|
|
self._retain_queue.qsize())
|
|
if self._prefetch_thread and self._prefetch_thread.is_alive():
|
|
self._prefetch_thread.join(timeout=5.0)
|
|
if self._client is not None:
|
|
try:
|
|
if self._mode == "local_embedded":
|
|
# HindsightEmbedded.close() closes its sync client from this
|
|
# thread, which can raise "attached to a different loop" before
|
|
# aiohttp releases the session. Close the inner async client on
|
|
# the shared loop first, then let the wrapper clean up bookkeeping.
|
|
inner_client = getattr(self._client, "_client", None)
|
|
if inner_client is not None and hasattr(inner_client, "aclose"):
|
|
_run_sync(inner_client.aclose())
|
|
try:
|
|
self._client._client = None
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self._client.close()
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
self._run_sync(self._client.aclose())
|
|
except Exception:
|
|
pass
|
|
self._client = None
|
|
# The module-global loop (_loop / _loop_thread) is intentionally NOT
|
|
# stopped: it's shared by every provider in the process (one per AIAgent,
|
|
# one AIAgent per gateway chat session). Stopping it would strand sibling
|
|
# providers' aiohttp sessions on a dead loop ("Unclosed client session").
|
|
# It runs on a daemon thread and is reclaimed at process exit.
|
|
|
|
|
|
def register(ctx) -> None:
|
|
"""Register Hindsight as a memory provider plugin."""
|
|
ctx.register_memory_provider(HindsightMemoryProvider())
|