From 49d15faae6805052226013effcadc1d9f7ab5653 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:55:33 -0700 Subject: [PATCH] refactor(tools/skills_sync,memory): split skills_sync client wire/org and bundled/optional ops; extract memory_tool_store and session_search common/discover --- tools/memory_tool.py | 1256 ++------------ tools/memory_tool_store.py | 543 ++++++ tools/session_search_tool.py | 1160 +++---------- tools/session_search_tool_common.py | 230 +++ tools/session_search_tool_discover.py | 192 +++ tools/skills_sync.py | 1590 ++++-------------- tools/skills_sync_bundled_ops.py | 240 +++ tools/skills_sync_client.py | 2193 ++++--------------------- tools/skills_sync_client_org.py | 381 +++++ tools/skills_sync_client_wire.py | 439 +++++ tools/skills_sync_optional.py | 262 +++ 11 files changed, 3291 insertions(+), 5195 deletions(-) create mode 100644 tools/memory_tool_store.py create mode 100644 tools/session_search_tool_common.py create mode 100644 tools/session_search_tool_discover.py create mode 100644 tools/skills_sync_bundled_ops.py create mode 100644 tools/skills_sync_client_org.py create mode 100644 tools/skills_sync_client_wire.py create mode 100644 tools/skills_sync_optional.py diff --git a/tools/memory_tool.py b/tools/memory_tool.py index 54dcaa71a6..b4e0c7bd98 100644 --- a/tools/memory_tool.py +++ b/tools/memory_tool.py @@ -1,42 +1,23 @@ #!/usr/bin/env python3 -""" -Memory Tool Module - Persistent Curated Memory - -Provides bounded, file-backed memory that persists across sessions. Two stores: - - MEMORY.md: agent's personal notes and observations (environment facts, project - conventions, tool quirks, things learned) - - USER.md: what the agent knows about the user (preferences, communication style, - expectations, workflow habits) - -Both are injected into the system prompt as a frozen snapshot at session start. -Mid-session writes update files on disk immediately (durable) but do NOT change -the system prompt -- this preserves the prefix cache for the entire session. -The snapshot refreshes on the next session start. - -Entry delimiter: § (section sign). Entries can be multiline. -Character limits (not tokens) because char counts are model-independent. - -Design: -- Single `memory` tool with action parameter: add, replace, remove -- replace/remove use short unique substring matching (not full text or IDs) -- Behavioral guidance lives in the tool schema description -- Frozen snapshot pattern: system prompt is stable, tool responses show live state -""" +"""Memory Tool - persistent curated memory (MEMORY.md = agent notes, USER.md = +user profile). Both enter the system prompt as a FROZEN snapshot at session +start; mid-session writes hit disk immediately but never change the prompt +(prefix cache stays intact). Single `memory` tool: add/replace/remove or a +batch `operations` list. The store lives in ``tools.memory_tool_store``.""" import copy import json import logging -import time -from contextlib import contextmanager from contextvars import ContextVar from pathlib import Path from hermes_constants import get_hermes_home from typing import Dict, Any, List, Optional, Tuple -from utils import atomic_write_text, is_truthy_value +from utils import is_truthy_value from tools.registry import no_cache_check_fn -# fcntl is Unix-only; on Windows use msvcrt for file locking +# fcntl is Unix-only; on Windows use msvcrt for file locking. MemoryStore reads +# these lazily from this module (tests inspect ``memory_tool.fcntl``). msvcrt = None try: import fcntl @@ -51,972 +32,93 @@ logger = logging.getLogger(__name__) # One tool-definition pass must use one config decision for both availability # and the dynamic target schema. ContextVar keeps concurrent profile/session -# builds isolated while allowing the check_fn result to flow to the immediately +# builds isolated while letting the check_fn result flow to the immediately # following dynamic_schema_overrides call in ToolRegistry.get_definitions(). _memory_surface_flags: ContextVar[Optional[Tuple[bool, bool]]] = ContextVar( "memory_surface_flags", default=None ) -# Where memory files live — resolved dynamically so profile overrides -# (HERMES_HOME env var changes) are always respected. The old module-level -# constant was cached at import time and could go stale if a profile switch -# happened after the first import. + def get_memory_dir() -> Path: - """Return the profile-scoped memories directory.""" + """Return the profile-scoped memories directory (resolved per call so + HERMES_HOME/profile switches after import are respected).""" return get_hermes_home() / "memories" -# Stable header prefixes for the system-prompt memory blocks rendered by -# MemoryStore._render_block. Exported so compression's prompt-retention check -# (agent/conversation_compression.py) can detect a leftover block for a -# target whose entries have since been emptied — keep in lockstep with -# _render_block below. -MEMORY_BLOCK_HEADERS = { - "memory": "MEMORY (your personal notes)", - "user": "USER PROFILE (who the user is)", -} -ENTRY_DELIMITER = "\n§\n" - - -# --------------------------------------------------------------------------- -# Memory content scanning — lightweight check for injection/exfiltration -# in content that gets injected into the system prompt. -# -# Patterns live in ``tools/threat_patterns.py`` — the single source of truth -# shared with the context-file scanner and the tool-result delimiter system. -# Memory uses the "strict" scope (broadest pattern set) because: -# - memory entries are user-curated; the user can rewrite a flagged entry -# - memory enters the system prompt as a FROZEN snapshot, so a poisoned -# entry persists for the entire session and across sessions until -# explicitly removed. -# --------------------------------------------------------------------------- - -from tools.threat_patterns import first_threat_message as _first_threat_message - - -def _scan_memory_content(content: str) -> Optional[str]: - """Scan memory content for injection/exfil patterns. Returns error string if blocked.""" - return _first_threat_message(content, scope="strict") - - -def _drift_error(path: "Path", bak_path: str) -> Dict[str, Any]: - """Build the error dict returned when external drift is detected. - - The on-disk memory file contains content that wouldn't round-trip - through the tool's parser/serializer — flushing would discard the - appended/edited content from a patch tool, shell append, manual edit, - or sister-session write. We refuse the mutation, point the operator at - the .bak. snapshot we took, and tell them what to do next. - """ - return { - "success": False, - "error": ( - f"Refusing to write {path.name}: file on disk has content that " - f"wouldn't round-trip through the memory tool (likely added by " - f"the patch tool, a shell append, a manual edit, or a " - f"concurrent session). A snapshot was saved to {bak_path}. " - f"Resolve the drift first — either rewrite the file as a clean " - f"§-delimited list of entries, or move the extra content out — " - f"then retry. This guard exists to prevent silent data loss " - f"(issue #26045)." - ), - "drift_backup": bak_path, - "remediation": ( - "Open the .bak file, integrate the missing entries into the " - "memory tool one at a time via memory(action=add, content=...), " - "then remove or rewrite the original file to a clean state." - ), - } - - -# Sentinel returned by ``_reload_target`` when the target file EXISTS but could -# not be read. Distinct from a drift-backup path (``str``) and from a clean -# reload (``None``): the caller must abort the mutation rather than persist over -# an unreadable file. -_READ_FAILED = object() - - -def _read_failed_error(path: "Path") -> Dict[str, Any]: - """Build the error dict returned when the on-disk memory file is unreadable. - - A file that exists but cannot be read is NOT an empty store. Reading it as - ``[]`` and then persisting would rewrite the whole file from an empty entry - list — wiping the user's memory. We refuse the write so nothing is lost. - """ - return { - "success": False, - "error": ( - f"Refusing to write {path.name}: the file exists on disk but could " - f"not be read right now (temporarily locked by another program, a " - f"permission change, invalid/corrupt text encoding, or a filesystem " - f"error). Treating an unreadable file as empty and saving would wipe " - f"existing memory, so the write is refused. Nothing was changed — " - f"retry in a moment." - ), - } - - -class MemoryStore: - """ - Bounded curated memory with file persistence. One instance per AIAgent. - - Maintains two parallel states: - - _system_prompt_snapshot: frozen at load time, used for system prompt injection. - Never mutated mid-session. Keeps prefix cache stable. - - memory_entries / user_entries: live state, mutated by tool calls, persisted to disk. - Tool responses always reflect this live state. - """ - - # After this many failed consolidation attempts (overflow / zero-match) in - # ONE turn, stop instructing the model to "retry in this turn" and return a - # terminal "save skipped" result so a fragile replace/add can't loop the - # turn to budget exhaustion and suppress the user's reply (issue #42405). - _MAX_CONSOLIDATION_FAILURES_PER_TURN = 3 - - def __init__( - self, - memory_char_limit: int = 2200, - user_char_limit: int = 1375, - *, - memory_enabled: bool = True, - user_profile_enabled: bool = True, - ): - self.memory_entries: List[str] = [] - self.user_entries: List[str] = [] - self.memory_char_limit = memory_char_limit - self.user_char_limit = user_char_limit - self.memory_enabled = memory_enabled - self.user_profile_enabled = user_profile_enabled - # Frozen snapshot for system prompt -- set once at load_from_disk() - self._system_prompt_snapshot: Dict[str, str] = {"memory": "", "user": ""} - # Per-turn counter of failed at-capacity consolidation attempts; reset - # at each turn boundary by reset_consolidation_failures() (#42405). - self._consolidation_failures = 0 - - def target_enabled(self, target: str) -> bool: - """Return whether this session's selected built-in store is writable.""" - return self.user_profile_enabled if target == "user" else self.memory_enabled - - def reset_consolidation_failures(self) -> None: - """Reset the per-turn consolidation-failure counter (call at turn start).""" - self._consolidation_failures = 0 - - def _consolidation_failure(self, response: Dict[str, Any]) -> Dict[str, Any]: - """Count an at-capacity consolidation failure and degrade gracefully. - - Under the per-turn cap, return ``response`` unchanged (it already tells - the model how to self-correct + retry in this turn). Once the cap is - exceeded, drop the retry instruction and return a TERMINAL result so the - model stops looping memory calls and proceeds to answer the user — a - failed memory side effect must never block the turn's reply (#42405). - """ - self._consolidation_failures += 1 - if self._consolidation_failures <= self._MAX_CONSOLIDATION_FAILURES_PER_TURN: - return response - return { - "success": False, - "done": True, - "error": ( - f"Memory consolidation failed {self._consolidation_failures} times " - "this turn. Stop retrying memory calls — leave memory unchanged for " - "now and continue with your reply to the user. The fact can be saved " - "in a later turn." - ), - } - - def load_from_disk(self): - """Load entries from MEMORY.md and USER.md, capture system prompt snapshot. - - The frozen snapshot is what enters the system prompt. We scan each - entry for injection/promptware patterns at snapshot-build time — - ANY hit replaces the entry text in the snapshot with a placeholder - like ``[BLOCKED: …]``, so a poisoned-on-disk memory file (supply - chain, compromised tool, sister-session write) cannot inject into - the system prompt. - - The live ``memory_entries`` / ``user_entries`` lists keep the - original text so the user can still SEE poisoned entries via - see poisoned entries by inspecting the source files directly, and remove them — silently dropping them would hide the attack from the user. - - Scanning is deterministic from disk bytes, so the snapshot remains - stable for the entire session (prefix-cache invariant holds). - """ - mem_dir = get_memory_dir() - mem_dir.mkdir(parents=True, exist_ok=True) - - self.memory_entries = self._read_file(mem_dir / "MEMORY.md") - self.user_entries = self._read_file(mem_dir / "USER.md") - - # Deduplicate entries (preserves order, keeps first occurrence) - self.memory_entries = list(dict.fromkeys(self.memory_entries)) - self.user_entries = list(dict.fromkeys(self.user_entries)) - - # Sanitize entries for the system-prompt snapshot only. Live state - # (memory_entries / user_entries) keeps the raw text so the user - # can see + remove poisoned entries via the memory tool. - sanitized_memory = self._sanitize_entries_for_snapshot(self.memory_entries, "MEMORY.md") - sanitized_user = self._sanitize_entries_for_snapshot(self.user_entries, "USER.md") - - # Capture frozen snapshot for system prompt injection - self._system_prompt_snapshot = { - "memory": self._render_block("memory", sanitized_memory), - "user": self._render_block("user", sanitized_user), - } - - @staticmethod - def _sanitize_entries_for_snapshot(entries: List[str], filename: str) -> List[str]: - """Return ``entries`` with any threat-matching entry replaced by a placeholder. - - Each entry is scanned with the shared threat-pattern library at the - ``"strict"`` scope (same as memory writes). On match, the entry is - replaced in the returned list with ``"[BLOCKED: entry - contained threat pattern: . Removed from system prompt.]"`` — - the placeholder enters the snapshot, the original entry stays in - live state for the user to inspect and delete. - - Empty or already-block-marker entries pass through unchanged. - """ - from tools.threat_patterns import scan_for_threats - - sanitized: List[str] = [] - for entry in entries: - if not entry or entry.startswith("[BLOCKED:"): - sanitized.append(entry) - continue - findings = scan_for_threats(entry, scope="strict") - if findings: - logger.warning( - "Memory entry from %s blocked at load time: %s", - filename, ", ".join(findings), - ) - sanitized.append( - f"[BLOCKED: {filename} entry contained threat pattern(s): " - f"{', '.join(findings)}. Removed from system prompt; " - f"use memory(action=remove) " - f"to delete the original.]" - ) - else: - sanitized.append(entry) - return sanitized - - @staticmethod - @contextmanager - def _file_lock(path: Path): - """Acquire an exclusive file lock for read-modify-write safety. - - Uses a separate .lock file so the memory file itself can still be - atomically replaced via os.replace(). - """ - lock_path = path.with_suffix(path.suffix + ".lock") - lock_path.parent.mkdir(parents=True, exist_ok=True) - - if fcntl is None and msvcrt is None: - yield - return - - fd = open(lock_path, "a+", encoding="utf-8") - try: - if fcntl: - fcntl.flock(fd, fcntl.LOCK_EX) - else: - fd.seek(0) - msvcrt.locking(fd.fileno(), msvcrt.LK_LOCK, 1) - yield - finally: - if fcntl: - try: - fcntl.flock(fd, fcntl.LOCK_UN) - except (OSError, IOError): - pass - elif msvcrt: - try: - fd.seek(0) - msvcrt.locking(fd.fileno(), msvcrt.LK_UNLCK, 1) - except (OSError, IOError): - pass - fd.close() - - @staticmethod - def _path_for(target: str) -> Path: - mem_dir = get_memory_dir() - if target == "user": - return mem_dir / "USER.md" - return mem_dir / "MEMORY.md" - - def _reload_target(self, target: str, *, skip_drift: bool = False): - """Re-read entries from disk into in-memory state. - - Called under file lock to get the latest state before mutating. - Returns the backup path if external drift was detected (the on-disk - file contains content that wouldn't round-trip through our - parser/serializer, OR an entry larger than the store's char limit). - When drift is detected the caller must abort the mutation — - flushing would discard the un-roundtrippable content. - Returns ``None`` on clean reload. - - Returns the ``_READ_FAILED`` sentinel when the file EXISTS but could not - be read. The caller MUST abort: the on-disk entries are unknown, so - overwriting from an assumed-empty view would wipe them. This is the real - exposure behind ``add`` — it skips the drift guard because appending is - safe, but that reasoning only holds when the reload actually saw the - file. A failed read reported as ``[]`` turned ``add`` into a full-file - rewrite down to a single entry. - - When *skip_drift* is True the round-trip / entry-size check is - bypassed. Used by the ``add`` action which appends without - rewriting, so existing content is never clobbered. - """ - path = self._path_for(target) - raw, read_ok = self._read_raw_checked(path) - if not read_ok: - # Leave in-memory entries untouched and tell the caller to abort; - # persisting over an unreadable file would destroy it. - return _READ_FAILED - # Derive BOTH the drift check and the entry parse from the same raw - # snapshot. The drift guard used to re-read the file itself and treat - # a failed second read as "no drift" — so a read failure between the - # checked reload and the drift check let replace/remove/apply_batch - # rewrite the file from a stale view, silently discarding whatever an - # external writer had just added. One read, one snapshot, no window. - bak = None if skip_drift else self._detect_external_drift(target, raw) - fresh = self._parse_entries(raw) - fresh = list(dict.fromkeys(fresh)) # deduplicate - self._set_entries(target, fresh) - return bak - - def save_to_disk(self, target: str): - """Persist entries to the appropriate file. Called after every mutation.""" - get_memory_dir().mkdir(parents=True, exist_ok=True) - self._write_file(self._path_for(target), self._entries_for(target)) - - def _entries_for(self, target: str) -> List[str]: - if target == "user": - return self.user_entries - return self.memory_entries - - def _set_entries(self, target: str, entries: List[str]): - if target == "user": - self.user_entries = entries - else: - self.memory_entries = entries - - def _char_count(self, target: str) -> int: - entries = self._entries_for(target) - if not entries: - return 0 - return len(ENTRY_DELIMITER.join(entries)) - - def _char_limit(self, target: str) -> int: - if target == "user": - return self.user_char_limit - return self.memory_char_limit - - def add(self, target: str, content: str) -> Dict[str, Any]: - """Append a new entry. Returns error if it would exceed the char limit.""" - content = content.strip() - if not content: - return {"success": False, "error": "Content cannot be empty."} - - # Scan for injection/exfiltration before accepting - scan_error = _scan_memory_content(content) - if scan_error: - return {"success": False, "error": scan_error} - - with self._file_lock(self._path_for(target)): - # Re-read from disk under lock to pick up writes from other sessions. - # For add (append-only), we skip the drift guard — appending never - # clobbers existing content, so round-trip mismatches from prior - # tool-written entries in the same session are harmless. The drift - # guard remains active for replace/remove where full-file rewrite - # would discard un-roundtrippable content (issue #26045). - # - # But "append never clobbers" only holds when the reload actually - # read the file. add rewrites the WHOLE file from the parsed - # entries, so a file that exists but read as empty (transient lock, - # permission blip, I/O error) would be rewritten down to just the - # new entry — wiping every prior memory. Refuse instead. - if self._reload_target(target, skip_drift=True) is _READ_FAILED: - return _read_failed_error(self._path_for(target)) - - entries = self._entries_for(target) - limit = self._char_limit(target) - - # Reject exact duplicates - if content in entries: - return self._success_response(target, "Entry already exists (no duplicate added).") - - # Calculate what the new total would be - new_entries = entries + [content] - new_total = len(ENTRY_DELIMITER.join(new_entries)) - - if new_total > limit: - current = self._char_count(target) - return self._consolidation_failure({ - "success": False, - "error": ( - f"Memory at {current:,}/{limit:,} chars. " - f"Adding this entry ({len(content)} chars) would exceed the limit. " - f"Consolidate now: use 'replace' to merge overlapping entries into " - f"shorter ones or 'remove' stale or less important entries (see " - f"current_entries below), then retry this add — all in this turn." - ), - "current_entries": entries, - "usage": f"{current:,}/{limit:,}", - }) - - entries.append(content) - self._set_entries(target, entries) - self.save_to_disk(target) - - return self._success_response(target, "Entry added.") - - def replace(self, target: str, old_text: str, new_content: str) -> Dict[str, Any]: - """Find entry containing old_text substring, replace it with new_content.""" - old_text = old_text.strip() - new_content = new_content.strip() - if not old_text: - return {"success": False, "error": "old_text cannot be empty."} - if not new_content: - return {"success": False, "error": "new_content cannot be empty. Use 'remove' to delete entries."} - - # Scan replacement content for injection/exfiltration - scan_error = _scan_memory_content(new_content) - if scan_error: - return {"success": False, "error": scan_error} - - with self._file_lock(self._path_for(target)): - bak = self._reload_target(target) - if bak is _READ_FAILED: - return _read_failed_error(self._path_for(target)) - if bak: - return _drift_error(self._path_for(target), bak) - - entries = self._entries_for(target) - matches = [(i, e) for i, e in enumerate(entries) if old_text in e] - - if not matches: - return self._consolidation_failure({ - "success": False, - "error": f"No entry matched '{old_text}'. Check current_entries below and retry with the exact text of the entry you want to replace.", - "current_entries": entries, - }) - - if len(matches) > 1: - # If all matches are identical (exact duplicates), operate on the first one - unique_texts = {e for _, e in matches} - if len(unique_texts) > 1: - previews = self._previews([e for _, e in matches]) - return { - "success": False, - "error": f"Multiple entries matched '{old_text}'. Be more specific.", - "matches": previews, - } - # All identical -- safe to replace just the first - - idx = matches[0][0] - limit = self._char_limit(target) - - # Check that replacement doesn't blow the budget - test_entries = entries.copy() - test_entries[idx] = new_content - new_total = len(ENTRY_DELIMITER.join(test_entries)) - - if new_total > limit: - current = self._char_count(target) - return self._consolidation_failure({ - "success": False, - "error": ( - f"Replacement would put memory at {new_total:,}/{limit:,} chars. " - f"Shorten the new content, or 'remove' other stale or less important " - f"entries to make room (see current_entries below), then retry — all " - f"in this turn." - ), - "current_entries": entries, - "usage": f"{current:,}/{limit:,}", - }) - - entries[idx] = new_content - self._set_entries(target, entries) - self.save_to_disk(target) - - return self._success_response(target, "Entry replaced.") - - def remove(self, target: str, old_text: str) -> Dict[str, Any]: - """Remove the entry containing old_text substring.""" - old_text = old_text.strip() - if not old_text: - return {"success": False, "error": "old_text cannot be empty."} - - with self._file_lock(self._path_for(target)): - bak = self._reload_target(target) - if bak is _READ_FAILED: - return _read_failed_error(self._path_for(target)) - if bak: - return _drift_error(self._path_for(target), bak) - - entries = self._entries_for(target) - matches = [(i, e) for i, e in enumerate(entries) if old_text in e] - - if not matches: - return self._consolidation_failure({ - "success": False, - "error": f"No entry matched '{old_text}'. Check current_entries below and retry with the exact text of the entry you want to remove.", - "current_entries": entries, - }) - - if len(matches) > 1: - # If all matches are identical (exact duplicates), remove the first one - unique_texts = {e for _, e in matches} - if len(unique_texts) > 1: - previews = self._previews([e for _, e in matches]) - return { - "success": False, - "error": f"Multiple entries matched '{old_text}'. Be more specific.", - "matches": previews, - } - # All identical -- safe to remove just the first - - idx = matches[0][0] - entries.pop(idx) - self._set_entries(target, entries) - self.save_to_disk(target) - - return self._success_response(target, "Entry removed.") - - def apply_batch(self, target: str, operations: List[Dict[str, Any]]) -> Dict[str, Any]: - """Apply a sequence of add/replace/remove ops to one target atomically. - - All operations are validated and applied against the FINAL budget -- - intermediate overflow is irrelevant. This lets the model free space - (remove/replace) and add new entries in a SINGLE tool call instead of - the multi-turn consolidate-then-retry dance that re-sends the whole - conversation context several times. - - Semantics: all-or-nothing. If any op is malformed, doesn't match, or - the net result would exceed the char limit, NOTHING is written and an - error is returned describing the first failure plus the live state. - """ - if not operations: - return {"success": False, "error": "operations list is empty."} - - # Scan every add/replace content for injection/exfil BEFORE touching - # disk -- a single poisoned op rejects the whole batch. - for i, op in enumerate(operations): - act = (op or {}).get("action") - new_content = (op or {}).get("content") - if act in {"add", "replace"} and new_content: - scan_error = _scan_memory_content(new_content) - if scan_error: - return {"success": False, "error": f"Operation {i + 1}: {scan_error}"} - - with self._file_lock(self._path_for(target)): - bak = self._reload_target(target) - if bak is _READ_FAILED: - return _read_failed_error(self._path_for(target)) - if bak: - return _drift_error(self._path_for(target), bak) - - # Work on a copy; only commit if the whole batch validates. - working: List[str] = list(self._entries_for(target)) - limit = self._char_limit(target) - - for i, op in enumerate(operations): - op = op or {} - act = op.get("action") - content = (op.get("content") or op.get("new_text") or "").strip() - old_text = (op.get("old_text") or "").strip() - pos = f"Operation {i + 1} ({act or 'unknown'})" - - if act == "add": - if not content: - return self._batch_error(target, f"{pos}: content is required.") - if content in working: - continue # idempotent -- skip duplicate, don't fail the batch - working.append(content) - - elif act == "replace": - if not old_text: - return self._batch_error(target, f"{pos}: old_text is required.") - if not content: - return self._batch_error( - target, - f"{pos}: content is required (use action='remove' to delete).", - ) - matches = [j for j, e in enumerate(working) if old_text in e] - if not matches: - return self._batch_error(target, f"{pos}: no entry matched '{old_text}'.") - if len({working[j] for j in matches}) > 1: - return self._batch_error( - target, - f"{pos}: '{old_text}' matched multiple distinct entries -- be more specific.", - ) - working[matches[0]] = content - - elif act == "remove": - if not old_text: - return self._batch_error(target, f"{pos}: old_text is required.") - matches = [j for j, e in enumerate(working) if old_text in e] - if not matches: - return self._batch_error(target, f"{pos}: no entry matched '{old_text}'.") - if len({working[j] for j in matches}) > 1: - return self._batch_error( - target, - f"{pos}: '{old_text}' matched multiple distinct entries -- be more specific.", - ) - working.pop(matches[0]) - - else: - return self._batch_error( - target, - f"{pos}: unknown action. Use add, replace, or remove.", - ) - - # Budget check against the FINAL state only. - new_total = len(ENTRY_DELIMITER.join(working)) if working else 0 - if new_total > limit: - current = self._char_count(target) - return self._consolidation_failure({ - "success": False, - "error": ( - f"After applying all {len(operations)} operations, memory would be at " - f"{new_total:,}/{limit:,} chars -- over the limit. Remove or shorten more " - f"entries in the same batch (see current_entries below), then retry." - ), - "current_entries": self._entries_for(target), - "usage": f"{current:,}/{limit:,}", - }) - - # Commit. - self._set_entries(target, working) - self.save_to_disk(target) - - return self._success_response(target, f"Applied {len(operations)} operation(s).") - - def _batch_error(self, target: str, message: str) -> Dict[str, Any]: - """Build a batch-abort error that reports live (uncommitted) state.""" - current = self._char_count(target) - limit = self._char_limit(target) - return self._consolidation_failure({ - "success": False, - "error": message + " No operations were applied (batch is all-or-nothing).", - "current_entries": self._entries_for(target), - "usage": f"{current:,}/{limit:,}", - }) - - def format_for_system_prompt(self, target: str) -> Optional[str]: - """ - Return the frozen snapshot for system prompt injection. - - This returns the state captured at load_from_disk() time, NOT the live - state. Mid-session writes do not affect this. This keeps the system - prompt stable across all turns, preserving the prefix cache. - - Returns None if the snapshot is empty (no entries at load time). - """ - block = self._system_prompt_snapshot.get(target, "") - return block if block else None - - # -- Internal helpers -- - - @staticmethod - def _previews(entries: List[str], width: int = 80) -> List[str]: - """Truncated one-line previews of entries for error feedback.""" - return [e[:width] + ("..." if len(e) > width else "") for e in entries] - - def _success_response(self, target: str, message: str = None) -> Dict[str, Any]: - # A successful write means the consolidation loop made progress, so the - # per-turn failure budget resets (the cap counts consecutive failures, - # not lifetime ones within a turn) (#42405). - self._consolidation_failures = 0 - entries = self._entries_for(target) - current = self._char_count(target) - limit = self._char_limit(target) - pct = min(100, int((current / limit) * 100)) if limit > 0 else 0 - - # The success response is intentionally TERMINAL: it confirms the write - # landed and tells the model to stop. We do NOT echo the full entries - # list here -- dumping it invites the model to "find more to fix" and - # re-issue the same operations (observed thrash: the correct batch on - # call 1, then 5 redundant repeats). Entries are only shown on the - # error/over-budget paths, where the model genuinely needs them to - # decide what to consolidate. - resp = { - "success": True, - "done": True, - "target": target, - "usage": f"{pct}% — {current:,}/{limit:,} chars", - "entry_count": len(entries), - } - if message: - resp["message"] = message - resp["note"] = "Write saved. This update is complete — do not repeat it." - return resp - - def _render_block(self, target: str, entries: List[str]) -> str: - """Render a system prompt block with header and usage indicator.""" - if not entries: - return "" - - limit = self._char_limit(target) - content = ENTRY_DELIMITER.join(entries) - current = len(content) - pct = min(100, int((current / limit) * 100)) if limit > 0 else 0 - - if target == "user": - header = f"{MEMORY_BLOCK_HEADERS['user']} [{pct}% — {current:,}/{limit:,} chars]" - else: - header = f"{MEMORY_BLOCK_HEADERS['memory']} [{pct}% — {current:,}/{limit:,} chars]" - - separator = "═" * 46 - return f"{separator}\n{header}\n{separator}\n{content}" - - @staticmethod - def _read_raw_checked(path: Path) -> Tuple[str, bool]: - """Read a memory file's raw text, distinguishing unreadable from empty. - - Returns ``(raw, read_ok)``. ``read_ok`` is False ONLY when the file - EXISTS but could not be read — an absent file is a clean ``("", True)``. - Invalid UTF-8 counts as unreadable too: the bytes on disk hold content - we cannot faithfully round-trip, so a rewrite would corrupt or discard - it just like a failed read. Read-modify-write callers must treat - ``read_ok=False`` as "abort" rather than "empty store", or a transient - read failure would let them persist over — and wipe — the on-disk - memory (issue #26045 is about the same class: never rewrite a file - from a view that isn't the real one). - - No file locking needed: _write_file uses atomic rename, so readers - always see either the previous complete file or the new complete file. - """ - if not path.exists(): - return "", True - try: - # utf-8-sig strips a leading UTF-8 BOM (Notepad-edited memory - # files on Windows) and is byte-identical to utf-8 otherwise. - # Plain utf-8 kept U+FEFF glued to the first entry, corrupting - # matching/dedup for that entry forever (#10878 / PR #10888). - # Decode errors stay STRICT on purpose: errors="replace" would - # hand read-modify-write callers a lossy view that a subsequent - # save persists over the real bytes — the wipe class documented - # above. Undecodable bytes must surface as read_ok=False. - return path.read_text(encoding="utf-8-sig"), True - except (OSError, IOError, UnicodeDecodeError): - return "", False - - @staticmethod - def _parse_entries(raw: str) -> List[str]: - """Split raw memory-file text into stripped, non-empty entries.""" - if not raw.strip(): - return [] - # Use ENTRY_DELIMITER for consistency with _write_file. Splitting by "§" - # alone would incorrectly split entries that contain "§" in their content. - entries = [e.strip() for e in raw.split(ENTRY_DELIMITER)] - return [e for e in entries if e] - - @staticmethod - def _read_entries_checked(path: Path) -> Tuple[List[str], bool]: - """Read + parse a memory file, distinguishing unreadable from empty. - - Returns ``(entries, read_ok)`` — see ``_read_raw_checked`` for the - ``read_ok`` contract. - """ - raw, read_ok = MemoryStore._read_raw_checked(path) - if not read_ok: - return [], False - return MemoryStore._parse_entries(raw), True - - @staticmethod - def _read_file(path: Path) -> List[str]: - """Read a memory file and split into entries (empty list on any error). - - Retained for read-only callers (``load_from_disk``) that build in-memory - state without persisting; a failed read degrading to ``[]`` there is - harmless because nothing is written back. Read-modify-write paths use - ``_read_raw_checked`` so they can refuse to overwrite an unreadable - file — see ``_reload_target``. - """ - return MemoryStore._read_entries_checked(path)[0] - - def _detect_external_drift(self, target: str, raw: str) -> Optional[str]: - """Return a backup-path string if on-disk content shows external drift. - - *raw* is the file content already read by the caller's checked read - (``_read_raw_checked``). Drift detection MUST operate on that same - snapshot — an earlier version re-read the file here and treated a - failed second read as "no drift", which let a mutation proceed from a - stale first snapshot and rewrite away content an external writer added - between the two reads. - - The memory file is supposed to be a list of small entries the tool - wrote, joined by §. Detect drift via two signals: - - 1. Round-trip mismatch — re-parsing and re-serializing the file - doesn't produce identical bytes (rare; would catch oddly-encoded - delimiters). - 2. Entry-size overflow — any single parsed entry exceeds the - store's whole-file char limit. The tool budgets the ENTIRE store - against that limit; no single tool-written entry can exceed it. - When we see one entry larger than the limit, an external writer - (patch tool, shell append, manual edit, sister session) appended - free-form content into what the tool will treat as one entry. - Flushing would then truncate that entry to the model's new - content, discarding the appended bytes — issue #26045. - - Returns the absolute path of the .bak file when drift was found and - backed up; returns None when the file looks tool-shaped. - - Note: this is an INSTANCE method (not static) because we need the - per-target char_limit for signal #2. - """ - path = self._path_for(target) - if not raw.strip(): - return None - - parsed = [e.strip() for e in raw.split(ENTRY_DELIMITER) if e.strip()] - roundtrip = ENTRY_DELIMITER.join(parsed) - - char_limit = self._char_limit(target) - max_entry_len = max((len(e) for e in parsed), default=0) - - drift_detected = (raw.strip() != roundtrip) or (max_entry_len > char_limit) - if not drift_detected: - return None - - # Drift confirmed — snapshot the file so the operator can recover - # whatever the external writer added, then return the .bak path so - # the caller can refuse the mutation. - ts = int(time.time()) - bak_path = path.with_suffix(path.suffix + f".bak.{ts}") - try: - bak_path.write_text(raw, encoding="utf-8") - except (OSError, IOError): - return str(bak_path) + " (BACKUP FAILED — file unchanged on disk)" - return str(bak_path) - - @staticmethod - def _write_file(path: Path, entries: List[str]): - """Write entries to a memory file using atomic temp-file + rename. - - Previous implementation used open("w") + flock, but "w" truncates the - file *before* the lock is acquired, creating a race window where - concurrent readers see an empty file. Atomic rename avoids this: - readers always see either the old complete file or the new one. - """ - content = ENTRY_DELIMITER.join(entries) if entries else "" - try: - atomic_write_text(path, content, tmp_prefix=".mem_") - except (OSError, IOError) as e: - raise RuntimeError(f"Failed to write memory file {path}: {e}") +from tools.memory_tool_store import ( # noqa: E402,F401 (re-exports) + ENTRY_DELIMITER, MEMORY_BLOCK_HEADERS, MemoryStore, _READ_FAILED, + _drift_error, _read_failed_error, _scan_memory_content, +) def load_on_disk_store() -> "MemoryStore": - """Build a fresh on-disk :class:`MemoryStore`, honoring configured char limits. - - Use this from any context that has no live agent (the messaging gateway, the - Desktop GUI, the bare CLI ``/memory`` handler) but still needs to read or - apply approved memory writes. Mirrors how the live agent constructs its store - in ``agent/agent_init.py`` — including the user's ``memory.memory_char_limit`` - / ``memory.user_char_limit`` overrides — so an approval applied without a live - agent enforces the SAME caps as one applied with one. - - Falls back to the built-in defaults if config can't be loaded, so this can - never raise on a missing/unreadable config. - """ - memory_char_limit = 2200 - user_char_limit = 1375 - memory_enabled = True - user_profile_enabled = True + """Fresh on-disk MemoryStore with configured limits/flags, for contexts with + no live agent (gateway, Desktop, bare CLI ``/memory``) so approvals enforce + the SAME caps as ``agent_init``. Defaults if config can't load; never raises.""" + kwargs: Dict[str, Any] = {} try: from hermes_cli.config import load_config config = load_config() or {} mem_cfg = get_builtin_memory_config(config) memory_enabled, user_profile_enabled = get_builtin_memory_store_flags(config) - memory_char_limit = int(mem_cfg.get("memory_char_limit", memory_char_limit)) - user_char_limit = int(mem_cfg.get("user_char_limit", user_char_limit)) + kwargs = { + "memory_char_limit": int(mem_cfg.get("memory_char_limit", 2200)), + "user_char_limit": int(mem_cfg.get("user_char_limit", 1375)), + "memory_enabled": memory_enabled, + "user_profile_enabled": user_profile_enabled, + } except Exception: - pass # config optional — fall back to defaults rather than break /memory - - store = MemoryStore( - memory_char_limit=memory_char_limit, - user_char_limit=user_char_limit, - memory_enabled=memory_enabled, - user_profile_enabled=user_profile_enabled, - ) + kwargs = {} # config optional — fall back to defaults rather than break /memory + store = MemoryStore(**kwargs) store.load_from_disk() return store -def _apply_write_gate(action: str, target: str, content: Optional[str], - old_text: Optional[str]) -> Optional[str]: - """Evaluate the memory write gate. Returns a JSON tool-result string when - the write should NOT proceed normally (blocked or staged), or None when the - caller should perform the real write. +# --------------------------------------------------------------------------- +# Write-approval gate +# --------------------------------------------------------------------------- - Only the mutating actions (add/replace/remove) are gated. - """ - if action not in {"add", "replace", "remove"}: - return None +def _target_label(target: str) -> str: + return "user profile" if target == "user" else "memory" + +def _gate_or_stage(summary: str, detail: str, payload: Dict[str, Any]) -> Optional[str]: + """Run the memory write gate. Returns a JSON tool-result string when the + write must NOT proceed (blocked, or staged for approval), None to proceed. + If the gate module can't load, fail open rather than block all writes.""" try: from tools import write_approval as wa except Exception: - # If the gate module can't load, fail open (current behaviour) rather - # than blocking all memory writes. return None - - # Build a small inline summary/detail for the foreground approval prompt. - label = "user profile" if target == "user" else "memory" - if action == "add": - summary = f"add to {label}" - detail = content or "" - elif action == "replace": - summary = f"replace in {label}" - detail = f"old: {old_text}\nnew: {content}" - else: # remove - summary = f"remove from {label}" - detail = old_text or "" - decision = wa.evaluate_gate(wa.MEMORY, inline_summary=summary, inline_detail=detail) - if decision.allow: return None - if decision.blocked: return tool_error(decision.message, success=False) + record = wa.stage_write(wa.MEMORY, payload, summary=f"{summary}: {detail[:120]}", origin=wa.current_origin()) + return json.dumps({"success": True, "staged": True, "pending_id": record["id"], "message": decision.message}, + ensure_ascii=False) - # stage - payload = { - "action": action, - "target": target, - "content": content, - "old_text": old_text, - } - record = wa.stage_write( - wa.MEMORY, payload, - summary=f"{summary}: {detail[:120]}", - origin=wa.current_origin(), - ) - return json.dumps( - {"success": True, "staged": True, "pending_id": record["id"], - "message": decision.message}, - ensure_ascii=False, - ) + +def _apply_write_gate(action: str, target: str, content: Optional[str], old_text: Optional[str]) -> Optional[str]: + """Gate a single mutating op (add/replace/remove); other actions pass.""" + if action not in _STORE_ACTIONS: + return None + label = _target_label(target) + if action == "add": + summary, detail = f"add to {label}", content or "" + elif action == "replace": + summary, detail = f"replace in {label}", f"old: {old_text}\nnew: {content}" + else: + summary, detail = f"remove from {label}", old_text or "" + payload = {"action": action, "target": target, "content": content, "old_text": old_text} + return _gate_or_stage(summary, detail, payload) def _apply_batch_write_gate(target: str, operations: List[Dict[str, Any]]) -> Optional[str]: - """Evaluate the write gate for a batch of memory operations. - - Returns a JSON tool-result string when the batch should NOT proceed - (blocked or staged), or None when the caller should perform the real - batch write. The whole batch is gated as a single unit. - """ - try: - from tools import write_approval as wa - except Exception: - return None - - label = "user profile" if target == "user" else "memory" - summary = f"apply {len(operations)} op(s) to {label}" + """Gate a whole batch as a single unit.""" + summary = f"apply {len(operations)} op(s) to {_target_label(target)}" detail_lines = [] for op in operations: op = op or {} @@ -1028,59 +130,47 @@ def _apply_batch_write_gate(target: str, operations: List[Dict[str, Any]]) -> Op detail_lines.append(f"- replace: {op.get('old_text', '')} -> {_op_content}") else: detail_lines.append(f"- {act}: {_op_content}") - detail = "\n".join(detail_lines) - - decision = wa.evaluate_gate(wa.MEMORY, inline_summary=summary, inline_detail=detail) - - if decision.allow: - return None - - if decision.blocked: - return tool_error(decision.message, success=False) - payload = {"action": "batch", "target": target, "operations": operations} - record = wa.stage_write( - wa.MEMORY, payload, - summary=f"{summary}: {detail[:120]}", - origin=wa.current_origin(), - ) - return json.dumps( - {"success": True, "staged": True, "pending_id": record["id"], - "message": decision.message}, - ensure_ascii=False, - ) + return _gate_or_stage(summary, "\n".join(detail_lines), payload) +# --------------------------------------------------------------------------- +# Tool entry point +# --------------------------------------------------------------------------- + def _missing_old_text_error(store: "MemoryStore", target: str, action: str) -> str: - """Build a recoverable error for a replace/remove call that arrived without - ``old_text``. + """Recoverable error for replace/remove without ``old_text``. It can't be + schema-required (needs a combinator the Codex backend rejects — see + test_memory_tool_schema.py) and some clients omit it, so return the current + inventory plus a retry instruction instead of a dead-end.""" + return json.dumps({ + "success": False, + "error": (f"'{action}' needs old_text -- a short unique substring of the entry " + f"to {action}. None was provided. Reissue the {action} with old_text " + f"set to part of one of the current_entries below."), + "current_entries": store._entries_for(target), + "usage": store._usage(target), + }, ensure_ascii=False) - ``replace``/``remove`` are inherently targeted -- without ``old_text`` there - is no entry to act on, so we cannot fulfil the call. But returning a bare - "old_text is required" is a dead-end: some structured-output clients omit the - optional ``old_text`` field (it isn't, and can't be, schema-required without - a top-level combinator the Codex backend rejects -- see - tests/tools/test_memory_tool_schema.py). So instead we return the current - entry inventory plus an explicit retry instruction, letting the model reissue - the call with ``old_text`` set to a unique substring of the entry it means. - Mirrors the batch path's ``_batch_error`` shape. (issues #43412, #49466) - """ - entries = store._entries_for(target) - current = store._char_count(target) - limit = store._char_limit(target) - return json.dumps( - { - "success": False, - "error": ( - f"'{action}' needs old_text -- a short unique substring of the entry " - f"to {action}. None was provided. Reissue the {action} with old_text " - f"set to part of one of the current_entries below." - ), - "current_entries": entries, - "usage": f"{current:,}/{limit:,}", - }, - ensure_ascii=False, - ) + +def _validate_single_op(store, action, target, content, old_text) -> Optional[str]: + """Validate required params BEFORE the gate so an invalid write is rejected + now rather than staged and failing at approve time.""" + if action == "add" and not content: + return tool_error("Content is required for 'add' action.", success=False) + if action in ("replace", "remove") and not old_text: + return _missing_old_text_error(store, target, action) + if action == "replace" and not content: + return tool_error("content is required for 'replace' action.", success=False) + return None + + +# action -> store call for both the live tool path and staged-write replay. +_STORE_ACTIONS = { + "add": lambda store, target, content, old_text: store.add(target, content), + "replace": lambda store, target, content, old_text: store.replace(target, old_text, content), + "remove": lambda store, target, content, old_text: store.remove(target, old_text), +} def memory_tool( @@ -1092,96 +182,48 @@ def memory_tool( operations: Optional[List[Dict[str, Any]]] = None, store: Optional[MemoryStore] = None, ) -> str: - """ - Single entry point for the memory tool. Dispatches to MemoryStore methods. - - Two shapes: - - Single op: action + (content / old_text). - - Batch: operations=[{action, content?, old_text?}, ...] applied - atomically against the final char budget in ONE call. - - ``new_text`` is accepted as an alias for ``content`` on both shapes. The - replace/remove ops target by ``old_text`` and supply the replacement via - ``content``; callers naturally reach for ``new_text`` to mirror - ``old_text`` (it's the patch tool's ``old_string``/``new_string`` shape), - which silently left ``content`` empty and errored. Coalescing here removes - that trap. - - Returns JSON string with results. - """ + """Tool entry point; returns a JSON string. Single op (action + content / + old_text) or batch (``operations`` applied atomically against the final + budget). ``new_text`` aliases ``content`` — callers mirror ``old_text`` + with it (patch-tool shape), which used to leave ``content`` empty.""" if store is None: return tool_error("Memory is not available. It may be disabled in config or this environment.", success=False) - # Accept new_text as an alias for content (single-op path). See docstring. if content is None and new_text is not None: content = new_text - - # Some strict providers fill optional schema fields with JSON null rather - # than omitting them. Treat ``target: null`` as omitted so memory writes - # still use the documented default store instead of failing validation. + # Strict providers send JSON null for optional fields; treat as omitted. if target is None: target = "memory" - target_error = _memory_target_error(store, target) if target_error is not None: return json.dumps(target_error) - # --- Batch path ------------------------------------------------------- if operations: if not isinstance(operations, list): return tool_error("operations must be a list of {action, content?, old_text?} objects.", success=False) gate_result = _apply_batch_write_gate(target, operations) if gate_result is not None: return gate_result - result = store.apply_batch(target, operations) - return json.dumps(result, ensure_ascii=False) - - # --- Single-op path --------------------------------------------------- - # Validate required params BEFORE the gate so an invalid write is rejected - # immediately instead of being staged and only failing at approve time. - if action == "add" and not content: - return tool_error("Content is required for 'add' action.", success=False) - if action == "replace" and (not old_text or not content): - missing = "old_text" if not old_text else "content" - if not old_text: - # The client/model omitted old_text. Replace is inherently targeted - # -- we can't guess which entry. Return the current inventory plus a - # retry instruction so the model can reissue with old_text set, - # instead of hitting a dead-end error. (issues #43412, #49466) - return _missing_old_text_error(store, target, "replace") - return tool_error(f"{missing} is required for 'replace' action.", success=False) - if action == "remove" and not old_text: - return _missing_old_text_error(store, target, "remove") + return json.dumps(store.apply_batch(target, operations), ensure_ascii=False) + run = _STORE_ACTIONS.get(action) + if run is None: + return tool_error(f"Unknown action '{action}'. Use: add, replace, remove", success=False) + invalid = _validate_single_op(store, action, target, content, old_text) + if invalid is not None: + return invalid # Approval gate: when on, stages the write (background/gateway) or prompts # inline (interactive CLI); when off (default) passes straight through. gate_result = _apply_write_gate(action, target, content, old_text) if gate_result is not None: return gate_result - - if action == "add": - result = store.add(target, content) - - elif action == "replace": - result = store.replace(target, old_text, content) - - elif action == "remove": - result = store.remove(target, old_text) - - else: - return tool_error(f"Unknown action '{action}'. Use: add, replace, remove", success=False) - - return json.dumps(result, ensure_ascii=False) + return json.dumps(run(store, target, content, old_text), ensure_ascii=False) def get_builtin_memory_config(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - """Return a normalized built-in memory config mapping. - - Missing, unreadable, or malformed sections become an empty mapping, whose - missing flags resolve to the enabled defaults. ``agent_init`` consumes this - same normalized section so tool availability and store construction cannot - diverge. - """ + """Normalized ``memory`` config section ({} when missing/malformed → flags + default to enabled). ``agent_init`` consumes the same section so tool + availability and store construction cannot diverge.""" if config is None: try: from hermes_cli.config import load_config_readonly @@ -1190,7 +232,6 @@ def get_builtin_memory_config(config: Optional[Dict[str, Any]] = None) -> Dict[s except Exception: logger.debug("Could not read memory config for availability", exc_info=True) return {} - section = config.get("memory") if isinstance(config, dict) else None return section if isinstance(section, dict) else {} @@ -1218,44 +259,30 @@ def _memory_target_error(store: "MemoryStore", target: str) -> Optional[Dict[str if target not in {"memory", "user"}: from tools.registry import _bound_error_text - return { - "success": False, - "error": _bound_error_text( - f"Invalid memory target '{target}'. Use 'memory' or 'user'." - ), - } + return {"success": False, + "error": _bound_error_text(f"Invalid memory target '{target}'. Use 'memory' or 'user'.")} if store.target_enabled(target): return None label = "USER.md" if target == "user" else "MEMORY.md" - return { - "success": False, - "error": f"Built-in {label} writes are disabled in memory config.", - "target": target, - } + return {"success": False, "error": f"Built-in {label} writes are disabled in memory config.", "target": target} def apply_memory_pending(payload: Dict[str, Any], store: "MemoryStore") -> Dict[str, Any]: - """Replay a staged memory write directly against the store, bypassing the - write gate. Called by the /memory approve handler. - - Returns the store's result dict. - """ + """Replay a staged write against the store, bypassing the gate (/memory approve).""" action = payload.get("action") target = payload.get("target", "memory") target_error = _memory_target_error(store, target) if target_error is not None: return target_error - content = payload.get("content") or "" - old_text = payload.get("old_text") or "" if action == "batch": return store.apply_batch(target, payload.get("operations") or []) - if action == "add": - return store.add(target, content) - if action == "replace": - return store.replace(target, old_text, content) - if action == "remove": - return store.remove(target, old_text) - return {"success": False, "error": f"Unknown staged action '{action}'."} + run = _STORE_ACTIONS.get(action) + if run is None: + return {"success": False, "error": f"Unknown staged action '{action}'."} + return run(store, target, payload.get("content") or "", payload.get("old_text") or "") + + +# ============================================================================= # OpenAI Function-Calling Schema # ============================================================================= @@ -1332,40 +359,39 @@ MEMORY_SCHEMA = { } +# Schema text when only one built-in store is enabled: (target description, TARGETS replacement). +_SINGLE_TARGET_TEXT = { + ("memory",): ( + "The enabled built-in store: 'memory' for personal notes.", + "TARGET: only 'memory' is enabled for personal notes (environment, conventions, " + "tool quirks, lessons).", + ), + ("user",): ( + "The enabled built-in store: 'user' for user profile.", + "TARGET: only 'user' is enabled for user profile facts (name, role, preferences, style).", + ), +} + + def _build_memory_schema_overrides() -> Dict[str, Any]: """Narrow the advertised target surface using the availability snapshot.""" flags = _memory_surface_flags.get() _memory_surface_flags.set(None) if flags is None: flags = get_builtin_memory_store_flags() - memory_enabled, user_profile_enabled = flags - targets = [] - if memory_enabled: - targets.append("memory") - if user_profile_enabled: - targets.append("user") - + targets = [t for t, on in zip(("memory", "user"), flags) if on] parameters = copy.deepcopy(MEMORY_SCHEMA["parameters"]) target_schema = parameters["properties"]["target"] target_schema["enum"] = targets - description = MEMORY_SCHEMA["description"] - if targets == ["memory"]: - target_schema["description"] = "The enabled built-in store: 'memory' for personal notes." + narrowed = _SINGLE_TARGET_TEXT.get(tuple(targets)) + if narrowed: + target_schema["description"], replacement = narrowed description = description.replace( "TARGETS: 'user' = who the user is (name, role, preferences, style). 'memory' = your " "notes (environment, conventions, tool quirks, lessons).", - "TARGET: only 'memory' is enabled for personal notes (environment, conventions, " - "tool quirks, lessons).", + replacement, ) - elif targets == ["user"]: - target_schema["description"] = "The enabled built-in store: 'user' for user profile." - description = description.replace( - "TARGETS: 'user' = who the user is (name, role, preferences, style). 'memory' = your " - "notes (environment, conventions, tool quirks, lessons).", - "TARGET: only 'user' is enabled for user profile facts (name, role, preferences, style).", - ) - return {"description": description, "parameters": parameters} @@ -1388,7 +414,3 @@ registry.register( emoji="🧠", dynamic_schema_overrides=_build_memory_schema_overrides, ) - - - - diff --git a/tools/memory_tool_store.py b/tools/memory_tool_store.py new file mode 100644 index 0000000000..2c9005cf71 --- /dev/null +++ b/tools/memory_tool_store.py @@ -0,0 +1,543 @@ +"""MemoryStore — bounded, file-backed curated memory (MEMORY.md / USER.md). +Entries are joined by ``ENTRY_DELIMITER``; budgets are in chars (model- +independent). Module state that tests monkeypatch (``get_memory_dir``, +``fcntl``/``msvcrt``) stays in ``tools.memory_tool`` and is read lazily.""" + +import logging +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from utils import atomic_write_text +from tools.threat_patterns import first_threat_message as _first_threat_message + +logger = logging.getLogger("tools.memory_tool") + +# System-prompt block header prefixes rendered by MemoryStore._render_block. +# agent/conversation_compression.py uses them to detect a leftover block for a +# target whose entries have since been emptied — keep in lockstep. +MEMORY_BLOCK_HEADERS = { + "memory": "MEMORY (your personal notes)", + "user": "USER PROFILE (who the user is)", +} + +ENTRY_DELIMITER = "\n§\n" + +# Sentinel from ``_reload_target``: the file EXISTS but could not be read. +# Distinct from a drift-backup path (str) and a clean reload (None); the caller +# must abort rather than persist over an unreadable file. +_READ_FAILED = object() + + +def _memory_dir() -> Path: + from tools import memory_tool + + return memory_tool.get_memory_dir() + + +def _scan_memory_content(content: str) -> Optional[str]: + """Scan memory content for injection/exfil patterns ("strict" scope: memory + enters the system prompt as a frozen snapshot, so a poisoned entry persists + across sessions). Returns the error string if blocked.""" + return _first_threat_message(content, scope="strict") + + +def _drift_error(path: "Path", bak_path: str) -> Dict[str, Any]: + """Error dict for external drift: the on-disk file wouldn't round-trip + through the parser/serializer, so flushing would discard content added by a + patch tool, shell append, manual edit, or sister session.""" + return { + "success": False, + "error": ( + f"Refusing to write {path.name}: file on disk has content that wouldn't round-trip " + f"through the memory tool (likely added by the patch tool, a shell append, a manual edit, " + f"or a concurrent session). A snapshot was saved to {bak_path}. Resolve the drift first — " + f"either rewrite the file as a clean §-delimited list of entries, or move the extra " + f"content out — then retry. This guard exists to prevent silent data loss (issue #26045)." + ), + "drift_backup": bak_path, + "remediation": ( + "Open the .bak file, integrate the missing entries into the memory tool one at a time via " + "memory(action=add, content=...), then remove or rewrite the original file to a clean state." + ), + } + + +def _read_failed_error(path: "Path") -> Dict[str, Any]: + """Error dict for an unreadable (but existing) memory file. Treating it as + empty and saving would rewrite the whole file from ``[]`` — wiping memory.""" + return {"success": False, "error": ( + f"Refusing to write {path.name}: the file exists on disk but could not be read right now " + f"(temporarily locked by another program, a permission change, invalid/corrupt text encoding, " + f"or a filesystem error). Treating an unreadable file as empty and saving would wipe existing " + f"memory, so the write is refused. Nothing was changed — retry in a moment." + )} + + +def _find_unique_match(entries: List[str], old_text: str) -> Tuple[Optional[int], bool]: + """``(index, ambiguous)`` for entries containing *old_text*. Exact-duplicate + matches are safe (first wins); distinct matches → ``(None, True)``.""" + matches = [i for i, e in enumerate(entries) if old_text in e] + if not matches: + return None, False + if len({entries[i] for i in matches}) > 1: + return None, True + return matches[0], False + + +class MemoryStore: + """Bounded curated memory with file persistence; one instance per AIAgent. + ``_system_prompt_snapshot`` is frozen at load time (prefix-cache stable); + ``memory_entries`` / ``user_entries`` are live state persisted to disk.""" + + # After this many failed consolidation attempts (overflow / zero-match) in + # ONE turn, return a terminal "save skipped" result instead of "retry in + # this turn", so a fragile replace/add can't loop the turn to budget + # exhaustion and suppress the user's reply. + _MAX_CONSOLIDATION_FAILURES_PER_TURN = 3 + + def __init__(self, memory_char_limit: int = 2200, user_char_limit: int = 1375, *, + memory_enabled: bool = True, user_profile_enabled: bool = True): + self.memory_entries: List[str] = [] + self.user_entries: List[str] = [] + self.memory_char_limit = memory_char_limit + self.user_char_limit = user_char_limit + self.memory_enabled = memory_enabled + self.user_profile_enabled = user_profile_enabled + self._system_prompt_snapshot: Dict[str, str] = {"memory": "", "user": ""} + # Per-turn counter of failed at-capacity consolidation attempts; reset + # at each turn boundary by reset_consolidation_failures(). + self._consolidation_failures = 0 + + def target_enabled(self, target: str) -> bool: + """Return whether this session's selected built-in store is writable.""" + return self.user_profile_enabled if target == "user" else self.memory_enabled + + def reset_consolidation_failures(self) -> None: + """Reset the per-turn consolidation-failure counter (call at turn start).""" + self._consolidation_failures = 0 + + def _consolidation_failure(self, response: Dict[str, Any]) -> Dict[str, Any]: + """Count an at-capacity consolidation failure. Under the per-turn cap + return ``response`` unchanged (it says how to retry); past it return a + TERMINAL result so the model stops looping — a failed memory side effect + must never block the turn's reply.""" + self._consolidation_failures += 1 + if self._consolidation_failures <= self._MAX_CONSOLIDATION_FAILURES_PER_TURN: + return response + return {"success": False, "done": True, "error": ( + f"Memory consolidation failed {self._consolidation_failures} times this turn. Stop retrying " + "memory calls — leave memory unchanged for now and continue with your reply to the user. " + "The fact can be saved in a later turn." + )} + + def load_from_disk(self): + """Load MEMORY.md / USER.md and capture the frozen system-prompt snapshot. + Threat hits are replaced in the SNAPSHOT only by a ``[BLOCKED: …]`` + placeholder; live lists keep the raw text so the user can see and remove + poisoned entries (dropping them silently would hide the attack). + Scanning is deterministic from disk bytes, so the snapshot stays stable.""" + mem_dir = _memory_dir() + mem_dir.mkdir(parents=True, exist_ok=True) + # Deduplicate (order-preserving, first occurrence wins). + self.memory_entries = list(dict.fromkeys(self._read_file(mem_dir / "MEMORY.md"))) + self.user_entries = list(dict.fromkeys(self._read_file(mem_dir / "USER.md"))) + self._system_prompt_snapshot = { + target: self._render_block(target, self._sanitize_entries_for_snapshot(entries, filename)) + for target, entries, filename in ( + ("memory", self.memory_entries, "MEMORY.md"), + ("user", self.user_entries, "USER.md"), + ) + } + + @staticmethod + def _sanitize_entries_for_snapshot(entries: List[str], filename: str) -> List[str]: + """Return *entries* with any threat-matching entry replaced by a + ``[BLOCKED: …]`` placeholder (strict scope, same as writes). Empty or + already-blocked entries pass through unchanged.""" + from tools.threat_patterns import scan_for_threats + + sanitized: List[str] = [] + for entry in entries: + findings = scan_for_threats(entry, scope="strict") if entry and not entry.startswith("[BLOCKED:") else None + if not findings: + sanitized.append(entry) + continue + logger.warning("Memory entry from %s blocked at load time: %s", filename, ", ".join(findings)) + sanitized.append(f"[BLOCKED: {filename} entry contained threat pattern(s): {', '.join(findings)}. " + f"Removed from system prompt; use memory(action=remove) to delete the original.]") + return sanitized + + @staticmethod + @contextmanager + def _file_lock(path: Path): + """Exclusive lock for read-modify-write safety, on a separate .lock file + so the memory file itself can still be atomically replaced.""" + from tools import memory_tool as _mt # fcntl/msvcrt live (and are patched) there + + fcntl, msvcrt = _mt.fcntl, _mt.msvcrt + lock_path = path.with_suffix(path.suffix + ".lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + if fcntl is None and msvcrt is None: + yield + return + fd = open(lock_path, "a+", encoding="utf-8") + try: + if fcntl: + fcntl.flock(fd, fcntl.LOCK_EX) + else: + fd.seek(0) + msvcrt.locking(fd.fileno(), msvcrt.LK_LOCK, 1) + yield + finally: + try: + if fcntl: + fcntl.flock(fd, fcntl.LOCK_UN) + else: + fd.seek(0) + msvcrt.locking(fd.fileno(), msvcrt.LK_UNLCK, 1) + except OSError: + pass + fd.close() + + @staticmethod + def _path_for(target: str) -> Path: + return _memory_dir() / ("USER.md" if target == "user" else "MEMORY.md") + + def _reload_target(self, target: str, *, skip_drift: bool = False): + """Re-read entries from disk (under file lock) before mutating. + + Returns ``None`` on a clean reload; the backup path (str) on external + drift (caller must abort — flushing would discard un-roundtrippable + content); or ``_READ_FAILED`` when the file exists but could not be + read (caller MUST abort — rewriting from an assumed-empty view would + wipe it; even append-only ``add`` rewrites the whole file). Drift check + and parse use the SAME raw snapshot — a failed second read used to count + as "no drift". *skip_drift* skips the round-trip check (``add``). + """ + raw, read_ok = self._read_raw_checked(self._path_for(target)) + if not read_ok: + return _READ_FAILED + bak = None if skip_drift else self._detect_external_drift(target, raw) + self._set_entries(target, list(dict.fromkeys(self._parse_entries(raw)))) + return bak + + def _reload_or_error(self, target: str, *, skip_drift: bool = False) -> Optional[Dict[str, Any]]: + """Reload under lock; return the abort error dict, or None to proceed.""" + bak = self._reload_target(target, skip_drift=skip_drift) + if bak is _READ_FAILED: + return _read_failed_error(self._path_for(target)) + if bak: + return _drift_error(self._path_for(target), bak) + return None + + def save_to_disk(self, target: str): + """Persist entries to the appropriate file. Called after every mutation.""" + _memory_dir().mkdir(parents=True, exist_ok=True) + self._write_file(self._path_for(target), self._entries_for(target)) + + def _entries_for(self, target: str) -> List[str]: + return self.user_entries if target == "user" else self.memory_entries + + def _set_entries(self, target: str, entries: List[str]): + if target == "user": + self.user_entries = entries + else: + self.memory_entries = entries + + def _char_count(self, target: str) -> int: + return len(ENTRY_DELIMITER.join(self._entries_for(target))) + + def _char_limit(self, target: str) -> int: + return self.user_char_limit if target == "user" else self.memory_char_limit + + def _usage(self, target: str) -> str: + return f"{self._char_count(target):,}/{self._char_limit(target):,}" + + def _failure_with_entries(self, target: str, message: str) -> Dict[str, Any]: + """Consolidation failure that shows the live entries so the model can + decide what to consolidate.""" + return self._consolidation_failure({"success": False, "error": message, + "current_entries": self._entries_for(target), "usage": self._usage(target)}) + + def _locate(self, target: str, old_text: str, verb: str): + """Resolve *old_text* to a unique entry index, or an error dict.""" + entries = self._entries_for(target) + idx, ambiguous = _find_unique_match(entries, old_text) + if ambiguous: + return None, {"success": False, "error": f"Multiple entries matched '{old_text}'. Be more specific.", + "matches": self._previews([e for e in entries if old_text in e])} + if idx is None: + return None, self._consolidation_failure({ + "success": False, + "error": f"No entry matched '{old_text}'. Check current_entries below and retry with the exact text of the entry you want to {verb}.", + "current_entries": entries, + }) + return idx, None + + def _commit(self, target: str, entries: List[str], message: str) -> Dict[str, Any]: + self._set_entries(target, entries) + self.save_to_disk(target) + return self._success_response(target, message) + + def add(self, target: str, content: str) -> Dict[str, Any]: + """Append a new entry. Returns error if it would exceed the char limit.""" + content = content.strip() + if not content: + return {"success": False, "error": "Content cannot be empty."} + scan_error = _scan_memory_content(content) + if scan_error: + return {"success": False, "error": scan_error} + with self._file_lock(self._path_for(target)): + # Append-only: skip the drift guard (appending never clobbers + # un-roundtrippable content), but still refuse on a failed read — + # add rewrites the WHOLE file from the parsed entries. + err = self._reload_or_error(target, skip_drift=True) + if err: + return err + entries = self._entries_for(target) + limit = self._char_limit(target) + if content in entries: + return self._success_response(target, "Entry already exists (no duplicate added).") + if len(ENTRY_DELIMITER.join(entries + [content])) > limit: + return self._failure_with_entries(target, ( + f"Memory at {self._char_count(target):,}/{limit:,} chars. Adding this entry " + f"({len(content)} chars) would exceed the limit. Consolidate now: use 'replace' to merge " + f"overlapping entries into shorter ones or 'remove' stale or less important entries (see " + f"current_entries below), then retry this add — all in this turn." + )) + entries.append(content) + return self._commit(target, entries, "Entry added.") + + def replace(self, target: str, old_text: str, new_content: str) -> Dict[str, Any]: + """Find entry containing old_text substring, replace it with new_content.""" + old_text = old_text.strip() + new_content = new_content.strip() + if not old_text: + return {"success": False, "error": "old_text cannot be empty."} + if not new_content: + return {"success": False, "error": "new_content cannot be empty. Use 'remove' to delete entries."} + scan_error = _scan_memory_content(new_content) + if scan_error: + return {"success": False, "error": scan_error} + with self._file_lock(self._path_for(target)): + err = self._reload_or_error(target) + if err: + return err + idx, err = self._locate(target, old_text, "replace") + if err: + return err + entries = self._entries_for(target) + limit = self._char_limit(target) + new_total = len(ENTRY_DELIMITER.join(entries[:idx] + [new_content] + entries[idx + 1:])) + if new_total > limit: + return self._failure_with_entries(target, ( + f"Replacement would put memory at {new_total:,}/{limit:,} chars. Shorten the new content, " + f"or 'remove' other stale or less important entries to make room (see current_entries " + f"below), then retry — all in this turn." + )) + entries[idx] = new_content + return self._commit(target, entries, "Entry replaced.") + + def remove(self, target: str, old_text: str) -> Dict[str, Any]: + """Remove the entry containing old_text substring.""" + old_text = old_text.strip() + if not old_text: + return {"success": False, "error": "old_text cannot be empty."} + with self._file_lock(self._path_for(target)): + err = self._reload_or_error(target) + if err: + return err + idx, err = self._locate(target, old_text, "remove") + if err: + return err + entries = self._entries_for(target) + entries.pop(idx) + return self._commit(target, entries, "Entry removed.") + + # -- Batch -- + + @staticmethod + def _apply_batch_op(working: List[str], act: str, content: str, old_text: str, pos: str) -> Optional[str]: + """Apply one batch op to *working* in place; return an error message or None.""" + if act == "add": + if not content: + return f"{pos}: content is required." + if content not in working: # idempotent -- skip duplicate, don't fail the batch + working.append(content) + return None + if act not in ("replace", "remove"): + return f"{pos}: unknown action. Use add, replace, or remove." + if not old_text: + return f"{pos}: old_text is required." + if act == "replace" and not content: + return f"{pos}: content is required (use action='remove' to delete)." + idx, ambiguous = _find_unique_match(working, old_text) + if ambiguous: + return f"{pos}: '{old_text}' matched multiple distinct entries -- be more specific." + if idx is None: + return f"{pos}: no entry matched '{old_text}'." + if act == "replace": + working[idx] = content + else: + working.pop(idx) + return None + + def apply_batch(self, target: str, operations: List[Dict[str, Any]]) -> Dict[str, Any]: + """Apply a sequence of add/replace/remove ops to one target atomically. + + Ops are validated and applied against the FINAL budget only — so a single + call can free space (remove/replace) and add new entries without the + multi-turn consolidate-then-retry dance. All-or-nothing: if any op is + malformed, doesn't match, or the net result exceeds the char limit, + NOTHING is written and the first failure plus live state is returned. + """ + if not operations: + return {"success": False, "error": "operations list is empty."} + + # Scan every add/replace content BEFORE touching disk -- one poisoned + # op rejects the whole batch. + for i, op in enumerate(operations): + op = op or {} + scan_error = op.get("action") in {"add", "replace"} and op.get("content") and _scan_memory_content(op["content"]) + if scan_error: + return {"success": False, "error": f"Operation {i + 1}: {scan_error}"} + + with self._file_lock(self._path_for(target)): + err = self._reload_or_error(target) + if err: + return err + # Work on a copy; only commit if the whole batch validates. + working: List[str] = list(self._entries_for(target)) + limit = self._char_limit(target) + for i, op in enumerate(operations): + op = op or {} + act = op.get("action") + content = (op.get("content") or op.get("new_text") or "").strip() + old_text = (op.get("old_text") or "").strip() + pos = f"Operation {i + 1} ({act or 'unknown'})" + msg = self._apply_batch_op(working, act, content, old_text, pos) + if msg: + return self._batch_error(target, msg) + # Budget check against the FINAL state only. + new_total = len(ENTRY_DELIMITER.join(working)) + if new_total > limit: + return self._failure_with_entries(target, ( + f"After applying all {len(operations)} operations, memory would be at " + f"{new_total:,}/{limit:,} chars -- over the limit. Remove or shorten more " + f"entries in the same batch (see current_entries below), then retry." + )) + return self._commit(target, working, f"Applied {len(operations)} operation(s).") + + def _batch_error(self, target: str, message: str) -> Dict[str, Any]: + """Build a batch-abort error that reports live (uncommitted) state.""" + return self._failure_with_entries( + target, message + " No operations were applied (batch is all-or-nothing)." + ) + + def format_for_system_prompt(self, target: str) -> Optional[str]: + """Return the frozen load-time snapshot for system-prompt injection (NOT + live state — mid-session writes don't affect it, preserving the prefix + cache). None if the snapshot is empty.""" + return self._system_prompt_snapshot.get(target, "") or None + + # -- Internal helpers -- + + @staticmethod + def _previews(entries: List[str], width: int = 80) -> List[str]: + """Truncated one-line previews of entries for error feedback.""" + return [e[:width] + ("..." if len(e) > width else "") for e in entries] + + def _usage_pct(self, target: str, current: int) -> str: + """``"% — / chars"`` for the given target.""" + limit = self._char_limit(target) + pct = min(100, int((current / limit) * 100)) if limit > 0 else 0 + return f"{pct}% — {current:,}/{limit:,} chars" + + def _success_response(self, target: str, message: str = None) -> Dict[str, Any]: + # A successful write means the consolidation loop made progress, so the + # per-turn failure budget resets (the cap counts consecutive failures). + self._consolidation_failures = 0 + # Intentionally TERMINAL and without the entries list: echoing entries + # invites the model to "find more to fix" and re-issue the same ops. + # Entries are only shown on error/over-budget paths. + resp = {"success": True, "done": True, "target": target, + "usage": self._usage_pct(target, self._char_count(target)), + "entry_count": len(self._entries_for(target))} + if message: + resp["message"] = message + resp["note"] = "Write saved. This update is complete — do not repeat it." + return resp + + def _render_block(self, target: str, entries: List[str]) -> str: + """Render a system prompt block with header and usage indicator.""" + if not entries: + return "" + content = ENTRY_DELIMITER.join(entries) + title = MEMORY_BLOCK_HEADERS["user" if target == "user" else "memory"] + separator = "═" * 46 + return f"{separator}\n{title} [{self._usage_pct(target, len(content))}]\n{separator}\n{content}" + + @staticmethod + def _read_raw_checked(path: Path) -> Tuple[str, bool]: + """Read raw text as ``(raw, read_ok)``. ``read_ok`` is False ONLY when the + file EXISTS but can't be read (absent file → ``("", True)``). Invalid + UTF-8 counts as unreadable; decoding stays STRICT because + ``errors="replace"`` would hand callers a lossy view that a save then + persists over the real bytes. ``utf-8-sig`` strips a Notepad BOM that + otherwise glues U+FEFF onto the first entry forever.""" + if not path.exists(): + return "", True + try: + return path.read_text(encoding="utf-8-sig"), True + except (OSError, UnicodeDecodeError): + return "", False + + @staticmethod + def _parse_entries(raw: str) -> List[str]: + """Split raw memory-file text into stripped, non-empty entries. Splits on + the full ENTRY_DELIMITER so a bare "§" inside an entry is preserved.""" + return [e for e in (x.strip() for x in raw.split(ENTRY_DELIMITER)) if e] + + @staticmethod + def _read_entries_checked(path: Path) -> Tuple[List[str], bool]: + """Read + parse as ``(entries, read_ok)`` — see ``_read_raw_checked``.""" + raw, read_ok = MemoryStore._read_raw_checked(path) + return MemoryStore._parse_entries(raw), read_ok + + @staticmethod + def _read_file(path: Path) -> List[str]: + """Read a memory file into entries (empty list on any error). Only for + read-only callers (``load_from_disk``, learning_mutations); mutation + paths must use ``_read_raw_checked`` so they can refuse to overwrite an + unreadable file.""" + return MemoryStore._read_entries_checked(path)[0] + + def _detect_external_drift(self, target: str, raw: str) -> Optional[str]: + """Backup-path string if *raw* (the caller's checked-read snapshot) shows + external drift, else None. Signals: round-trip mismatch, or one parsed + entry exceeding the whole-file char limit (no tool-written entry can — + an external writer appended free-form content a flush would truncate). + The file is snapshotted to ``.bak.`` so the operator can recover it.""" + if not raw.strip(): + return None + parsed = self._parse_entries(raw) + if raw.strip() == ENTRY_DELIMITER.join(parsed) and max(map(len, parsed), default=0) <= self._char_limit(target): + return None + path = self._path_for(target) + bak_path = path.with_suffix(path.suffix + f".bak.{int(time.time())}") + try: + bak_path.write_text(raw, encoding="utf-8") + except OSError: + return str(bak_path) + " (BACKUP FAILED — file unchanged on disk)" + return str(bak_path) + + @staticmethod + def _write_file(path: Path, entries: List[str]): + """Atomic temp-file + rename: readers see the old or the new complete + file, never a truncated one.""" + try: + atomic_write_text(path, ENTRY_DELIMITER.join(entries), tmp_prefix=".mem_") + except OSError as e: + raise RuntimeError(f"Failed to write memory file {path}: {e}") diff --git a/tools/session_search_tool.py b/tools/session_search_tool.py index f2daafeaf1..5a8ff6fd4d 100644 --- a/tools/session_search_tool.py +++ b/tools/session_search_tool.py @@ -1,360 +1,36 @@ #!/usr/bin/env python3 -""" -Session Search Tool - Long-Term Conversation Recall +"""Session Search Tool - long-term conversation recall over the SQLite session DB. -Single-shape tool with four calling modes (inferred from args, no explicit -mode parameter): - - 1. DISCOVERY — pass ``query``. Runs FTS5 and dedupes hits by session lineage. - Adaptive detail (the default) fully hydrates the top result with a ±5 - message window and bookends, while lower-ranked results keep the exact - anchor message plus metadata. Pass ``detail="full"`` to fully hydrate - every result. Zero LLM cost. - - 2. SCROLL — pass ``session_id`` + ``around_message_id``. Returns a window - of ±window messages centered on the anchor, no FTS5, no bookends. To - scroll forward / backward, re-anchor on the last / first message id of - the returned window. - - 3. READ — pass ``session_id`` without an anchor. Returns the whole session, - or a bounded head/tail view for large sessions. - - 4. BROWSE — no args. Returns recent sessions chronologically (titles, - previews, timestamps). - -All four modes operate on the SQLite session DB via the FTS5 index and -the get_anchored_view / get_messages_around primitives in hermes_state. -No LLM calls anywhere — every shape returns actual messages from the DB. - -History: PR #20238 (JabberELF) seeded a fast/summary dual-mode split; the -toolkit expansion in PR #26419 (yoniebans) added the anchored drill-down, -bookends, and sort. This module merges all of that into a single calling -shape with no mode parameter, no summary LLM path, and explicit scroll -support. +Single-shape tool; the mode is inferred from the args: DISCOVERY (``query``; +FTS5 deduped by lineage, adaptive detail hydrates only the top result), +SCROLL (``session_id`` + ``around_message_id``; ±window around the anchor), +READ (``session_id`` alone; whole session or head/tail), BROWSE (no args). +No LLM calls — every shape returns actual DB messages. Helpers live in +``session_search_tool_common`` / ``_discover`` and are re-exported here. """ import json import logging -from typing import Any, Dict, List, Optional, Union +from typing import Any, List, Optional -from hermes_state_common import _RESET_END_REASONS - -# Sources that are excluded from session browsing/searching by default. -# Third-party integrations tag their sessions with HERMES_SESSION_SOURCE=tool; -# delegate subagent runs are tagged "subagent"; kanban dispatcher workers are -# tagged "kanban" — none belongs in the user's session history. -_HIDDEN_SESSION_SOURCES = ("kanban", "subagent", "tool") - -# Automation sources that are kept searchable but DEMOTED below interactive -# sessions in discover ranking. Cron jobs run on a schedule and accumulate -# large volumes of repetitive vocabulary (recurring project names, dates, -# "session", summaries); under bare BM25 they dominate the top-N FTS rows and -# starve out the user's own interactive sessions, producing "recall blindness" -# where only cron sessions surface (#19434). Demoting — not excluding — keeps -# cron content reachable when it's the only match, while interactive sessions -# always win when both match. -_DEMOTED_SESSION_SOURCES = ("cron",) - -# How many FTS rows discover scans before dedup-by-lineage. The interactive -# vs automation split below only helps if enough rows are in hand to find -# interactive matches buried under a wall of cron hits, so this is well above -# the handful of distinct sessions a typical query returns. -_DISCOVER_SCAN_LIMIT = 300 - -# Raw FTS rows are only a discovery-plan input. The final response hydrates -# its own anchored message window and bookends after lineage deduplication. -_DISCOVER_SEARCH_FIELDS = ( - "id", - "session_id", - "role", - "snippet", - "source", - "model", - "session_started", +from tools.session_search_tool_common import ( # noqa: F401 (re-exports) + _COMPACTION_PREFIXES, _DEMOTED_SESSION_SOURCES, _DISCOVER_SCAN_LIMIT, + _DISCOVER_SEARCH_FIELDS, _FRESH_RESET_END_REASONS, _HIDDEN_SESSION_SOURCES, + _annotate_rebuild_status, _format_timestamp, _get_message_storage_state, + _is_compacted_message, _is_compacted_state, _is_compaction_summary, + _ok, _order_for_recall, _quiet, _resolve_lineage, _resolve_to_parent, _session_end_reason, + _session_left_live_context, _session_link, _session_meta_block, _shape_message, ) - -# Prefixes that identify generated context-compaction handoff summaries. -# These are inserted by agent/context_compressor.py as normal user/assistant -# messages but contain machine-generated summary metadata — not user content. -# They must be excluded from discovery bookends to avoid re-introducing huge -# compaction payloads into fresh sessions via session_search. (#43175) -_COMPACTION_PREFIXES = ( - "[CONTEXT COMPACTION", - "[CONTEXT SUMMARY]:", +from tools.session_search_tool_discover import ( # noqa: F401 (re-exports) + _discover, _normalize_title_query, _title_match_result, ) -# Gateway /new, /reset, idle/daily expiry, and CLI /new end the predecessor -# without carrying its transcript into the child. Those children share a -# parent_session_id lineage with the current session, but the prior content -# is NOT in live context — unlike compression continuations (summary carried -# forward) and live delegation children (parent still running). -# -# Derived from the canonical gateway reset-reason set so the recovery fence -# and this tool cannot drift (see the comment on _RESET_END_REASONS). -# "new_session" is the CLI /new end reason (cli.py), which the gateway set -# does not include. -_FRESH_RESET_END_REASONS = frozenset(_RESET_END_REASONS) | {"new_session"} - - -def _format_timestamp(ts: Union[int, float, str, None]) -> str: - """Convert a Unix timestamp (float/int) or ISO string to a human-readable date. - - Returns "unknown" for None, str(ts) if conversion fails. - """ - if ts is None: - return "unknown" - try: - if isinstance(ts, (int, float)): - from datetime import datetime - dt = datetime.fromtimestamp(ts) - return dt.strftime("%B %d, %Y at %I:%M %p") - if isinstance(ts, str): - if ts.replace(".", "").replace("-", "").isdigit(): - from datetime import datetime - dt = datetime.fromtimestamp(float(ts)) - return dt.strftime("%B %d, %Y at %I:%M %p") - return ts - except (ValueError, OSError, OverflowError) as e: - logging.debug("Failed to format timestamp %s: %s", ts, e, exc_info=True) - except Exception as e: - logging.debug("Unexpected error formatting timestamp %s: %s", ts, e, exc_info=True) - return str(ts) - - -def _is_compaction_summary(content: str) -> bool: - """Return True if *content* looks like a generated compaction handoff.""" - if not content: - return False - stripped = content.lstrip() - return any(stripped.startswith(p) for p in _COMPACTION_PREFIXES) - - -def _resolve_to_parent(db, session_id: str) -> tuple[str, bool]: - """Walk parent_session_id chain to the lineage root. - - Returns ``(root_id, has_compression_hop)`` where ``has_compression_hop`` is - True if any session along the chain ended with ``end_reason = 'compression'`` - — i.e. at least one parent/ancestor was compression-rotated into this - lineage. That flag lets callers distinguish a compression-split lineage - (parent content summarised away, no longer in live context) from a - delegation lineage (child content still visible to the parent agent). - - Falls back to ``(session_id, False)`` on errors. - """ - if not session_id: - return session_id, False - visited: set[str] = set() - cur = session_id - has_compression = False - while cur and cur not in visited: - visited.add(cur) - try: - s = db.get_session(cur) - if not s: - break - if s.get("end_reason") == "compression": - has_compression = True - parent = s.get("parent_session_id") - if not parent: - break - cur = parent - except Exception as e: - logging.debug("Error resolving parent for %s: %s", cur, e, exc_info=True) - break - return cur, has_compression - - -def _resolve_lineage(db, session_id: str) -> str: - """Convenience: return only the lineage root (ignores compression hop).""" - return _resolve_to_parent(db, session_id)[0] - - -def _session_end_reason(db, session_id: str) -> Optional[str]: - """Return the session's ``end_reason``, or None if missing/unended/error.""" - if not session_id: - return None - try: - s = db.get_session(session_id) - if not s: - return None - return s.get("end_reason") or None - except Exception: - return None - - -def _is_compression_ended(db, session_id: str) -> bool: - """Return True if *session_id* itself ended with ``end_reason='compression'``. - - Unlike the ``has_compression_hop`` flag from :func:`_resolve_to_parent` - (which is True for any descendant of a compression-ended ancestor), this - checks only the session's own ``end_reason``. A delegation child created - under a compression continuation has ``parent_session_id`` set but its own - ``end_reason`` is ``None`` — its content is still live to the parent agent, - so it must stay excluded from discovery. - """ - return _session_end_reason(db, session_id) == "compression" - - -def _session_left_live_context(db, session_id: str) -> bool: - """True when *session_id*'s transcript is no longer in anyone's live context. - - Two shapes qualify: - - - ``compression``: the transcript was summarised into the continuation - child, so the original rows left live context. - - fresh resets (:data:`_FRESH_RESET_END_REASONS`): every - ``_RESET_END_REASONS`` member plus CLI ``new_session`` — the child - starts empty and carries nothing forward. - - Everything else stays excluded from same-lineage recall: live delegation - children (``end_reason is None``) are still visible to the parent agent, - and ``branched`` parents were verbatim-copied into the branch child, so - their content IS the current context. - """ - end_reason = _session_end_reason(db, session_id) - return end_reason == "compression" or _is_fresh_reset_session(end_reason) - - -def _is_fresh_reset_session(end_reason: Optional[str]) -> bool: - """True when *end_reason* is a /new-style reset (transcript not carried forward).""" - return end_reason in _FRESH_RESET_END_REASONS - - -def _get_message_storage_state(db, message_id) -> Optional[Dict[str, Any]]: - """Return the owning session and visibility flags for *message_id*.""" - if not message_id: - return None - try: - with db._lock: - cursor = db._conn.execute( - "SELECT session_id, active, compacted FROM messages WHERE id = ?", - (message_id,), - ) - row = cursor.fetchone() - except Exception: - logging.debug( - "message storage-state lookup failed for %s", message_id, exc_info=True - ) - return None - return dict(row) if row is not None else None - - -def _is_compacted_message(db, message_id) -> bool: - """Return True if *message_id* is a compaction-archived row. - - Compaction archives are ``active=0, compacted=1`` — the content was - summarised away from live context by :meth:`archive_and_compact`. - Rewind/undo rows are ``active=0, compacted=0`` and must stay hidden. - - Used by ``_discover`` to distinguish a compaction-archived FTS hit on the - current session (pre-compaction content no longer in live context — should - stay discoverable) from an active live hit (already in context — skip). - Returns False on any error so the caller falls back to the safe default - (skip the current session). - """ - state = _get_message_storage_state(db, message_id) - return state is not None and state["active"] == 0 and state["compacted"] == 1 - - -def _annotate_rebuild_status(db, payload: Dict[str, Any]) -> None: - """Add a rebuild-progress note when the deferred FTS backfill (schema - v23) is still running, so the agent can tell the user why older results - may be incomplete/slower instead of treating a thin result set as - ground truth. No-op (and never raises) when no rebuild is pending.""" - try: - status = db.fts_rebuild_status() - except Exception: - return - if status is None: - return - payload["index_rebuild"] = { - "percent": status["percent"], - "note": ( - f"The search index is rebuilding in the background " - f"({status['percent']}% done, {status['indexed']:,} of " - f"{status['total']:,} messages). Results from older messages " - f"may be incomplete until it finishes." - ), - } - - -def _order_for_recall(raw_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Stable-sort FTS rows so interactive sessions rank above automation. - - Within each class (interactive vs demoted) the original BM25 ``rank`` - order is preserved — Python's sort is stable, and rows arrive already - ranked by relevance. This only changes cross-class ordering: a cron hit - never displaces an interactive hit during lineage dedup, so the user's - own conversations surface first even when cron rows out-rank them under - bare BM25 (#19434). Demoted rows still appear when they're the only - matches. - """ - return sorted( - raw_results, - key=lambda r: 1 if (r.get("source") or "") in _DEMOTED_SESSION_SOURCES else 0, - ) - - -def _shape_message( - m: Dict[str, Any], - anchor_id: Optional[int] = None, - max_content_len: Optional[int] = None, -) -> Dict[str, Any]: - """Slim a message row for the tool response. Keeps content even if empty. - - When *max_content_len* is set, ``content`` is truncated to that many - characters and ``content_truncated`` / ``original_content_chars`` metadata - is added so callers know the payload was bounded. - """ - raw_content = m.get("content") - if isinstance(raw_content, str) and "\x1b" in raw_content: - # Recalled messages can carry ANSI escape sequences (e.g. archived - # terminal output). Strip them before returning content to the model. - from tools.ansi_strip import strip_ansi - - raw_content = strip_ansi(raw_content) - if max_content_len and raw_content and len(raw_content) > max_content_len: - content = raw_content[:max_content_len] + "…" - truncated = True - original_chars = len(raw_content) - else: - content = raw_content - truncated = False - original_chars = None - entry = { - "id": m.get("id"), - "role": m.get("role"), - "content": content, - "timestamp": m.get("timestamp"), - } - if m.get("tool_name"): - entry["tool_name"] = m.get("tool_name") - if m.get("tool_calls"): - entry["tool_calls"] = m.get("tool_calls") - if m.get("tool_call_id"): - entry["tool_call_id"] = m.get("tool_call_id") - if anchor_id is not None and m.get("id") == anchor_id: - entry["anchor"] = True - if truncated: - entry["content_truncated"] = True - entry["original_content_chars"] = original_chars - # Strip None values to keep payload tight, but always keep content - # (absent content is meaningful — tool-call-only assistant turns). - return {k: v for k, v in entry.items() if v is not None or k in ("content",)} - def _resolve_profile_db(profile: str): - """Open another profile's ``state.db`` read-only, or None for the current one. - - The desktop's ``@session:/`` links always carry the source - profile, so a linked session from profile B can be read while the agent - runs in profile A. ``read_only=True`` (mode=ro) takes no write lock — safe - to point at a live profile's DB, including our own. Returns None when no - profile is given (use the caller's default db). - """ + """Open another profile's ``state.db`` read-only (no write lock — safe on a + live DB), or None for the current profile.""" if profile is None or not str(profile).strip(): return None - from hermes_cli import profiles as profiles_mod from hermes_state import SessionDB @@ -362,41 +38,12 @@ def _resolve_profile_db(profile: str): profiles_mod.validate_profile_name(canon) if not profiles_mod.profile_exists(canon): raise ValueError(f"profile '{canon}' does not exist") - return SessionDB(db_path=profiles_mod.get_profile_dir(canon) / "state.db", read_only=True) -def _session_link(session_id: str, profile: str = None) -> str: - """The reference the agent writes to point the user at a session. - - Same value the desktop composer emits when a session is dragged into a - message, so the desktop renders it as a link carrying the session's title. - The profile segment is omitted when we can't name it confidently — a bare - id still resolves, it just can't disambiguate across profiles. - """ - name = (profile or "").strip() - if not name: - try: - from hermes_cli.profiles import get_active_profile_name - - resolved = get_active_profile_name() - name = "" if resolved == "custom" else resolved - except Exception: - logging.debug("get_active_profile_name failed for session link", exc_info=True) - name = "" - - return f"@session:{name}/{session_id}" if name else f"@session:{session_id}" - - def _locate_session_db(session_id: str): - """Scan every profile's ``state.db`` (read-only) for a session id. - - Returns ``(db, profile_name)`` for the first profile that owns the id, or - ``(None, None)``. Session ids are globally unique (timestamp + random hex), - so the first hit is authoritative. This is the safety net for linked-session - reads where the model dropped the owning profile from the link and passed a - bare id — we find it wherever it actually lives instead of failing. - """ + """Scan every profile's ``state.db`` for a session id -> ``(db, profile_name)`` + or ``(None, None)``. Ids are globally unique, so the first hit is authoritative.""" from pathlib import Path try: @@ -404,47 +51,36 @@ def _locate_session_db(session_id: str): from hermes_state import SessionDB except Exception: return None, None - targets = [("default", profiles_mod.get_profile_dir("default"))] - try: - targets += [(info.name, info.path) for info in profiles_mod.list_profiles()] - except Exception: - logging.debug("list_profiles failed during session locate", exc_info=True) - + targets += _quiet(lambda: [(info.name, info.path) for info in profiles_mod.list_profiles()], + [], "list_profiles failed during session locate") seen: set = set() for name, home in targets: db_path = Path(home) / "state.db" - key = str(db_path) - if key in seen or not db_path.exists(): + if str(db_path) in seen or not db_path.exists(): continue - seen.add(key) + seen.add(str(db_path)) try: pdb = SessionDB(db_path=db_path, read_only=True) except Exception: continue - try: - if pdb.get_session(session_id): - return pdb, name - except Exception: - logging.debug("get_session probe failed for %s in %s", session_id, name, exc_info=True) + if _quiet(lambda: pdb.get_session(session_id), None, + "get_session probe failed for %s in %s", session_id, name): + return pdb, name pdb.close() - return None, None -def _read_session(db, session_id: str, head: int = 20, tail: int = 10, link_profile: str = None) -> str: - """Read shape: dump a whole session by id (head + tail when large). +def _get_session_meta(db, session_id: str) -> dict: + """``db.get_session`` that degrades to ``{}`` on error.""" + return _quiet(lambda: db.get_session(session_id), None, + "get_session failed for %s: %s", session_id, with_exc=True) or {} - Serves the linked-session case — the user dropped an @session reference and - the agent wants the transcript. Bounded payload: small sessions return in - full, large ones return the first ``head`` and last ``tail`` messages with a - pointer to scroll the middle. - """ - try: - meta = db.get_session(session_id) or {} - except Exception as e: - logging.debug("get_session failed for %s: %s", session_id, e, exc_info=True) - meta = {} + +def _read_session(db, session_id: str, head: int = 20, tail: int = 10, link_profile: str = None) -> str: + """Read shape: whole session, or first ``head`` + last ``tail`` messages + with a pointer to scroll the middle.""" + meta = _get_session_meta(db, session_id) if not meta: return tool_error(f"session_id not found: {session_id}", success=False) @@ -457,163 +93,126 @@ def _read_session(db, session_id: str, head: int = 20, tail: int = 10, link_prof shaped = [_shape_message(m) for m in rows] total = len(shaped) truncated = total > head + tail - window = shaped[:head] + shaped[-tail:] if truncated else shaped - - response = { - "success": True, - "mode": "read", - "session_id": session_id, - "link": _session_link(session_id, link_profile), - "session_meta": { - "when": _format_timestamp(meta.get("started_at")), - "source": meta.get("source"), - "model": meta.get("model"), - "title": meta.get("title"), - }, - "message_count": total, - "truncated": truncated, - "messages": window, - } - if truncated: - response["message"] = ( - f"Session has {total} messages; showing first {head} + last {tail}. " - "Pass around_message_id (any id above) to scroll the middle." - ) - return json.dumps(response, ensure_ascii=False) + extra = {"message": (f"Session has {total} messages; showing first {head} + last {tail}. " + "Pass around_message_id (any id above) to scroll the middle.")} if truncated else {} + return _ok(mode="read", session_id=session_id, link=_session_link(session_id, link_profile), + session_meta=_session_meta_block(meta), message_count=total, truncated=truncated, + messages=shaped[:head] + shaped[-tail:] if truncated else shaped, **extra) def _list_recent_sessions(db, limit: int, current_session_id: str = None, link_profile: str = None) -> str: - """Return metadata for the most recent sessions (no LLM calls, no FTS5).""" + """Browse shape: metadata for the most recent sessions (no LLM, no FTS5).""" try: # list_sessions_rich (include_children=False) already applies the - # canonical child classifier (_LISTABLE_CHILD_SQL): roots, /branch - # children, and /new-reset children are admitted (stable markers plus - # the legacy same-key heuristic), while delegation/compression - # children are hidden. Re-classifying rows here in Python duplicated - # that predicate and re-hid legacy pre-marker reset children the SQL - # deliberately admits — trust the query instead (#85756). + # canonical child classifier: roots, /branch children and /new-reset + # children are admitted, delegation/compression children hidden. + # Re-classifying here re-hid legacy reset children — trust the query. sessions = db.list_sessions_rich( - limit=limit + 15, + limit=limit + 15, # extra so we can skip current / compression roots exclude_sources=list(_HIDDEN_SESSION_SOURCES), order_by_last_active=True, - ) # fetch extra so we can skip current / compression roots + ) current_root, has_compression_hop = ( _resolve_to_parent(db, current_session_id) if current_session_id else (None, False) ) - results = [] for s in sessions: sid = s.get("id", "") if sid == current_session_id: continue - # Compression continuation: the root's original turns were - # summarised into the live child, so hide the root. /new-reset - # children share a lineage root but carry no transcript — keep - # that root browsable. + # Compression continuation: the root's turns were summarised into + # the live child, so hide the root. /new-reset children share a + # root but carry no transcript — keep that root browsable. if has_compression_hop and current_root and sid == current_root: continue results.append({ - "session_id": sid, - "link": _session_link(sid, link_profile), - "title": s.get("title") or None, - "source": s.get("source", ""), - "started_at": s.get("started_at", ""), - "last_active": s.get("last_active", ""), - "message_count": s.get("message_count", 0), + "session_id": sid, "link": _session_link(sid, link_profile), "title": s.get("title") or None, + "source": s.get("source", ""), "started_at": s.get("started_at", ""), + "last_active": s.get("last_active", ""), "message_count": s.get("message_count", 0), "preview": s.get("preview", ""), }) if len(results) >= limit: break - - return json.dumps({ - "success": True, - "mode": "browse", - "results": results, - "count": len(results), - "message": f"Showing {len(results)} most recent sessions. Pass a query= to search, or session_id+around_message_id to scroll.", - }, ensure_ascii=False) + return _ok(mode="browse", results=results, count=len(results), message=( + f"Showing {len(results)} most recent sessions. Pass a query= to search, " + "or session_id+around_message_id to scroll.")) except Exception as e: logging.error("Error listing recent sessions: %s", e, exc_info=True) return tool_error(f"Failed to list recent sessions: {e}", success=False) -def _scroll( - db, - session_id: str, - around_message_id: int, - window: int = 5, - current_session_id: str = None, -) -> str: - """Scroll shape: return a window of messages centered on an anchor. +def _clamp_int(value, default: int, lo: int, hi: int) -> int: + if not isinstance(value, int): + try: + value = int(value) + except (TypeError, ValueError): + value = default + return max(lo, min(value, hi)) - No FTS5, no bookends — just the slice. The discovery shape's lineage - fixup is preserved: if the anchor doesn't live in the named session - but does live in a child session in the same lineage, rebind silently. - """ + +def _anchor_in_live_context(db, anchor_state, anchor_session_id: str, current_session_id: str) -> bool: + """True when the scroll anchor is still in the caller's active context and + must be rejected. Same-lineage history that has LEFT live context (compacted + rows, compression-ended parents, /new-reset predecessors) is allowed, so + scroll never rejects a result discovery just returned.""" + a_root = _resolve_lineage(db, anchor_session_id) + c_root = _resolve_lineage(db, current_session_id) + if not (a_root and c_root and a_root == c_root): + return False + if _is_compacted_state(anchor_state): + return False + # Rewind/undo rows (active=0, compacted!=1) never count as out-of-context history. + is_inactive_non_compacted = ( + anchor_state is not None + and anchor_state["active"] == 0 + and anchor_state["compacted"] != 1 + ) + return is_inactive_non_compacted or not _session_left_live_context(db, anchor_session_id) + + +def _rebind_to_owner(db, session_id: str, owning: str, around_message_id: int, window: int): + """Lineage rebind: the caller paired a parent session_id with a message id + that lives in a descendant (compaction / delegation create child sessions). + Returns ``(view, warning)`` from the owning session, or ``(None, None)``.""" + a_root = _resolve_lineage(db, session_id) + o_root = _resolve_lineage(db, owning) + if not (a_root and o_root and a_root == o_root): + return None, None + rebind_view = _quiet(lambda: db.get_messages_around(owning, around_message_id, window=window), + None, "rebind get_messages_around failed: %s", with_exc=True) + if not (rebind_view and rebind_view.get("window")): + return None, None + return rebind_view, f"around_message_id {around_message_id} lives in {owning} (child of {session_id}); rebound transparently" + + +def _scroll(db, session_id: str, around_message_id: int, window: int = 5, + current_session_id: str = None) -> str: + """Scroll shape: a window of messages centered on an anchor (no FTS5, no + bookends). Rebinds silently if the anchor lives in a same-lineage child.""" if not isinstance(session_id, str) or not session_id.strip(): return tool_error("scroll requires session_id", success=False) session_id = session_id.strip() - try: around_message_id = int(around_message_id) except (TypeError, ValueError): return tool_error("scroll requires integer around_message_id", success=False) + window = _clamp_int(window, 5, 1, 20) - # Window clamp [1, 20] - if not isinstance(window, int): - try: - window = int(window) - except (TypeError, ValueError): - window = 5 - window = max(1, min(window, 20)) - - # Locate the anchor before applying the current-lineage guard. Discovery - # intentionally surfaces same-lineage history that is no longer in live - # context: in-place compacted rows, compression-ended parents, and - # /new-reset predecessors. Scroll must preserve that distinction instead - # of rejecting the discovery result it just returned. + # Locate the anchor BEFORE the current-lineage guard (see _anchor_in_live_context). anchor_state = _get_message_storage_state(db, around_message_id) - owning_session_id = ( - anchor_state.get("session_id") if anchor_state is not None else None - ) + owning_session_id = anchor_state.get("session_id") if anchor_state is not None else None - if current_session_id: - anchor_session_id = owning_session_id or session_id - a_root = _resolve_lineage(db, anchor_session_id) - c_root = _resolve_lineage(db, current_session_id) - if a_root and c_root and a_root == c_root: - is_compacted_anchor = ( - anchor_state is not None - and anchor_state["active"] == 0 - and anchor_state["compacted"] == 1 - ) - is_inactive_non_compacted_anchor = ( - anchor_state is not None - and anchor_state["active"] == 0 - and anchor_state["compacted"] != 1 - ) - is_out_of_context_history = ( - not is_inactive_non_compacted_anchor - and _session_left_live_context(db, anchor_session_id) - ) - if not (is_compacted_anchor or is_out_of_context_history): - return tool_error( - "scroll rejected: anchor lives in the current session lineage (already in your active context)", - success=False, - ) + if current_session_id and _anchor_in_live_context( + db, anchor_state, owning_session_id or session_id, current_session_id + ): + return tool_error("scroll rejected: anchor lives in the current session lineage (already in your active context)", success=False) - # Session existence check - try: - session_meta = db.get_session(session_id) or {} - except Exception as e: - logging.debug("get_session failed for %s: %s", session_id, e, exc_info=True) - session_meta = {} + session_meta = _get_session_meta(db, session_id) if not session_meta: return tool_error(f"session_id not found: {session_id}", success=False) - # Fetch the window try: view = db.get_messages_around(session_id, around_message_id, window=window) except Exception as e: @@ -621,370 +220,60 @@ def _scroll( return tool_error(f"failed to load messages: {e}", success=False) messages = view.get("window") or [] - - # Lineage rebind: caller may have paired a parent session_id with a - # message id that lives in a descendant (compaction / delegation creates - # child sessions). Locate the real owning session and refetch. rebind_warning = None - if not messages: - owning = owning_session_id - if owning and owning != session_id: - a_root = _resolve_lineage(db, session_id) - o_root = _resolve_lineage(db, owning) - if a_root and o_root and a_root == o_root: - try: - rebind_view = db.get_messages_around(owning, around_message_id, window=window) - messages = rebind_view.get("window") or [] - if messages: - view = rebind_view - rebind_warning = ( - f"around_message_id {around_message_id} lives in {owning} " - f"(child of {session_id}); rebound transparently" - ) - try: - session_meta = db.get_session(owning) or session_meta - except Exception: - pass - session_id = owning - except Exception as e: - logging.debug("rebind get_messages_around failed: %s", e, exc_info=True) + if not messages and owning_session_id and owning_session_id != session_id: + rebind_view, rebind_warning = _rebind_to_owner( + db, session_id, owning_session_id, around_message_id, window + ) + if rebind_view is not None: + view = rebind_view + messages = view["window"] + session_meta = _get_session_meta(db, owning_session_id) or session_meta + session_id = owning_session_id if not messages: - return tool_error( - f"around_message_id {around_message_id} not in session_id {session_id}", - success=False, - ) + return tool_error(f"around_message_id {around_message_id} not in session_id {session_id}", success=False) - response = { - "success": True, - "mode": "scroll", - "session_id": session_id, - "around_message_id": around_message_id, - "session_meta": { - "when": _format_timestamp(session_meta.get("started_at")), - "source": session_meta.get("source"), - "model": session_meta.get("model"), - "title": session_meta.get("title"), - }, - "window": window, - "messages": [_shape_message(m, anchor_id=around_message_id) for m in messages], - "messages_before": view.get("messages_before", 0), - "messages_after": view.get("messages_after", 0), - "hint": ( - "Scroll forward: re-call with around_message_id = the LAST message's " - "id; backward: the FIRST message's id (the boundary message repeats " - "as an orientation marker). messages_before/messages_after < window " - "means you've hit that end of the session." - ), - } - if rebind_warning: - response["warning"] = rebind_warning - return json.dumps(response, ensure_ascii=False) + return _ok( + mode="scroll", session_id=session_id, around_message_id=around_message_id, + session_meta=_session_meta_block(session_meta), window=window, + messages=[_shape_message(m, anchor_id=around_message_id) for m in messages], + messages_before=view.get("messages_before", 0), messages_after=view.get("messages_after", 0), + hint=("Scroll forward: re-call with around_message_id = the LAST message's " + "id; backward: the FIRST message's id (the boundary message repeats " + "as an orientation marker). messages_before/messages_after < window " + "means you've hit that end of the session."), + **({"warning": rebind_warning} if rebind_warning else {}), + ) -def _normalize_title_query(query: str) -> str: - """Strip common quoting the model may include around a remembered title.""" - return query.strip().strip("`'\"") - - -def _title_match_result( - db, - query: str, - current_lineage_root: Optional[str], -) -> Optional[Dict[str, Any]]: - """Return a discovery-shaped result when the query matches a session title.""" - title_query = _normalize_title_query(query) - if not title_query: - return None - - try: - session_id = db.resolve_session_by_title(title_query) - except Exception: - logging.debug("resolve_session_by_title failed for %r", title_query, exc_info=True) - return None - if not session_id: - return None - - lineage_root = _resolve_lineage(db, session_id) - if current_lineage_root and lineage_root == current_lineage_root: - # Same-lineage title hits are in-context only when the session is - # still live. /new-reset and compression-ended parents are not. - if not _session_left_live_context(db, session_id): - return None - - try: - session_meta = db.get_session(lineage_root) or db.get_session(session_id) or {} - except Exception: - logging.debug("get_session failed for title match %s", session_id, exc_info=True) - session_meta = {} - if session_meta.get("source") in _HIDDEN_SESSION_SOURCES: - return None - - try: - messages = db.get_messages(session_id) - except Exception: - logging.debug("get_messages failed for title match %s", session_id, exc_info=True) - messages = [] - - anchor_id = messages[0].get("id") if messages else None - if anchor_id is not None: +def _read_with_profile_fallback(db, sid: str, profile: Optional[str]) -> str: + """Read shape. On a miss in the target profile, scan every profile (the + model may have dropped the owning profile from the link) and tag the result + with the profile it was found in.""" + result = _read_session(db, sid, link_profile=profile) + if json.loads(result).get("success"): + return result + located, owner = _locate_session_db(sid) + if located is not None: try: - view = db.get_anchored_view(session_id, anchor_id, window=5, bookend=3) - except Exception: - logging.debug("get_anchored_view failed for title match %s/%s", session_id, anchor_id, exc_info=True) - view = {} - else: - view = {} - - entry = { - "session_id": session_id, - "when": _format_timestamp(session_meta.get("started_at")), - "source": session_meta.get("source", "unknown"), - "model": session_meta.get("model") or "unknown", - "title": session_meta.get("title") or title_query, - "matched_role": "session_title", - "match_message_id": anchor_id, - "snippet": f"Session title matched: {session_meta.get('title') or title_query}", - "bookend_start": [_shape_message(m) for m in (view.get("bookend_start") or messages[:3])], - "messages": [_shape_message(m, anchor_id=anchor_id) for m in (view.get("window") or messages[:5])], - "bookend_end": [_shape_message(m) for m in (view.get("bookend_end") or messages[-3:])], - "messages_before": view.get("messages_before", 0), - "messages_after": view.get("messages_after", max(len(messages) - 5, 0)), - "detail": "full", - "_lineage_root": lineage_root, - } - if lineage_root and lineage_root != session_id: - entry["parent_session_id"] = lineage_root - return entry + found = json.loads(_read_session(located, sid, link_profile=owner)) + finally: + located.close() + if found.get("success"): + found["profile"] = owner + return json.dumps(found, ensure_ascii=False) + return result -def _discover( - db, - query: str, - role_filter: Optional[List[str]], - limit: int, - sort: Optional[str], - detail: str, - current_session_id: str = None, - link_profile: str = None, -) -> str: - """Discovery shape: FTS5 plus adaptive or full result hydration.""" - role_list = role_filter if role_filter else ["user", "assistant"] - current_lineage_root = _resolve_lineage(db, current_session_id) if current_session_id else None - title_result = _title_match_result(db, query, current_lineage_root) - - try: - raw_results = db.search_messages( - query=query, - role_filter=role_list, - exclude_sources=list(_HIDDEN_SESSION_SOURCES), - limit=_DISCOVER_SCAN_LIMIT, # widen so dedup-by-lineage can find - # distinct sessions AND so interactive matches buried under a wall - # of cron rows are still in hand for the demotion pass below. - offset=0, - sort=sort, - fields=_DISCOVER_SEARCH_FIELDS, - ) - except Exception as e: - logging.error("FTS5 search failed: %s", e, exc_info=True) - return tool_error(f"Search failed: {e}", success=False) - - # Demote automation (cron) rows below interactive ones before dedup, so a - # high-volume cron corpus can't starve the user's own sessions out of the - # top `limit` results (#19434). Stable — preserves BM25/recency order - # within each class. - raw_results = _order_for_recall(raw_results) - - if not raw_results and not title_result: - _empty_payload = { - "success": True, - "mode": "discover", - "query": query, - "detail": detail, - "results": [], - "count": 0, - "message": ( - "No matching sessions found. FTS5 ANDs all terms by default — " - "broaden with OR (`alpha OR beta`), exact-match with quoted " - "phrases, exclude with NOT, or prefix-match with `deploy*`." - ), - } - _annotate_rebuild_status(db, _empty_payload) - return json.dumps(_empty_payload, ensure_ascii=False) - - # Dedupe by lineage. Keep the raw owning session_id on the surviving - # row — only that pairs validly with the FTS5 match id for the anchored - # window. parent_session_id is exposed separately when different. - seen_sessions = {} - results = [] - - if title_result: - title_lineage = title_result.pop("_lineage_root", None) - if title_lineage: - seen_sessions[title_lineage] = {"_title_only": True} - results.append(title_result) - - for r in raw_results: - if len(seen_sessions) >= limit: - break - raw_sid = r["session_id"] - resolved_sid, _ = _resolve_to_parent(db, raw_sid) - # Skip the current session lineage — UNLESS the hit's transcript has - # left live context. Three sub-cases: - # - # Legacy compression rotation: the FTS hit lives in a session that - # itself ended with end_reason='compression'. That session's content - # has been replaced by a summary in the continuation child, so it - # must stay discoverable. - # - # /new-reset (and idle/daily/CLI new_session): the predecessor was - # ended without carrying any transcript into the child. Same lineage - # root, but the prior conversation is NOT in the active context — - # hiding it made gateway recall go blind after every /new (#85756). - # A live delegation child has end_reason=None, so it stays excluded. - # - # In-place compaction: the FTS hit lives on the SAME session_id as the - # current session, but the matched message row is an archived - # (active=0, compacted=1) row. The live-context load filters active=1, - # so that content is no longer in context — let it through. - is_compacted_hit = _is_compacted_message(db, r.get("id")) - is_ended_session = _session_left_live_context(db, raw_sid) - if current_lineage_root and resolved_sid == current_lineage_root: - if not (is_ended_session or is_compacted_hit): - continue - if current_session_id and raw_sid == current_session_id: - # Same-session hit: only skip if the matched message is still live - # (active=1). Archived/compacted rows are pre-compaction content - # that's been summarised away — let them through. - if not is_compacted_hit: - continue - if resolved_sid not in seen_sessions: - row = dict(r) - row["_lineage_root"] = resolved_sid - seen_sessions[resolved_sid] = row - if len(seen_sessions) >= limit: - break - - for lineage_root, match_info in seen_sessions.items(): - if match_info.get("_title_only"): - continue - hit_sid = match_info.get("session_id") or lineage_root - msg_id = match_info.get("id") - try: - view = db.get_anchored_view(hit_sid, msg_id, window=5, bookend=3) - except Exception as e: - logging.warning("get_anchored_view failed for %s/%s: %s", hit_sid, msg_id, e, exc_info=True) - continue - - try: - session_meta = db.get_session(lineage_root) or {} - except Exception: - session_meta = {} - - result_detail = "full" if detail == "full" or not results else "compact" - window_messages = view.get("window") or [] - if result_detail == "compact": - window_messages = [m for m in window_messages if m.get("id") == msg_id] - - entry = { - "session_id": hit_sid, - "when": _format_timestamp( - session_meta.get("started_at") or match_info.get("session_started") - ), - "source": session_meta.get("source") or match_info.get("source", "unknown"), - "model": session_meta.get("model") or match_info.get("model") or "unknown", - "title": session_meta.get("title") or None, - "matched_role": match_info.get("role"), - "match_message_id": msg_id, - "snippet": match_info.get("snippet") or "", - "bookend_start": ( - [ - _shape_message(m, max_content_len=1200) - for m in (view.get("bookend_start") or []) - if not _is_compaction_summary(m.get("content", "")) - ] - if result_detail == "full" - else [] - ), - "messages": [ - _shape_message(m, anchor_id=msg_id, max_content_len=4000) - for m in window_messages - ], - "bookend_end": ( - [ - _shape_message(m, max_content_len=1200) - for m in (view.get("bookend_end") or []) - if not _is_compaction_summary(m.get("content", "")) - ] - if result_detail == "full" - else [] - ), - "messages_before": view.get("messages_before", 0), - "messages_after": view.get("messages_after", 0), - "detail": result_detail, - } - if lineage_root and lineage_root != hit_sid: - entry["parent_session_id"] = lineage_root - results.append(entry) - - for entry in results: - entry["link"] = _session_link(entry["session_id"], link_profile) - - _final_payload = { - "success": True, - "mode": "discover", - "query": query, - "detail": detail, - "results": results, - "count": len(results), - "sessions_searched": len(seen_sessions), - "link_hint": ( - "When referring the user to a session, write its `link` value " - "verbatim inline mid-sentence (it renders as a titled link) — never " - "as markdown, in backticks, on its own line, or next to the " - "title/id/date. To read more around a compact result, scroll: " - "session_search(session_id=..., around_message_id=match_message_id)." - ), - } - _annotate_rebuild_status(db, _final_payload) - return json.dumps(_final_payload, ensure_ascii=False) - - -def _session_search_impl( - query: str = "", - role_filter: str = None, - limit: int = 3, - db=None, - current_session_id: str = None, - # Scroll shape - session_id: str = None, - around_message_id: int = None, - window: int = 5, - # Discovery shape - sort: str = None, - # Cross-profile (any shape) - profile: str = None, - # Discovery result shaping (appended to preserve positional compatibility) - detail: str = "adaptive", - *, - _owned_dbs: Optional[List[Any]] = None, -) -> str: - """Single-shape tool. Mode inferred from which args are set. - - Discovery: pass ``query``; ``detail="full"`` hydrates every result. - Scroll: pass ``session_id`` + ``around_message_id``. - Read: pass ``session_id`` (no anchor) — dumps the whole session. - Browse: pass nothing. - - Pass ``profile`` to read another profile's sessions (e.g. resolving an - ``@session:/`` link). Scroll wins over read/discovery when an - anchor is set — the agent has asked for a specific slice. - """ - # Normalise a raw `@session:/` link value passed as session_id. - # Session ids never contain "/", so a slash unambiguously means profile/id — - # always strip the prefix off the id, and adopt the embedded profile only - # when one wasn't passed explicitly. Handles every permutation the model - # might send (full value as id, with or without a separate profile=). +def _dispatch(query, role_filter, limit, db, current_session_id, session_id, + around_message_id, window, sort, profile, detail, owned_dbs) -> str: + """Mode dispatch (see module docstring). Scroll wins over read/discovery when + an anchor is set — the agent asked for a specific slice. Profile DBs opened + here are appended to *owned_dbs* for the caller to close.""" + # A raw `@session:/` link passed as session_id: ids never + # contain "/", so a slash means profile/id — strip the prefix and adopt the + # embedded profile only when none was passed explicitly. if isinstance(session_id, str) and "/" in session_id: emb_profile, _, emb_id = session_id.partition("/") if emb_id: @@ -993,110 +282,43 @@ def _session_search_impl( profile = emb_profile # Cross-profile read: swap in the named profile's DB (read-only) for every - # shape below. The current-session-lineage guards no longer apply across - # profiles, but they key off ids that won't collide, so they stay inert. + # shape. Current-lineage guards key off ids that won't collide, so they + # stay inert. if profile is not None and str(profile).strip(): try: profile_db = _resolve_profile_db(profile) except Exception as e: return tool_error(f"profile '{profile}': {e}", success=False) if profile_db is not None: - db = profile_db - if _owned_dbs is not None: - _owned_dbs.append(profile_db) - current_session_id = None + db, current_session_id = profile_db, None + owned_dbs.append(profile_db) - # Scroll shape takes precedence — explicit anchor beats any query. - if (isinstance(session_id, str) and session_id.strip()) and around_message_id is not None: - return _scroll( - db=db, - session_id=session_id, - around_message_id=around_message_id, - window=window, - current_session_id=current_session_id, - ) + has_session = isinstance(session_id, str) and bool(session_id.strip()) + if has_session and around_message_id is not None: + return _scroll(db, session_id, around_message_id, window, current_session_id) + if has_session: + return _read_with_profile_fallback(db, session_id.strip(), profile) - # Read shape: a session_id with no anchor → dump the whole session. - if isinstance(session_id, str) and session_id.strip(): - sid = session_id.strip() - result = _read_session(db, sid, link_profile=profile) - if json.loads(result).get("success"): - return result - - # Miss in the target profile — the model may have dropped the owning - # profile from the link. Scan every profile and read it from wherever - # it lives, tagging the profile it was found in. - located, owner = _locate_session_db(sid) - if located is not None: - try: - found = json.loads(_read_session(located, sid, link_profile=owner)) - finally: - located.close() - if found.get("success"): - found["profile"] = owner - return json.dumps(found, ensure_ascii=False) - return result - - # Limit clamp [1, 10] - if not isinstance(limit, int): - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 3 - limit = max(1, min(limit, 10)) - - # Browse shape: no query → recent sessions. + limit = _clamp_int(limit, 3, 1, 10) if not query or not isinstance(query, str) or not query.strip(): return _list_recent_sessions(db, limit, current_session_id, link_profile=profile) - # Parse role_filter - role_list: Optional[List[str]] = None - if isinstance(role_filter, str) and role_filter.strip(): - role_list = [r.strip() for r in role_filter.split(",") if r.strip()] - - # Normalise sort - sort_norm: Optional[str] = None - if isinstance(sort, str): - candidate = sort.strip().lower() - if candidate in ("newest", "oldest"): - sort_norm = candidate - - detail_norm = ( - "full" - if isinstance(detail, str) and detail.strip().lower() == "full" - else "adaptive" - ) - + role_list = ([r.strip() for r in role_filter.split(",") if r.strip()] or None) if isinstance(role_filter, str) else None + sort_norm = sort.strip().lower() if isinstance(sort, str) else None + if sort_norm not in ("newest", "oldest"): + sort_norm = None + detail_norm = "full" if isinstance(detail, str) and detail.strip().lower() == "full" else "adaptive" return _discover( - db=db, - query=query.strip(), - role_filter=role_list, - limit=limit, - sort=sort_norm, - detail=detail_norm, - current_session_id=current_session_id, - link_profile=profile, + db=db, query=query.strip(), role_filter=role_list, limit=limit, sort=sort_norm, + detail=detail_norm, current_session_id=current_session_id, link_profile=profile, ) -def session_search( - query: str = "", - role_filter: str = None, - limit: int = 3, - db=None, - current_session_id: str = None, - # Scroll shape - session_id: str = None, - around_message_id: int = None, - window: int = 5, - # Discovery shape - sort: str = None, - # Cross-profile (any shape) - profile: str = None, - # Discovery result shaping (appended to preserve positional compatibility) - detail: str = "adaptive", -) -> str: - """Run session search and close databases opened by this invocation.""" +def session_search(query: str = "", role_filter: str = None, limit: int = 3, db=None, + current_session_id: str = None, session_id: str = None, around_message_id: int = None, + window: int = 5, sort: str = None, profile: str = None, detail: str = "adaptive") -> str: + """Run session search and close databases opened by this invocation. + Parameter order is positional-compatible with older callers.""" owned_dbs: List[Any] = [] if db is None: try: @@ -1111,27 +333,13 @@ def session_search( return tool_error(format_session_db_unavailable(), success=False) try: - return _session_search_impl( - query=query, - role_filter=role_filter, - limit=limit, - db=db, - current_session_id=current_session_id, - session_id=session_id, - around_message_id=around_message_id, - window=window, - sort=sort, - profile=profile, - detail=detail, - _owned_dbs=owned_dbs, - ) + return _dispatch(query, role_filter, limit, db, current_session_id, session_id, + around_message_id, window, sort, profile, detail, owned_dbs) finally: + from hermes_state import release_or_close + for owned_db in reversed(owned_dbs): - try: - from hermes_state import release_or_close - release_or_close(owned_db) - except Exception: - logging.debug("Failed to close session_search SessionDB", exc_info=True) + _quiet(lambda: release_or_close(owned_db), None, "Failed to close session_search SessionDB") def check_session_search_requirements() -> bool: diff --git a/tools/session_search_tool_common.py b/tools/session_search_tool_common.py new file mode 100644 index 0000000000..0a0d2c44f0 --- /dev/null +++ b/tools/session_search_tool_common.py @@ -0,0 +1,230 @@ +"""Shared helpers for the session_search tool: source classification, lineage +resolution, message storage state, and response shaping. Imported by +``tools.session_search_tool`` (which re-exports the names) and +``tools.session_search_tool_discover``.""" + +import json +import logging +from datetime import datetime +from typing import Any, Dict, List, Optional, Union + +from hermes_state_common import _RESET_END_REASONS + +# Hidden from browsing/searching: integrations (HERMES_SESSION_SOURCE=tool), +# delegate subagent runs, kanban workers — not the user's history. +_HIDDEN_SESSION_SOURCES = ("kanban", "subagent", "tool") + +# Searchable but DEMOTED below interactive sessions: cron sessions' repetitive +# vocabulary dominates bare BM25 and starves out the user's own sessions +# ("recall blindness"). Demoting keeps them reachable when they're the only match. +_DEMOTED_SESSION_SOURCES = ("cron",) + +# FTS rows scanned before dedup-by-lineage — well above the handful of distinct +# sessions a query returns, so interactive matches buried under cron hits are +# still in hand for the demotion pass. +_DISCOVER_SCAN_LIMIT = 300 + +# Raw FTS rows are only a discovery-plan input; the response hydrates its own +# anchored window and bookends after lineage dedup. +_DISCOVER_SEARCH_FIELDS = ("id", "session_id", "role", "snippet", "source", "model", "session_started") + +# Generated context-compaction handoff summaries (agent/context_compressor.py); +# excluded from bookends so huge compaction payloads aren't re-introduced. +_COMPACTION_PREFIXES = ("[CONTEXT COMPACTION", "[CONTEXT SUMMARY]:") + +# /new, /reset, idle/daily expiry and CLI /new ("new_session") end the +# predecessor WITHOUT carrying its transcript forward — unlike compression +# continuations and live delegation children. Derived from the gateway set so +# this tool and the recovery fence cannot drift. +_FRESH_RESET_END_REASONS = frozenset(_RESET_END_REASONS) | {"new_session"} + + +def _quiet(fn, default, msg, *log_args, with_exc: bool = False): + """Call ``fn()``; on any exception debug-log *msg* (appending the exception + as a final ``%s`` arg when *with_exc*) and return *default*.""" + try: + return fn() + except Exception as e: + logging.debug(msg, *(log_args + (e,) if with_exc else log_args), exc_info=True) + return default + + +def _format_timestamp(ts: Union[int, float, str, None]) -> str: + """Unix timestamp (number or numeric string) or ISO string -> readable date. + "unknown" for None; str(ts) if conversion fails.""" + if ts is None: + return "unknown" + try: + value = ts + if isinstance(ts, str): + if not ts.replace(".", "").replace("-", "").isdigit(): + return ts + value = float(ts) + if isinstance(value, (int, float)): + return datetime.fromtimestamp(value).strftime("%B %d, %Y at %I:%M %p") + except (ValueError, OSError, OverflowError) as e: + logging.debug("Failed to format timestamp %s: %s", ts, e, exc_info=True) + except Exception as e: + logging.debug("Unexpected error formatting timestamp %s: %s", ts, e, exc_info=True) + return str(ts) + + +def _session_meta_block(meta: Dict[str, Any]) -> Dict[str, Any]: + """The ``session_meta`` sub-object shared by read/scroll responses.""" + return {"when": _format_timestamp(meta.get("started_at")), "source": meta.get("source"), + "model": meta.get("model"), "title": meta.get("title")} + + +def _ok(**payload) -> str: + """Serialize a successful tool result (``success`` first, then *payload* in order).""" + return json.dumps({"success": True, **payload}, ensure_ascii=False) + + +def _is_compaction_summary(content: str) -> bool: + """Return True if *content* looks like a generated compaction handoff.""" + return bool(content) and content.lstrip().startswith(_COMPACTION_PREFIXES) + + +def _resolve_to_parent(db, session_id: str) -> tuple[str, bool]: + """Walk parent_session_id to the lineage root -> ``(root_id, has_compression_hop)``. + The flag distinguishes a compression-split lineage (parent content summarised + away) from a delegation lineage (child content still visible to the parent). + Falls back to ``(session_id, False)`` on errors.""" + if not session_id: + return session_id, False + visited: set[str] = set() + cur, has_compression = session_id, False + while cur and cur not in visited: + visited.add(cur) + s = _quiet(lambda: db.get_session(cur), None, "Error resolving parent for %s: %s", cur, with_exc=True) + if not s: + break + if s.get("end_reason") == "compression": + has_compression = True + if not s.get("parent_session_id"): + break + cur = s["parent_session_id"] + return cur, has_compression + + +def _resolve_lineage(db, session_id: str) -> str: + """Return only the lineage root (ignores the compression hop).""" + return _resolve_to_parent(db, session_id)[0] + + +def _session_end_reason(db, session_id: str) -> Optional[str]: + """Return the session's ``end_reason``, or None if missing/unended/error.""" + if not session_id: + return None + try: + s = db.get_session(session_id) + return (s.get("end_reason") or None) if s else None + except Exception: + return None + + +def _session_left_live_context(db, session_id: str) -> bool: + """True when *session_id*'s transcript is no longer in anyone's live context: + ``compression`` (summarised into the continuation child) or a fresh reset + (child starts empty). Everything else stays excluded from same-lineage + recall — live delegation children (``end_reason is None``) are visible to + the parent agent, and ``branched`` parents were copied verbatim into the + branch child, so their content IS the current context.""" + end_reason = _session_end_reason(db, session_id) + return end_reason == "compression" or end_reason in _FRESH_RESET_END_REASONS + + +def _get_message_storage_state(db, message_id) -> Optional[Dict[str, Any]]: + """Return the owning session and visibility flags for *message_id*.""" + if not message_id: + return None + + def _lookup(): + with db._lock: + return db._conn.execute( + "SELECT session_id, active, compacted FROM messages WHERE id = ?", (message_id,) + ).fetchone() + + row = _quiet(_lookup, None, "message storage-state lookup failed for %s", message_id) + return dict(row) if row is not None else None + + +def _is_compacted_state(state: Optional[Dict[str, Any]]) -> bool: + """Compaction archives are ``active=0, compacted=1`` (content summarised + away by archive_and_compact). Rewind/undo rows are ``active=0, compacted=0`` + and must stay hidden.""" + return state is not None and state["active"] == 0 and state["compacted"] == 1 + + +def _is_compacted_message(db, message_id) -> bool: + """True if *message_id* is a compaction-archived row — pre-compaction content + no longer in live context, so it should stay discoverable even on the + current session. False on any error (caller falls back to skipping).""" + return _is_compacted_state(_get_message_storage_state(db, message_id)) + + +def _annotate_rebuild_status(db, payload: Dict[str, Any]) -> None: + """Add a rebuild-progress note while the deferred FTS backfill is running, + so the agent can explain thin/slow results instead of treating them as + ground truth. No-op (never raises) when no rebuild is pending.""" + try: + status = db.fts_rebuild_status() + except Exception: + return + if status is None: + return + payload["index_rebuild"] = {"percent": status["percent"], "note": ( + f"The search index is rebuilding in the background ({status['percent']}% done, " + f"{status['indexed']:,} of {status['total']:,} messages). Results from older messages " + f"may be incomplete until it finishes." + )} + + +def _order_for_recall(raw_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Stable-sort FTS rows so interactive sessions rank above automation. + BM25 order is preserved within each class; only cross-class order changes, + so a cron hit never displaces an interactive hit during lineage dedup.""" + return sorted(raw_results, key=lambda r: 1 if (r.get("source") or "") in _DEMOTED_SESSION_SOURCES else 0) + + +def _shape_message(m: Dict[str, Any], anchor_id: Optional[int] = None, + max_content_len: Optional[int] = None) -> Dict[str, Any]: + """Slim a message row for the tool response. Keeps content even if empty + (absent content is meaningful — tool-call-only assistant turns). With + *max_content_len*, content is truncated and ``content_truncated`` / + ``original_content_chars`` added.""" + content = m.get("content") + if isinstance(content, str) and "\x1b" in content: + # Recalled messages can carry ANSI escapes (archived terminal output). + from tools.ansi_strip import strip_ansi + + content = strip_ansi(content) + original_chars = None + if max_content_len and content and len(content) > max_content_len: + original_chars = len(content) + content = content[:max_content_len] + "…" + entry = {"id": m.get("id"), "role": m.get("role"), "content": content, "timestamp": m.get("timestamp")} + entry.update({k: m.get(k) for k in ("tool_name", "tool_calls", "tool_call_id") if m.get(k)}) + if anchor_id is not None and m.get("id") == anchor_id: + entry["anchor"] = True + if original_chars is not None: + entry["content_truncated"] = True + entry["original_content_chars"] = original_chars + return {k: v for k, v in entry.items() if v is not None or k == "content"} + + +def _session_link(session_id: str, profile: str = None) -> str: + """The reference the agent writes to point the user at a session — same + value the desktop composer emits, so it renders as a titled link. The + profile segment is omitted when it can't be named confidently (a bare id + still resolves, it just can't disambiguate across profiles).""" + name = (profile or "").strip() + if not name: + def _active(): + from hermes_cli.profiles import get_active_profile_name + + resolved = get_active_profile_name() + return "" if resolved == "custom" else resolved + + name = _quiet(_active, "", "get_active_profile_name failed for session link") + return f"@session:{name}/{session_id}" if name else f"@session:{session_id}" diff --git a/tools/session_search_tool_discover.py b/tools/session_search_tool_discover.py new file mode 100644 index 0000000000..08ba526421 --- /dev/null +++ b/tools/session_search_tool_discover.py @@ -0,0 +1,192 @@ +"""Discovery shape of session_search: FTS5 query, title match, lineage dedup +and adaptive/full hydration of the surviving results.""" + +import json +import logging +from typing import Any, Dict, List, Optional + +from tools.registry import tool_error +from tools.session_search_tool_common import ( + _DISCOVER_SCAN_LIMIT, _DISCOVER_SEARCH_FIELDS, _HIDDEN_SESSION_SOURCES, _annotate_rebuild_status, + _format_timestamp, _is_compacted_message, _is_compaction_summary, _order_for_recall, _quiet, + _resolve_lineage, _resolve_to_parent, _session_left_live_context, _session_link, _shape_message, +) + + +def _normalize_title_query(query: str) -> str: + """Strip common quoting the model may include around a remembered title.""" + return query.strip().strip("`'\"") + + +def _title_match_result(db, query: str, current_lineage_root: Optional[str]) -> Optional[Dict[str, Any]]: + """Return a discovery-shaped result when the query matches a session title.""" + title_query = _normalize_title_query(query) + if not title_query: + return None + session_id = _quiet(lambda: db.resolve_session_by_title(title_query), None, + "resolve_session_by_title failed for %r", title_query) + if not session_id: + return None + lineage_root = _resolve_lineage(db, session_id) + # Same-lineage title hits are in-context only while the session is live; + # /new-reset and compression-ended parents are not. + if ( + current_lineage_root + and lineage_root == current_lineage_root + and not _session_left_live_context(db, session_id) + ): + return None + + session_meta = _quiet(lambda: db.get_session(lineage_root) or db.get_session(session_id), None, + "get_session failed for title match %s", session_id) or {} + if session_meta.get("source") in _HIDDEN_SESSION_SOURCES: + return None + messages = _quiet(lambda: db.get_messages(session_id), [], "get_messages failed for title match %s", session_id) + anchor_id = messages[0].get("id") if messages else None + view = {} + if anchor_id is not None: + view = _quiet(lambda: db.get_anchored_view(session_id, anchor_id, window=5, bookend=3), {}, + "get_anchored_view failed for title match %s/%s", session_id, anchor_id) + entry = { + "session_id": session_id, "when": _format_timestamp(session_meta.get("started_at")), + "source": session_meta.get("source", "unknown"), "model": session_meta.get("model") or "unknown", + "title": session_meta.get("title") or title_query, "matched_role": "session_title", + "match_message_id": anchor_id, + "snippet": f"Session title matched: {session_meta.get('title') or title_query}", + "bookend_start": [_shape_message(m) for m in (view.get("bookend_start") or messages[:3])], + "messages": [_shape_message(m, anchor_id=anchor_id) for m in (view.get("window") or messages[:5])], + "bookend_end": [_shape_message(m) for m in (view.get("bookend_end") or messages[-3:])], + "messages_before": view.get("messages_before", 0), + "messages_after": view.get("messages_after", max(len(messages) - 5, 0)), + "detail": "full", "_lineage_root": lineage_root, + } + if lineage_root and lineage_root != session_id: + entry["parent_session_id"] = lineage_root + return entry + + +def _discover_payload(db, query: str, detail: str, results: list, **extra) -> str: + payload = {"success": True, "mode": "discover", "query": query, "detail": detail, + "results": results, "count": len(results), **extra} + _annotate_rebuild_status(db, payload) + return json.dumps(payload, ensure_ascii=False) + + +def _dedupe_by_lineage(db, raw_results, limit, seen_sessions, current_session_id, current_lineage_root) -> None: + """Fill *seen_sessions* (lineage_root -> first surviving FTS row) up to *limit*. + + The raw owning session_id stays on the row — only it pairs validly with the + FTS match id for the anchored window. Current-lineage hits are skipped + UNLESS the transcript left live context: compression-ended session, /new- + reset predecessor (hiding it made gateway recall blind after every /new), + or an in-place compacted row on the SAME session_id. A live delegation + child has end_reason=None, so it stays excluded. + """ + for r in raw_results: + if len(seen_sessions) >= limit: + break + raw_sid = r["session_id"] + resolved_sid, _ = _resolve_to_parent(db, raw_sid) + is_compacted_hit = _is_compacted_message(db, r.get("id")) + is_ended_session = _session_left_live_context(db, raw_sid) + if ( + current_lineage_root + and resolved_sid == current_lineage_root + and not (is_ended_session or is_compacted_hit) + ): + continue + if current_session_id and raw_sid == current_session_id and not is_compacted_hit: + continue + seen_sessions.setdefault(resolved_sid, {**r, "_lineage_root": resolved_sid}) + + +def _bookend(view: Dict[str, Any], key: str) -> List[Dict[str, Any]]: + return [_shape_message(m, max_content_len=1200) for m in (view.get(key) or []) + if not _is_compaction_summary(m.get("content", ""))] + + +def _hydrate_hit(db, lineage_root: str, match_info: Dict[str, Any], result_detail: str) -> Optional[Dict[str, Any]]: + """Build one discovery result from a surviving FTS row; None if the anchored + view can't be loaded (the hit is dropped).""" + hit_sid = match_info.get("session_id") or lineage_root + msg_id = match_info.get("id") + try: + view = db.get_anchored_view(hit_sid, msg_id, window=5, bookend=3) + except Exception as e: + logging.warning("get_anchored_view failed for %s/%s: %s", hit_sid, msg_id, e, exc_info=True) + return None + session_meta = _quiet(lambda: db.get_session(lineage_root), None, "get_session failed for %s", lineage_root) or {} + full = result_detail == "full" + window_messages = view.get("window") or [] + if not full: + window_messages = [m for m in window_messages if m.get("id") == msg_id] + entry = { + "session_id": hit_sid, + "when": _format_timestamp(session_meta.get("started_at") or match_info.get("session_started")), + "source": session_meta.get("source") or match_info.get("source", "unknown"), + "model": session_meta.get("model") or match_info.get("model") or "unknown", + "title": session_meta.get("title") or None, "matched_role": match_info.get("role"), + "match_message_id": msg_id, "snippet": match_info.get("snippet") or "", + "bookend_start": _bookend(view, "bookend_start") if full else [], + "messages": [_shape_message(m, anchor_id=msg_id, max_content_len=4000) for m in window_messages], + "bookend_end": _bookend(view, "bookend_end") if full else [], + "messages_before": view.get("messages_before", 0), "messages_after": view.get("messages_after", 0), + "detail": result_detail, + } + if lineage_root and lineage_root != hit_sid: + entry["parent_session_id"] = lineage_root + return entry + + +def _discover(db, query: str, role_filter: Optional[List[str]], limit: int, sort: Optional[str], + detail: str, current_session_id: str = None, link_profile: str = None) -> str: + """Discovery shape: FTS5 plus adaptive or full result hydration.""" + role_list = role_filter if role_filter else ["user", "assistant"] + current_lineage_root = _resolve_lineage(db, current_session_id) if current_session_id else None + title_result = _title_match_result(db, query, current_lineage_root) + + try: + raw_results = db.search_messages( + query=query, role_filter=role_list, exclude_sources=list(_HIDDEN_SESSION_SOURCES), + limit=_DISCOVER_SCAN_LIMIT, offset=0, sort=sort, fields=_DISCOVER_SEARCH_FIELDS, + ) + except Exception as e: + logging.error("FTS5 search failed: %s", e, exc_info=True) + return tool_error(f"Search failed: {e}", success=False) + + # Demote cron rows below interactive ones BEFORE dedup so a high-volume + # cron corpus can't starve the user's own sessions out of the top `limit`. + raw_results = _order_for_recall(raw_results) + + if not raw_results and not title_result: + return _discover_payload(db, query, detail, [], message=( + "No matching sessions found. FTS5 ANDs all terms by default — " + "broaden with OR (`alpha OR beta`), exact-match with quoted " + "phrases, exclude with NOT, or prefix-match with `deploy*`." + )) + + seen_sessions: Dict[str, Dict[str, Any]] = {} + results = [] + if title_result: + title_lineage = title_result.pop("_lineage_root", None) + if title_lineage: + seen_sessions[title_lineage] = {"_title_only": True} + results.append(title_result) + _dedupe_by_lineage(db, raw_results, limit, seen_sessions, current_session_id, current_lineage_root) + + for lineage_root, match_info in seen_sessions.items(): + if match_info.get("_title_only"): + continue + # Adaptive: only the top-ranked result is fully hydrated. + entry = _hydrate_hit(db, lineage_root, match_info, "full" if detail == "full" or not results else "compact") + if entry is not None: + results.append(entry) + for entry in results: + entry["link"] = _session_link(entry["session_id"], link_profile) + return _discover_payload(db, query, detail, results, sessions_searched=len(seen_sessions), link_hint=( + "When referring the user to a session, write its `link` value " + "verbatim inline mid-sentence (it renders as a titled link) — never " + "as markdown, in backticks, on its own line, or next to the " + "title/id/date. To read more around a compact result, scroll: " + "session_search(session_id=..., around_message_id=match_message_id)." + )) diff --git a/tools/skills_sync.py b/tools/skills_sync.py index cdf3ab0925..2ab5adcec2 100644 --- a/tools/skills_sync.py +++ b/tools/skills_sync.py @@ -2,43 +2,29 @@ """ Skills Sync -- Manifest-based seeding and updating of bundled skills. -Copies bundled skills from the repo's skills/ directory into ~/.hermes/skills/ -and uses a manifest to track which skills have been synced and their origin hash. +Copies bundled skills from the repo's skills/ directory into ~/.hermes/skills/, +tracking each synced skill's origin hash in ~/.hermes/skills/.bundled_manifest +(v2: one "skill_name:origin_hash" line each; v1 plain-name lines auto-migrate). -Manifest format (v2): each line is "skill_name:origin_hash" where origin_hash -is the MD5 of the bundled skill at the time it was last synced to the user dir. -Old v1 manifests (plain names without hashes) are auto-migrated. - -Update logic: - - NEW skills (not in manifest): copied to user dir, origin hash recorded. - - EXISTING skills (in manifest, present in user dir): - * If bundled still matches origin hash: no update → skip without reading - the user copy. - * If bundled changed and user copy matches origin hash: safe to update. - * If bundled changed and user copy differs: user customized it → SKIP. - - DELETED by user (in manifest, absent from user dir): respected, not re-added. - - REMOVED from bundled (in manifest, gone from repo): cleaned from manifest. - -The manifest lives at ~/.hermes/skills/.bundled_manifest. +Update logic: NEW skills are copied and recorded; EXISTING skills update only +when bundled changed AND the user copy still matches the origin hash (a differing +user copy is user-customized -> SKIP); skills the user DELETED are not re-added; +skills REMOVED upstream are cleaned from the manifest. """ import hashlib -import json import logging import os import shutil +import stat import sys -from datetime import datetime, timezone -from pathlib import Path, PurePosixPath +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Iterator, List, Optional, Set, Tuple -# Force stdout/stderr to UTF-8. On non-UTF-8 Windows locales (e.g. CP936/GBK -# on zh-CN), Python's default stream encoding can't represent the checkmark / -# arrow glyphs this script prints (✓ U+2713, ↑ U+2191), raising -# UnicodeEncodeError mid-run. The bootstrap installer (install.ps1) captures -# this script's stdout and parses it as UTF-8; a GBK byte stream then surfaces -# as "stream did not contain valid UTF-8" and aborts the config-templates -# stage even though the script itself exits 0. Reconfigure unconditionally so -# output is valid UTF-8 regardless of the active codepage or caller. +# Force stdout/stderr to UTF-8: on GBK-style Windows locales the default encoding +# can't represent the glyphs printed here (✓ ↑ →) and raises mid-run; install.ps1 +# also parses this script's stdout as UTF-8 and aborts on a GBK byte stream. for _stream in (sys.stdout, sys.stderr): if hasattr(_stream, "reconfigure"): try: @@ -46,188 +32,122 @@ for _stream in (sys.stdout, sys.stderr): except (ValueError, TypeError): pass from hermes_constants import get_bundled_skills_dir, get_hermes_home, get_optional_skills_dir -from agent.skill_utils import is_excluded_skill_path -from typing import Dict, List, Optional, Set, Tuple -from utils import atomic_replace, atomic_write_text +from agent.skill_utils import ESSENTIAL_SKILLS, is_excluded_skill_path +from utils import atomic_write_text logger = logging.getLogger(__name__) - HERMES_HOME = get_hermes_home() SKILLS_DIR = HERMES_HOME / "skills" MANIFEST_FILE = SKILLS_DIR / ".bundled_manifest" -# Import-time snapshots backing the call-time accessors below. Same bug class -# and same fix as skills_tool (f8723c478) and skill_manager_tool (c6a3d412d): -# long-lived multi-profile runtimes (Dashboard console, TUI/Desktop backend, -# cron, kanban workers) import this module once under the launch HERMES_HOME -# and later scope requests to a different profile via -# set_hermes_home_override(). Frozen module constants would then resolve — -# and for reset_bundled_skill() DELETE — against the wrong profile's skills -# root (#65828). The accessors honor an explicitly patched module global -# (tests, and web_server's _profile_scope retargeting) and otherwise -# re-resolve from the live profile-scoped HERMES_HOME on every call. +# Import-time snapshots backing the call-time accessors below. Long-lived +# multi-profile runtimes import this module once and later retarget HERMES_HOME +# via set_hermes_home_override(); frozen constants would then resolve (and for +# reset_bundled_skill() DELETE) against the wrong profile. The accessors honor +# an explicitly patched module global and otherwise re-resolve on every call. _HERMES_HOME_AT_IMPORT = HERMES_HOME _SKILLS_DIR_AT_IMPORT = SKILLS_DIR _MANIFEST_FILE_AT_IMPORT = MANIFEST_FILE +def _live(configured, at_import: Path, fallback) -> Path: + """The patched module global if it changed since import, else the live value.""" + return Path(configured) if Path(configured) != at_import else fallback() + + def _hermes_home() -> Path: - """Return the active profile's HERMES_HOME at call time.""" - configured = Path(HERMES_HOME) - if configured != _HERMES_HOME_AT_IMPORT: - return configured - return get_hermes_home() + return _live(HERMES_HOME, _HERMES_HOME_AT_IMPORT, get_hermes_home) def _skills_dir() -> Path: - """Return the active profile's skills directory at call time.""" - configured = Path(SKILLS_DIR) - if configured != _SKILLS_DIR_AT_IMPORT: - return configured - return _hermes_home() / "skills" + return _live(SKILLS_DIR, _SKILLS_DIR_AT_IMPORT, lambda: _hermes_home() / "skills") def _manifest_file() -> Path: - """Return the active profile's bundled-skills manifest at call time.""" - configured = Path(MANIFEST_FILE) - if configured != _MANIFEST_FILE_AT_IMPORT: - return configured - return _skills_dir() / ".bundled_manifest" + return _live(MANIFEST_FILE, _MANIFEST_FILE_AT_IMPORT, lambda: _skills_dir() / ".bundled_manifest") -# Marker file written by `hermes profile create --no-skills` (named profiles) -# and by the installer's `--no-skills` flag (the default ~/.hermes profile). -# When present in HERMES_HOME, sync_skills() is a no-op so neither the -# installer, `hermes update`, nor a direct sync re-injects bundled skills. -# Delete the file to opt back in. Mirrors -# hermes_cli.profiles.NO_BUNDLED_SKILLS_MARKER (kept as a literal here to -# avoid importing the CLI layer into this low-level sync module). + +# Written by `hermes profile create --no-skills` / the installer's `--no-skills`; +# when present in HERMES_HOME, sync_skills() seeds only the essential skills. Mirrors +# hermes_cli.profiles.NO_BUNDLED_SKILLS_MARKER (literal: no CLI import in this module). NO_BUNDLED_SKILLS_MARKER = ".no-bundled-skills" -def _essential_names() -> frozenset: - """Names of skills that must always exist (see skill_utils.ESSENTIAL_SKILLS).""" - try: - from agent.skill_utils import ESSENTIAL_SKILLS - return ESSENTIAL_SKILLS - except Exception: - return frozenset({"hermes-agent"}) - - -def _get_bundled_dir() -> Path: - """Locate the bundled skills/ directory. - - Checks HERMES_BUNDLED_SKILLS env var first (set by Nix wrapper), - then falls back to the relative path from this source file. - """ +def _get_bundled_dir() -> Path: # HERMES_BUNDLED_SKILLS env first, then repo-relative return get_bundled_skills_dir(Path(__file__).parent.parent / "skills") def _get_optional_dir() -> Path: - """Locate the official optional-skills/ directory.""" return get_optional_skills_dir(Path(__file__).parent.parent / "optional-skills") -def _build_external_skill_index() -> Set[str]: - """Index every skill available in external_dirs by name and frontmatter name. +def _iter_active_skill_mds(sort: bool = False) -> Iterator[Path]: + """Yield every non-excluded SKILL.md in the user's skills tree.""" + root = _skills_dir() + found = root.rglob("SKILL.md") if root.exists() else iter(()) + for skill_md in sorted(found) if sort else found: + if not is_excluded_skill_path(skill_md): + yield skill_md - Returns a set of skill names that are already provided by external dirs. - Used to prevent sync_skills from shadowing externally-delegated skills. - """ + +def _build_external_skill_index() -> Set[str]: + """Names (directory and frontmatter) of every skill provided by external_dirs, + so sync_skills never shadows an externally-delegated skill.""" try: from agent.skill_utils import get_external_skills_dirs, _external_dirs_cache_clear except ImportError: return set() - # Clear the external dirs cache so a config edit (or a test patch) is seen. - _external_dirs_cache_clear() - + _external_dirs_cache_clear() # so a config edit (or a test patch) is seen external_names: Set[str] = set() for ext_dir in get_external_skills_dirs(): for skill_md in ext_dir.rglob("SKILL.md"): - if is_excluded_skill_path(skill_md): - continue - skill_dir = skill_md.parent - # Index by directory name (how _find_skill resolves skills) - external_names.add(skill_dir.name) - # Also index by frontmatter name (alternate identifier) - frontmatter_name = _read_skill_name(skill_md, "") - if frontmatter_name: - external_names.add(frontmatter_name) + if not is_excluded_skill_path(skill_md): + external_names.update({skill_md.parent.name, _read_skill_name(skill_md, "")}) + external_names.discard("") return external_names def _read_manifest() -> Dict[str, str]: - """ - Read the manifest as a dict of {skill_name: origin_hash}. - - Handles both v1 (plain names) and v2 (name:hash) formats. - v1 entries get an empty hash string which triggers migration on next sync. - """ - if not _manifest_file().exists(): - return {} + """Read the manifest as ``{skill_name: origin_hash}``; v1 plain-name lines get an + empty hash, which triggers migration on the next sync.""" try: - result = {} - for line in _manifest_file().read_text(encoding="utf-8").splitlines(): - line = line.strip() - if not line: - continue - if ":" in line: - # v2 format: name:hash - name, _, hash_val = line.partition(":") - result[name.strip()] = hash_val.strip() - else: - # v1 format: plain name — empty hash triggers migration - result[line] = "" - return result + lines = _manifest_file().read_text(encoding="utf-8").splitlines() if _manifest_file().exists() else [] except (OSError, IOError): return {} + result = {} + for line in map(str.strip, lines): + if line: + name, _, hash_val = line.partition(":") + result[name.strip()] = hash_val.strip() + return result def _read_suppressed_names() -> set: - """Built-in skills the curator pruned — must NOT be re-seeded on sync. - - Delegates to ``tools.skill_usage`` (single source of truth) and falls back - to reading ``~/.hermes/skills/.curator_suppressed`` directly if that import - is unavailable in a packaged/update context. - """ + """Built-in skills the curator pruned — must NOT be re-seeded. Delegates to + ``tools.skill_usage`` (source of truth), falling back to reading ``.curator_suppressed`` + directly if that import is unavailable in a packaged/update context.""" try: from tools.skill_usage import read_suppressed_names return read_suppressed_names() except Exception: path = _skills_dir() / ".curator_suppressed" - if not path.exists(): - return set() - names = set() try: - for line in path.read_text(encoding="utf-8").splitlines(): - line = line.strip() - if line and not line.startswith("#"): - names.add(line) + lines = path.read_text(encoding="utf-8").splitlines() if path.exists() else [] except OSError: - pass - return names + return set() + return {line.strip() for line in lines if line.strip() and not line.strip().startswith("#")} def _write_manifest(entries: Dict[str, str]): - """Write the manifest file atomically in v2 format (name:hash). - - Uses the shared atomic writer so an existing manifest's permission - bits (and owner, best-effort) survive the replace instead of being - reset to mkstemp's 0600 — the same mode-preservation contract as the - skill manager's document writes. - """ + """Write the manifest atomically in v2 format, preserving an existing file's + permission bits/owner instead of resetting them to mkstemp's 0600.""" _manifest_file().parent.mkdir(parents=True, exist_ok=True) data = "\n".join(f"{name}:{hash_val}" for name, hash_val in sorted(entries.items())) + "\n" - try: - atomic_write_text( - _manifest_file(), - data, - tmp_prefix=".bundled_manifest_", - preserve_mode=True, - ) + atomic_write_text(_manifest_file(), data, tmp_prefix=".bundled_manifest_", preserve_mode=True) except Exception as e: logger.debug("Failed to write skills manifest %s: %s", _manifest_file(), e, exc_info=True) @@ -239,819 +159,341 @@ def _read_skill_name(skill_md: Path, fallback: str) -> str: except OSError: return fallback in_frontmatter = False - for line in content.split("\n"): - stripped = line.strip() + for stripped in map(str.strip, content.split("\n")): if stripped == "---": if in_frontmatter: break in_frontmatter = True - continue - if in_frontmatter and stripped.startswith("name:"): - value = stripped.split(":", 1)[1].strip().strip("\"'") - if value: + elif in_frontmatter and stripped.startswith("name:"): + if value := stripped.split(":", 1)[1].strip().strip("\"'"): return value return fallback def _discover_bundled_skills(bundled_dir: Path) -> List[Tuple[str, Path]]: - """ - Find all SKILL.md files in the bundled directory. - Returns list of (skill_name, skill_directory_path) tuples. - """ - skills = [] + """``(skill_name, skill_dir)`` for every SKILL.md under the bundled dir. Exclusions + are evaluated relative to the bundled tree: the install prefix itself may contain + ``venv``/``site-packages``, which once made every wheel install discover zero skills.""" if not bundled_dir.exists(): - return skills - - for skill_md in bundled_dir.rglob("SKILL.md"): - # Exclusions apply inside the bundled tree. The install prefix itself - # may legitimately contain names such as ``venv`` or ``site-packages``; - # treating those parent components as skill content makes every wheel - # install discover zero bundled skills. - if is_excluded_skill_path( - skill_md.relative_to(bundled_dir), root=bundled_dir - ): - continue - skill_dir = skill_md.parent - skill_name = _read_skill_name(skill_md, skill_dir.name) - skills.append((skill_name, skill_dir)) - - return skills + return [] + return [ + (_read_skill_name(md, md.parent.name), md.parent) + for md in bundled_dir.rglob("SKILL.md") + if not is_excluded_skill_path(md.relative_to(bundled_dir), root=bundled_dir) + ] def _compute_relative_dest(skill_dir: Path, bundled_dir: Path) -> Path: - """ - Compute the destination path in the skills dir preserving the category structure. - e.g., bundled/skills/mlops/axolotl -> ~/.hermes/skills/mlops/axolotl - """ - rel = skill_dir.relative_to(bundled_dir) - return _skills_dir() / rel + """Destination in the skills dir preserving category structure + (bundled/skills/mlops/axolotl -> ~/.hermes/skills/mlops/axolotl).""" + return _skills_dir() / skill_dir.relative_to(bundled_dir) def _dir_hash(directory: Path) -> str: - """Compute a hash of all file contents in a directory for change detection.""" + """MD5 over relative paths + contents of every file in a directory.""" hasher = hashlib.md5() try: for fpath in sorted(directory.rglob("*")): if fpath.is_file(): - rel = fpath.relative_to(directory) - hasher.update(str(rel).encode("utf-8")) + hasher.update(str(fpath.relative_to(directory)).encode("utf-8")) hasher.update(fpath.read_bytes()) except (OSError, IOError): pass return hasher.hexdigest() -def _safe_rel_install_path(path: Path, base: Path) -> str: - """Return a normalized relative POSIX path, rejecting traversal/absolute paths.""" - rel = path.relative_to(base) - posix = rel.as_posix() - pure = PurePosixPath(posix) - parts = [part for part in pure.parts if part not in {"", "."}] - if pure.is_absolute() or not parts or any(part == ".." for part in parts): - raise ValueError(f"Unsafe optional skill path: {posix}") - return "/".join(parts) - - -def _skill_file_list(skill_dir: Path) -> List[str]: - """List files inside a skill directory in lock-file format.""" - files: List[str] = [] - for fpath in sorted(skill_dir.rglob("*")): - if fpath.is_file(): - files.append(fpath.relative_to(skill_dir).as_posix()) - return files - - -def _content_hash(directory: Path) -> str: - """Return the same hash style the skills hub lock uses, falling back locally.""" - try: - from tools.skills_guard import content_hash - - return content_hash(directory) - except Exception: - # Hashing is provenance metadata only; keep sync resilient if guard - # dependencies are unavailable in a packaged/update context. - return _dir_hash(directory) - - -def _optional_skill_index() -> Dict[str, Tuple[str, str, Path]]: - """Return official optional skills keyed by folder name and frontmatter name. - - Values are ``(folder_name, install_path, source_dir)``. Multiple keys may - point to the same skill so callers can accept either the folder slug used - by the hub lock or the user-facing frontmatter name. - """ - optional_dir = _get_optional_dir() - index: Dict[str, Tuple[str, str, Path]] = {} - if not optional_dir.exists(): - return index - for skill_md in sorted(optional_dir.rglob("SKILL.md")): - if is_excluded_skill_path( - skill_md.relative_to(optional_dir), root=optional_dir - ): - continue - src = skill_md.parent - try: - install_path = _safe_rel_install_path(src, optional_dir) - except ValueError: - continue - folder_name = src.name - frontmatter_name = _read_skill_name(skill_md, folder_name) - value = (folder_name, install_path, src) - index[folder_name] = value - index[frontmatter_name] = value - return index - - -def _move_to_restore_backup(path: Path, backup_root: Path) -> str: - """Move an existing skill directory into a restore backup, preserving rel path.""" - rel = path.relative_to(_skills_dir()) - target = backup_root / rel - target.parent.mkdir(parents=True, exist_ok=True) - if target.exists(): - suffix = 1 - while target.with_name(f"{target.name}-{suffix}").exists(): - suffix += 1 - target = target.with_name(f"{target.name}-{suffix}") - shutil.move(str(path), str(target)) - return rel.as_posix() - - -def restore_official_optional_skill(name: str, *, restore: bool = False) -> dict: - """Restore one or all official optional skills from repo source. - - ``restore=False`` only performs exact-match provenance backfill. ``restore=True`` - repairs already-mutated/reorganized skills by backing up matching active - copies and copying the official optional source into its canonical path. - """ - index = _optional_skill_index() - if not index: - return {"ok": False, "message": "No official optional skills directory found.", "restored": [], "backfilled": [], "backed_up": []} - - targets = sorted(set(index.values()), key=lambda item: item[1]) if name in {"all", "*"} else [] - if not targets: - target = index.get(name) - if target is None: - return {"ok": False, "message": f"Official optional skill not found: {name}", "restored": [], "backfilled": [], "backed_up": []} - targets = [target] - - restored: List[str] = [] - backed_up: List[str] = [] - timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - backup_root = _skills_dir() / ".restore-backups" / f"official-optional-{timestamp}" - - for folder_name, install_path, src in targets: - dest = _skills_dir() / Path(*install_path.split("/")) - src_hash = _dir_hash(src) - canonical_ok = dest.exists() and _dir_hash(dest) == src_hash - - # Find already-active copies of this official skill by frontmatter name - # or folder slug, even if curator moved it into another category. - src_frontmatter = _read_skill_name(src / "SKILL.md", folder_name) - matches: List[Path] = [] - if _skills_dir().exists(): - for skill_md in sorted(_skills_dir().rglob("SKILL.md")): - if is_excluded_skill_path(skill_md): - continue - candidate = skill_md.parent - try: - candidate.relative_to(_skills_dir()) - except ValueError: - continue - candidate_name = _read_skill_name(skill_md, candidate.name) - if candidate == dest: - continue - if candidate.name == folder_name or candidate_name in {folder_name, src_frontmatter}: - matches.append(candidate) - - if restore: - for match in matches: - if match.exists(): - backed_up.append(_move_to_restore_backup(match, backup_root)) - if dest.exists() and not canonical_ok: - backed_up.append(_move_to_restore_backup(dest, backup_root)) - if not dest.exists(): - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(src, dest) - restored.append(folder_name) - elif not canonical_ok: - continue - - backfilled = _backfill_optional_provenance(quiet=True) - return { - "ok": True, - "message": "Official optional skill repair complete.", - "restored": restored, - "backfilled": backfilled, - "backed_up": backed_up, - "backup_dir": str(backup_root) if backed_up else "", - } - - -def _index_installed_skill_dirs_by_name() -> Dict[str, List[Path]]: - """Index installed skills by directory name with one active-tree scan.""" - index: Dict[str, List[Path]] = {} - if not _skills_dir().exists(): - return index - for skill_md in _skills_dir().rglob("SKILL.md"): - if is_excluded_skill_path(skill_md): - continue - candidate = skill_md.parent - # Never reach outside the skills tree (symlinked/external dirs). - try: - candidate.resolve().relative_to(_skills_dir().resolve()) - except (OSError, ValueError): - continue - index.setdefault(candidate.name, []).append(candidate) - return index - - -def _find_installed_skill_dir_by_name( - skill_dir_name: str, - installed_index: Optional[Dict[str, List[Path]]] = None, -) -> Optional[Path]: - """Locate an installed skill directory by its directory name. - - Used only as a fallback when the repo-derived install path doesn't exist in - the active tree (upstream recategorized the skill after it was installed). - Returns None when there is no match, or when the name is AMBIGUOUS — two - skills sharing a directory name give us no basis to pick one, and guessing - would write provenance onto the wrong skill. The caller still verifies a - byte-identical content hash before recording anything. - """ - if not skill_dir_name or not _skills_dir().exists(): - return None - if installed_index is None: - installed_index = _index_installed_skill_dirs_by_name() - matches = installed_index.get(skill_dir_name, []) - if len(matches) != 1: - return None - return matches[0] - - -def _backfill_optional_provenance(quiet: bool = False) -> List[str]: - """Mark already-present official optional skills as hub-installed. - - This covers the migration case where a skill used to be bundled (or was - manually copied into the active skills tree) and later lives under - optional-skills/. If the active copy is byte-identical to the official - optional source, record official hub provenance without copying or - reinstalling anything. Modified/local skills are left alone. - """ - optional_dir = _get_optional_dir() - if not optional_dir.exists(): - return [] - - lock_path = _skills_dir() / ".hub" / "lock.json" - try: - data = json.loads(lock_path.read_text(encoding="utf-8")) if lock_path.exists() else {"version": 1, "installed": {}} - except (json.JSONDecodeError, OSError): - data = {"version": 1, "installed": {}} - installed = data.setdefault("installed", {}) - existing_paths = { - entry.get("install_path") - for entry in installed.values() - if isinstance(entry, dict) - } - - backfilled: List[str] = [] - changed = False - installed_dir_index: Optional[Dict[str, List[Path]]] = None - for skill_md in sorted(optional_dir.rglob("SKILL.md")): - if is_excluded_skill_path(skill_md): - continue - src = skill_md.parent - try: - install_path = _safe_rel_install_path(src, optional_dir) - except ValueError as e: - logger.debug("Skipping optional skill with unsafe path %s: %s", src, e) - continue - lock_name = src.name - if lock_name in installed or install_path in existing_paths: - continue - dest = _skills_dir() / Path(*install_path.split("/")) - if not dest.exists() or not dest.is_dir(): - # The active tree may hold the same skill under a DIFFERENT - # category path than the repo uses — categories get reorganized - # upstream (e.g. mlops/chroma → mlops/vector-databases/chroma) - # while the already-installed copy keeps its old location. A - # path-only lookup misses every one of those, so provenance repair - # silently skips them forever. Fall back to a unique - # same-directory-name match anywhere in the tree, then still - # require a byte-identical hash below before claiming provenance. - if installed_dir_index is None: - installed_dir_index = _index_installed_skill_dirs_by_name() - dest = _find_installed_skill_dir_by_name(src.name, installed_dir_index) - if dest is None: - continue - try: - install_path = _safe_rel_install_path(dest, _skills_dir()) - except ValueError as e: - logger.debug("Skipping relocated optional skill %s: %s", dest, e) - continue - if install_path in existing_paths: - continue - if _dir_hash(dest) != _dir_hash(src): - continue - - timestamp = datetime.now(timezone.utc).isoformat() - installed[lock_name] = { - "source": "official", - "identifier": f"official/{install_path}", - "trust_level": "builtin", - "scan_verdict": "backfilled", - "content_hash": _content_hash(dest), - "install_path": install_path, - "files": _skill_file_list(dest), - "metadata": {"backfilled_from": "optional-skills"}, - "installed_at": timestamp, - "updated_at": timestamp, - } - existing_paths.add(install_path) - backfilled.append(lock_name) - changed = True - if not quiet: - print(f" = {lock_name} (official optional provenance backfilled)") - - if changed: - lock_path.parent.mkdir(parents=True, exist_ok=True) - # Atomic write so a crash mid-write can't silently wipe all provenance - # via the JSONDecodeError fallback above (which resets `installed` to - # an empty dict). - import tempfile - - payload = json.dumps(data, indent=2, ensure_ascii=False) + "\n" - fd, tmp_path = tempfile.mkstemp( - dir=str(lock_path.parent), - prefix=".lock_", - suffix=".tmp", - ) - try: - with os.fdopen(fd, "w", encoding="utf-8") as f: - f.write(payload) - f.flush() - os.fsync(f.fileno()) - atomic_replace(tmp_path, lock_path) - except BaseException: - try: - os.unlink(tmp_path) - except OSError: - pass - raise - return backfilled - - -def _read_hub_install_paths() -> Set[str]: - """Return install paths recorded in the skills-hub lock, as POSIX strings. - - Hub-installed skills are owned by the hub (``hermes skills uninstall``), - never by bundled sync. Rename recovery must not move them even when their - content happens to match a bundled origin hash, or the lock's - ``install_path`` would point at a directory that no longer exists. - """ - lock_path = _skills_dir() / ".hub" / "lock.json" - if not lock_path.exists(): - return set() - try: - data = json.loads(lock_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - return set() - paths: Set[str] = set() - for entry in (data.get("installed") or {}).values(): - if isinstance(entry, dict): - install_path = entry.get("install_path") - if install_path: - paths.add(str(install_path).strip("/")) - return paths - - def _index_active_skills() -> Dict[str, List[Path]]: - """Index every skill in the user's tree by frontmatter name. - - Returns ``{skill_name: [skill_dir, ...]}``. Used by rename recovery to - locate a bundled skill that upstream moved to a new category/directory. - """ + """Index every skill in the user's tree by frontmatter name (for rename recovery).""" index: Dict[str, List[Path]] = {} - if not _skills_dir().exists(): - return index - for skill_md in _skills_dir().rglob("SKILL.md"): - if is_excluded_skill_path(skill_md): - continue - skill_dir = skill_md.parent - name = _read_skill_name(skill_md, skill_dir.name) - index.setdefault(name, []).append(skill_dir) + for md in _iter_active_skill_mds(): + index.setdefault(_read_skill_name(md, md.parent.name), []).append(md.parent) return index -def _recover_renamed_skill( - skill_name: str, - origin_hash: str, - dest: Path, - active_index: Dict[str, List[Path]], - hub_paths: Set[str], - quiet: bool, -) -> Optional[str]: - """Move a bundled skill's stale copy to its new canonical path. +def _rel_skills_posix(path: Path) -> str: + return path.relative_to(_skills_dir()).as_posix() - When upstream RENAMES or RECATEGORIZES a bundled skill, the manifest key - (frontmatter name) still matches but ``dest`` is a brand-new path that does - not exist yet. Without recovery, ``sync_skills()`` falls through to its - "in manifest but not on disk" branch and misreads the skill as - *user-deleted*: the old directory is stranded forever and never receives - another update. - A stale copy is only moved when it is byte-identical to ``origin_hash`` — - the hash recorded the last time sync wrote that skill — which proves the - directory is the copy *we* placed there rather than the user's own work. - Anything else (user-edited, hub-installed) is left untouched. +def _move_dir(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dest)) - Returns the relative source path when a move happened, else ``None``. + +def _copy_dir(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dest) + + +def _recover_renamed_skill(st: "_SyncState", skill_name: str, dest: Path) -> Optional[str]: + """Move a bundled skill's stale copy to its new canonical path after an upstream + RENAME/RECATEGORIZATION (manifest key still matches, ``dest`` doesn't exist yet; + otherwise the skill is misread as user-deleted and the old dir stranded forever). + + Only a copy byte-identical to the origin hash — proof *we* placed it — is moved; + user-edited or hub-installed copies are left. Returns the rel source path on move. """ + origin_hash = st.manifest.get(skill_name, "") if not origin_hash: return None + if st.active_index is None: + st.active_index = _index_active_skills() + st.hub_paths = _read_hub_install_paths() - for candidate in active_index.get(skill_name, []): + for candidate in st.active_index.get(skill_name, []): if candidate == dest or not candidate.is_dir(): continue try: - rel = candidate.relative_to(_skills_dir()).as_posix() + rel = _rel_skills_posix(candidate) except ValueError: continue - # Never relocate a hub-installed skill — the hub owns its path. - if rel in hub_paths: + if rel in st.hub_paths: # the hub owns its install paths continue if _dir_hash(candidate) != origin_hash: - # User customized the copy at the old path. Moving it would edit - # their work; leaving it avoids a duplicate-name collision. Warn - # so they can migrate deliberately. - if not quiet: - print( - f" ⚠ {skill_name}: upstream moved this skill to " - f"{dest.relative_to(_skills_dir()).as_posix()}, but your " - f"modified copy at {rel} was kept — it will not receive " - f"updates. Run `hermes skills reset {skill_name} --restore` " - f"to move to the new location." - ) + # Moving a customized copy would edit the user's work; leaving it + # avoids a duplicate-name collision. Warn so they migrate deliberately. + st.say( + f" ⚠ {skill_name}: upstream moved this skill to {_rel_skills_posix(dest)}, but your " + f"modified copy at {rel} was kept — it will not receive updates. " + f"Run `hermes skills reset {skill_name} --restore` to move to the new location." + ) continue try: - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(candidate), str(dest)) + _move_dir(candidate, dest) except (OSError, IOError): - logger.warning( - "Could not relocate renamed skill %s -> %s", candidate, dest, - exc_info=True, - ) + logger.warning("Could not relocate renamed skill %s -> %s", candidate, dest, exc_info=True) return None logger.info("Relocated renamed bundled skill: %s -> %s", candidate, dest) - if not quiet: - print(f" → {skill_name} (moved {rel} → {dest.relative_to(_skills_dir()).as_posix()})") + st.say(f" → {skill_name} (moved {rel} → {_rel_skills_posix(dest)})") return rel return None -def sync_skills(quiet: bool = False) -> dict: - """ - Sync bundled skills into ~/.hermes/skills/ using the manifest. +@dataclass +class _SyncState: + """Mutable accumulator threaded through one sync_skills() run.""" - Returns: - dict with keys: copied (list), updated (list), skipped (int), - user_modified (list), cleaned (list), total_bundled (int) - """ - # Opt-out: a profile (named or the default ~/.hermes) that wrote the - # .no-bundled-skills marker gets zero bundled-skill seeding — EXCEPT the - # essential skills (agent/skill_utils.ESSENTIAL_SKILLS). The - # ``hermes-agent`` skill is the agent's own operating manual and the - # system prompt always points at it, so even a Blank Slate / --no-skills - # profile keeps that one skill. Returning the empty-result shape with - # skipped_opt_out lets callers report "opted out" instead of - # "synced 0 / failed". This is the default-profile counterpart to - # seed_profile_skills()'s marker check for named profiles. + manifest: Dict[str, str] + quiet: bool + skipped: int = 0 + copied: List[str] = field(default_factory=list) + updated: List[str] = field(default_factory=list) + user_modified: List[str] = field(default_factory=list) + suppressed: List[str] = field(default_factory=list) + relocated: List[str] = field(default_factory=list) + shadowed_by_external: List[str] = field(default_factory=list) + # Rename-recovery indexes are expensive on host bind mounts: built lazily, + # only when a tracked skill is actually missing from its canonical path. + active_index: Optional[Dict[str, List[Path]]] = None + hub_paths: Set[str] = field(default_factory=set) + + def say(self, msg: str) -> None: + if not self.quiet: + print(msg) + + +def _recover_orphan_backup(dest: Path) -> None: + """If an interrupted update left the user's only copy in ``dest.bak`` with + dest gone, move it back so the skill isn't misread as user-deleted.""" + orphan = dest.with_suffix(".bak") + if not orphan.exists() or dest.exists(): + return + try: + _move_dir(orphan, dest) + logger.info("Recovered orphaned skill backup: %s", orphan) + except (OSError, IOError): + logger.warning("Could not recover orphaned skill backup %s", orphan, exc_info=True) + + +def _defer_to_external(st: _SyncState, skill_name: str, dest: Path, bundled_hash: str) -> None: + """An external_dirs source provides this skill; a local copy would be a name collision + the loader refuses to resolve. Defer for ALL manifest states and self-heal a stale local + shadow from an earlier sync — only when byte-identical (a user's own skill differs).""" + st.shadowed_by_external.append(skill_name) + st.skipped += 1 + st.say(f" ⇢ {skill_name} (deferred to external_dirs, not written to local tree)") + if dest.exists() and _dir_hash(dest) == bundled_hash: + _rmtree_writable(dest) + st.say(f" ✓ removed stale shadow of {skill_name}") + st.manifest.pop(skill_name, None) + + +def _install_new_skill(st: _SyncState, skill_name: str, skill_src: Path, dest: Path, bundled_hash: str) -> None: + """Handle a skill never offered before (not in manifest).""" + try: + if dest.exists(): + # Never overwrite a same-named user skill. Baseline the manifest only when + # byte-identical to bundled: recording bundled_hash for a differing copy + # would read as "user-modified" forever and block every bundled update. + st.skipped += 1 + if _dir_hash(dest) == bundled_hash: + st.manifest[skill_name] = bundled_hash + else: + st.say( + f" ⚠ {skill_name}: bundled version shipped but you already have a local skill by this name — " + f"yours was kept. Run `hermes skills reset {skill_name}` to replace it with the bundled version." + ) + else: + _copy_dir(skill_src, dest) + st.copied.append(skill_name) + st.manifest[skill_name] = bundled_hash + st.say(f" + {skill_name}") + except (OSError, IOError) as e: + st.say(f" ! Failed to copy {skill_name}: {e}") + # Not added to manifest — next sync retries. + + +def _replace_skill_dir(skill_src: Path, dest: Path) -> None: + """Replace ``dest`` with a fresh copy of ``skill_src`` via a ``.bak`` sibling, + restoring the original on failure.""" + backup = dest.with_suffix(".bak") + if backup.exists(): # a stale .bak would make shutil.move() nest dest INSIDE it; dest is authoritative + _rmtree_writable(backup) + shutil.move(str(dest), str(backup)) + try: + shutil.copytree(skill_src, dest) + except (OSError, IOError): + # Clear a partially-written dest so it can't shadow or block the restore. + if backup.exists() and dest.exists(): + try: + _rmtree_writable(dest) + except (OSError, IOError): + logger.warning("Could not clear partial copy %s during restore", dest, exc_info=True) + if backup.exists() and not dest.exists(): + shutil.move(str(backup), str(dest)) + raise + try: + _rmtree_writable(backup) + except (OSError, IOError): + logger.debug("Could not remove backup %s", backup, exc_info=True) + + +def _update_existing_skill(st: _SyncState, skill_name: str, skill_src: Path, dest: Path, bundled_hash: str) -> None: + """Handle a skill that is in the manifest AND on disk.""" + origin_hash = st.manifest.get(skill_name, "") + if origin_hash and bundled_hash == origin_hash: # bundled unchanged: skip without hashing the user copy + st.skipped += 1 + return + user_hash = _dir_hash(dest) + if not origin_hash: + # v1 migration: baseline from the user's copy so future syncs can detect edits. + # Can't tell user-edit from upstream change — be safe and skip. + st.manifest[skill_name] = user_hash + st.skipped += 1 + return + if _is_tracked_user_modification(origin_hash, user_hash): + st.user_modified.append(skill_name) + st.say(f" ~ {skill_name} (user-modified, skipping)") + return + # bundled changed and the user copy is pristine -> update + try: + _replace_skill_dir(skill_src, dest) + except (OSError, IOError) as e: + st.say(f" ! Failed to update {skill_name}: {e}") + return + st.manifest[skill_name] = bundled_hash + st.updated.append(skill_name) + st.say(f" ↑ {skill_name} (updated)") + + +def _seed_category_descriptions(bundled_dir: Path, only_dirs: Optional[Set[Path]]) -> None: + """Copy category DESCRIPTION.md files not already present; ``only_dirs`` + restricts seeding to the essential skills' categories on opted-out profiles.""" + for desc_md in bundled_dir.rglob("DESCRIPTION.md"): + dest_desc = _skills_dir() / desc_md.relative_to(bundled_dir) + if (only_dirs is not None and dest_desc.parent not in only_dirs) or dest_desc.exists(): + continue + try: + dest_desc.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(desc_md, dest_desc) + except (OSError, IOError) as e: + logger.debug("Could not copy %s: %s", desc_md, e) + + +def sync_skills(quiet: bool = False) -> dict: + """Sync bundled skills into ~/.hermes/skills/ using the manifest. Returns a dict + with keys copied, updated, skipped, user_modified, cleaned, suppressed, relocated, + total_bundled, optional_provenance_backfilled, shadowed_by_external, skipped_opt_out.""" + # Opted-out profiles seed ONLY ESSENTIAL_SKILLS: the system prompt always points at + # ``hermes-agent``, so even a Blank Slate profile keeps it. essential_only = (_hermes_home() / NO_BUNDLED_SKILLS_MARKER).exists() if essential_only and not quiet: - print( - " (profile opted out of bundled skills via .no-bundled-skills — " - "seeding essential skills only)" - ) - + print(" (profile opted out of bundled skills via .no-bundled-skills — seeding essential skills only)") bundled_dir = _get_bundled_dir() if not bundled_dir.exists(): - return { - "copied": [], "updated": [], "skipped": 0, - "user_modified": [], "cleaned": [], "suppressed": [], "total_bundled": 0, - "optional_provenance_backfilled": [], - } + return {"copied": [], "updated": [], "skipped": 0, "user_modified": [], "cleaned": [], + "suppressed": [], "total_bundled": 0, "optional_provenance_backfilled": []} _skills_dir().mkdir(parents=True, exist_ok=True) - manifest = _read_manifest() bundled_skills = _discover_bundled_skills(bundled_dir) if essential_only: - # Opted-out profile: only the essential skills are synced. - bundled_skills = [ - (name, src) for name, src in bundled_skills - if name in _essential_names() - ] - bundled_names = {name for name, _ in bundled_skills} + bundled_skills = [(name, src) for name, src in bundled_skills if name in ESSENTIAL_SKILLS] suppressed = _read_suppressed_names() - # Index of skills already provided by external_dirs (skip writing them) external_index = _build_external_skill_index() - shadowed_by_external: List[str] = [] - # Rename recovery indexes are expensive on host bind mounts. Build them - # only if a tracked skill is actually missing from its canonical path. - active_index: Optional[Dict[str, List[Path]]] = None - hub_paths: Optional[Set[str]] = None - - copied = [] - updated = [] - user_modified = [] - suppressed_skipped: List[str] = [] - relocated: List[str] = [] - skipped = 0 + st = _SyncState(manifest=_read_manifest(), quiet=quiet) for skill_name, skill_src in bundled_skills: - # Curator-pruned built-ins: do not re-seed. The suppression list - # (~/.hermes/skills/.curator_suppressed) is written when the curator - # archives a bundled skill with curator.prune_builtins enabled. Without - # this skip, every `hermes update` would resurrect a skill the user - # deliberately pruned. Restoring the skill clears its suppression entry. - # Essential skills are exempt — they must always come back. - if skill_name in suppressed and skill_name not in _essential_names(): - suppressed_skipped.append(skill_name) + # Curator-pruned built-ins must not resurrect on every update; essentials are exempt. + if skill_name in suppressed and skill_name not in ESSENTIAL_SKILLS: + st.suppressed.append(skill_name) continue dest = _compute_relative_dest(skill_src, bundled_dir) bundled_hash = _dir_hash(skill_src) - # Recover an orphaned backup before classifying. If a previous - # update was interrupted between moving dest aside and copying the - # new version in, the user's only copy sits in ``dest.bak`` while - # dest is gone — without this, the "in manifest but not on disk" - # branch below misreads the skill as user-deleted and it silently - # vanishes from discovery. - _orphan = dest.with_suffix(".bak") - if _orphan.exists() and not dest.exists(): - try: - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.move(str(_orphan), str(dest)) - logger.info("Recovered orphaned skill backup: %s", _orphan) - except (OSError, IOError): - logger.warning( - "Could not recover orphaned skill backup %s", _orphan, - exc_info=True, - ) - - # Recover an upstream RENAME / RECATEGORIZATION before classifying. - # The manifest key (frontmatter name) survives a directory move, but - # ``dest`` is a new path that does not exist yet — without this the - # "in manifest but not on disk" branch below misreads the skill as - # user-deleted, stranding the old copy at its stale path forever. - if not dest.exists() and skill_name in manifest: - if active_index is None: - active_index = _index_active_skills() - hub_paths = _read_hub_install_paths() - _moved_from = _recover_renamed_skill( - skill_name, - manifest.get(skill_name, ""), - dest, - active_index, - hub_paths or set(), - quiet, - ) - if _moved_from: - relocated.append(skill_name) + # Recoveries run BEFORE classification so a missing dest isn't misread as user-deleted. + _recover_orphan_backup(dest) + if not dest.exists() and skill_name in st.manifest and _recover_renamed_skill(st, skill_name, dest): + st.relocated.append(skill_name) if skill_name in external_index: - # An external_dirs source already provides this skill. Writing it - # into the profile-local tree would create a name collision the - # loader refuses to resolve (#28126). Defer to the external copy - # for ALL manifest states (new, previously-synced, user-deleted). - shadowed_by_external.append(skill_name) - skipped += 1 - if not quiet: - print( - f" ⇢ {skill_name} (deferred to external_dirs, " - "not written to local tree)" - ) - # Self-healing: a prior sync (before external_dirs was configured, - # or an older buggy sync) may have left a local shadow that now - # collides. We own that shadow only when it is byte-identical to - # the bundled source — a user's own customized skill by the same - # name differs, so never delete or re-baseline it. Drop the stale - # manifest entry so the skill isn't later misread as user-deleted. - if dest.exists() and _dir_hash(dest) == bundled_hash: - _rmtree_writable(dest) - if not quiet: - print(f" ✓ removed stale shadow of {skill_name}") - manifest.pop(skill_name, None) - continue - - if skill_name not in manifest: - # ── New skill — never offered before ── - try: - if dest.exists(): - # User already has a skill with the same name — don't overwrite. - # Only baseline in the manifest when the on-disk copy is - # byte-identical to bundled (e.g. a reset that re-syncs, or - # a coincidentally identical install); that case is harmless - # to track. If the copy differs (custom skill, hub-installed, - # or user-edited) skip the manifest write: recording - # bundled_hash there would poison update detection by making - # user_hash != origin_hash read as "user-modified" on every - # subsequent sync, permanently blocking bundled updates. - skipped += 1 - if _dir_hash(dest) == bundled_hash: - manifest[skill_name] = bundled_hash - elif not quiet: - print( - f" ⚠ {skill_name}: bundled version shipped but you " - f"already have a local skill by this name — yours " - f"was kept. Run `hermes skills reset {skill_name}` " - f"to replace it with the bundled version." - ) - else: - dest.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(skill_src, dest) - copied.append(skill_name) - manifest[skill_name] = bundled_hash - if not quiet: - print(f" + {skill_name}") - except (OSError, IOError) as e: - if not quiet: - print(f" ! Failed to copy {skill_name}: {e}") - # Do NOT add to manifest — next sync should retry - + _defer_to_external(st, skill_name, dest, bundled_hash) + elif skill_name not in st.manifest: + _install_new_skill(st, skill_name, skill_src, dest, bundled_hash) elif dest.exists(): - # ── Existing skill — in manifest AND on disk ── - origin_hash = manifest.get(skill_name, "") - - # If the bundled source still matches the version recorded when - # it was installed, there is no update to apply. Avoid recursively - # hashing the user's copy just to rediscover that fact; when the - # bundled source changes, the normal user-modification check below - # still protects local edits before any overwrite. - if origin_hash and bundled_hash == origin_hash: - skipped += 1 - continue - - user_hash = _dir_hash(dest) - - if not origin_hash: - # v1 migration: no origin hash recorded. Set baseline from - # user's current copy so future syncs can detect modifications. - manifest[skill_name] = user_hash - if user_hash == bundled_hash: - skipped += 1 # already in sync - else: - # Can't tell if user modified or bundled changed — be safe - skipped += 1 - continue - - if _is_tracked_user_modification(origin_hash, user_hash): - # User modified this skill — don't overwrite their changes - user_modified.append(skill_name) - if not quiet: - print(f" ~ {skill_name} (user-modified, skipping)") - continue - - # User copy matches origin — check if bundled has a newer version - if bundled_hash != origin_hash: - try: - # Move old copy to a backup so we can restore on failure - backup = dest.with_suffix(".bak") - # A stale backup left by an earlier failure would make - # shutil.move() nest dest *inside* it (or fail outright) - # and would poison the restore path below. The current - # dest is the authoritative copy — clear the leftover. - if backup.exists(): - _rmtree_writable(backup) - shutil.move(str(dest), str(backup)) - try: - shutil.copytree(skill_src, dest) - manifest[skill_name] = bundled_hash - updated.append(skill_name) - if not quiet: - print(f" ↑ {skill_name} (updated)") - # Remove backup after successful copy - try: - _rmtree_writable(backup) - except (OSError, IOError): - logger.debug("Could not remove backup %s", backup, exc_info=True) - except (OSError, IOError): - # Restore from backup. A partially-written dest must - # not shadow the user's copy or block the restore — - # clear it first, then move the backup home. - if backup.exists(): - if dest.exists(): - try: - _rmtree_writable(dest) - except (OSError, IOError): - logger.warning( - "Could not clear partial copy %s during restore", - dest, exc_info=True, - ) - if not dest.exists(): - shutil.move(str(backup), str(dest)) - raise - except (OSError, IOError) as e: - if not quiet: - print(f" ! Failed to update {skill_name}: {e}") - else: - skipped += 1 # bundled unchanged, user unchanged - + _update_existing_skill(st, skill_name, skill_src, dest, bundled_hash) else: - # ── In manifest but not on disk — user deleted it ── - skipped += 1 + st.skipped += 1 # in manifest but not on disk — user deleted it - # Clean stale manifest entries (skills removed from bundled dir). - # Skip on an opted-out profile: bundled_skills was filtered to the - # essential set there, and cleaning would drop tracking for every other - # previously-synced skill still on disk. - if essential_only: - cleaned = [] - else: - cleaned = sorted(set(manifest.keys()) - bundled_names) - for name in cleaned: - del manifest[name] + # Clean manifest entries for skills removed upstream. Skipped when opted out: bundled_skills + # is only the essential set there, so cleaning would drop tracking for every other skill. + cleaned = [] if essential_only else sorted(set(st.manifest) - {name for name, _ in bundled_skills}) + for name in cleaned: + del st.manifest[name] - # Also copy DESCRIPTION.md files for categories (if not already present). - # On an opted-out profile only the essential skills' own category - # descriptions are seeded — not the full catalog's. - _essential_cat_dirs = { - _compute_relative_dest(src, bundled_dir).parent - for _, src in bundled_skills - } if essential_only else None - for desc_md in bundled_dir.rglob("DESCRIPTION.md"): - rel = desc_md.relative_to(bundled_dir) - dest_desc = _skills_dir() / rel - if _essential_cat_dirs is not None and dest_desc.parent not in _essential_cat_dirs: - continue - if not dest_desc.exists(): - try: - dest_desc.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(desc_md, dest_desc) - except (OSError, IOError) as e: - logger.debug("Could not copy %s: %s", desc_md, e) - - _write_manifest(manifest) - optional_provenance_backfilled = _backfill_optional_provenance(quiet=quiet) + _seed_category_descriptions( + bundled_dir, + {_compute_relative_dest(src, bundled_dir).parent for _, src in bundled_skills} if essential_only else None, + ) + _write_manifest(st.manifest) return { - "copied": copied, - "updated": updated, - "skipped": skipped, - "user_modified": user_modified, - "cleaned": cleaned, - "suppressed": suppressed_skipped, - "relocated": relocated, + "copied": st.copied, "updated": st.updated, "skipped": st.skipped, "user_modified": st.user_modified, + "cleaned": cleaned, "suppressed": st.suppressed, "relocated": st.relocated, "total_bundled": len(bundled_skills), - "optional_provenance_backfilled": optional_provenance_backfilled, - "shadowed_by_external": shadowed_by_external, - # Opted-out profiles still seed essential skills; the flag lets - # callers report "opted out" rather than a normal full sync. - "skipped_opt_out": essential_only, + "optional_provenance_backfilled": _backfill_optional_provenance(quiet=quiet), + "shadowed_by_external": st.shadowed_by_external, + "skipped_opt_out": essential_only, # lets callers report "opted out" rather than a normal sync } def _rmtree_writable(path: Path) -> None: - """Remove a directory tree, making read-only entries writable first. + """Remove a directory tree, making read-only entries writable first (Nix/deb/rpm + sources keep r-x dirs; unlinking a child needs a writable parent, so chmod both). - Handles immutable package sources (Nix store, deb/rpm installs) that - preserve read-only permissions on copied files *and* directories - (``r-xr-xr-x``). Removing a child requires write permission on its - parent directory, so the retry handler makes the failing path **and its - parent** writable before re-attempting. See #34860, #34972. + Scope guard: refuses anything not a STRICT child of the active profile's skills + root, so a bad path join / missing HERMES_HOME / malicious manifest entry raises + a loud ValueError instead of wiping ``~/.hermes``. Callers always pass a skill dir + or its ``.bak`` sibling; the skills root itself must never be removed. """ - # Defense in depth (#48200): refuse to rmtree anything outside - # ``HERMES_HOME/skills/`` to prevent the catastrophic wipe of - # ``~/.hermes/`` (``.env``, ``MEMORY.md``, ``kanban.db``, custom - # skills, scripts, …) that an earlier incident observed. Five call - # sites in this file invoke this helper; if any one of them ever - # computes a destination outside the skills root — through a bad - # path join, a missing ``HERMES_HOME`` default, a malicious - # bundled-manifest entry, or a mid-flight exception that leaves a - # stale path in scope — this guard turns the resulting - # ``shutil.rmtree(~/.hermes)`` into a loud, recoverable ``ValueError`` - # instead of silently destroying the user's install. target = Path(path).resolve() skills_root = _skills_dir().resolve() - # Every legitimate caller passes a skill directory or its ``.bak`` - # sibling — always a strict child of the skills root. The skills root - # itself must never be removed: a ``dest`` that collapses to - # ``SKILLS_DIR`` (e.g. a relative path resolving to ``.``) would wipe - # every installed skill, and its ``.bak`` sibling lands one level up in - # ``HERMES_HOME``. Require a strict-child relationship so both escape - # into the skills root and out of it are refused. if skills_root not in target.parents: - raise ValueError( - f"refusing to rmtree {target!r}: not strictly under {skills_root!r} " - f"(scope guard — see #48200)" - ) - import stat + raise ValueError(f"refusing to rmtree {target!r}: not strictly under {skills_root!r} (scope guard — see #48200)") def _on_error(func, fpath, exc_info): - # Unlinking a child requires the parent dir to be writable, so chmod - # the parent as well as the failing path, then retry. - for target in (os.path.dirname(fpath), fpath): + for p in (os.path.dirname(fpath), fpath): try: - os.chmod(target, stat.S_IRWXU) + os.chmod(p, stat.S_IRWXU) except OSError: pass func(fpath) @@ -1059,411 +501,25 @@ def _rmtree_writable(path: Path) -> None: shutil.rmtree(path, onerror=_on_error) -def reset_bundled_skill(name: str, restore: bool = False) -> dict: - """ - Reset a bundled skill's manifest tracking so future syncs work normally. - - When a user edits a bundled skill, subsequent syncs mark it as - ``user_modified`` and skip it forever — even if the user later copies - the bundled version back into place, because the manifest still holds - the *old* origin hash. This function breaks that loop. - - Args: - name: The skill name (matches the manifest key / skill frontmatter name). - restore: If True, also delete the user's copy in the skills dir and let - the next sync re-copy the current bundled version. If False - (default), only clear the manifest entry — the user's - current copy is preserved but future updates work again. - - Returns: - dict with keys: - - ok: bool, whether the reset succeeded - - action: one of "manifest_cleared", "restored", "not_in_manifest", - "bundled_missing" - - message: human-readable description - - synced: dict from sync_skills() if a sync was triggered, else None - """ - manifest = _read_manifest() - bundled_dir = _get_bundled_dir() - bundled_skills = _discover_bundled_skills(bundled_dir) - bundled_by_name = dict(bundled_skills) - - in_manifest = name in manifest - is_bundled = name in bundled_by_name - - if not in_manifest and not is_bundled: - return { - "ok": False, - "action": "not_in_manifest", - "message": ( - f"'{name}' is not a tracked bundled skill. Nothing to reset. " - f"(Hub-installed skills use `hermes skills uninstall`.)" - ), - "synced": None, - } - - # Step 1 (optional): delete the user's copy so next sync re-copies bundled. - # Must happen BEFORE manifest deletion so that a failed rmtree does not - # leave the skill in a manifest-less limbo state (see #34972). - deleted_user_copy = False - if restore: - if not is_bundled: - return { - "ok": False, - "action": "bundled_missing", - "message": ( - f"'{name}' has no bundled source — manifest entry preserved " - f"but cannot restore from bundled (skill was removed upstream)." - ), - "synced": None, - } - dest = _compute_relative_dest(bundled_by_name[name], bundled_dir) - if dest.exists(): - try: - _rmtree_writable(dest) - deleted_user_copy = True - except (OSError, IOError) as e: - return { - "ok": False, - "action": "not_reset", - "message": ( - f"Could not delete user copy at {dest}: {e}. " - f"Manifest entry preserved — nothing was changed." - ), - "synced": None, - } - - # Step 2: drop the manifest entry so next sync treats it as new - if in_manifest: - del manifest[name] - _write_manifest(manifest) - - # Step 3: run sync to re-baseline (or re-copy if we deleted) - synced = sync_skills(quiet=True) - - if restore and deleted_user_copy: - action = "restored" - message = f"Restored '{name}' from bundled source." - elif restore: - # Nothing on disk to delete, but we re-synced — acts like a fresh install - action = "restored" - message = f"Restored '{name}' (no prior user copy, re-copied from bundled)." - else: - action = "manifest_cleared" - message = ( - f"Cleared manifest entry for '{name}'. Future `hermes update` runs " - f"will re-baseline against your current copy and accept upstream changes." - ) - - return {"ok": True, "action": action, "message": message, "synced": synced} - - -def _is_tracked_user_modification(origin_hash: str, user_hash: str) -> bool: - """Whether an on-disk skill counts as a user modification ``hermes update`` keeps. - - Shared by the sync loop (which decides what to skip) and - ``list_user_modified_bundled_skills`` (which surfaces the names) so the two - can never drift. A skill is a tracked modification only when it has a - recorded origin hash (an un-baselined / v1 entry with an empty hash is not) - and its current content hash differs from that origin. - """ - return bool(origin_hash) and user_hash != origin_hash - - -def list_user_modified_bundled_skills() -> List[dict]: - """Return the bundled skills that ``hermes update`` keeps because the user - edited them locally. - - A skill counts as user-modified when its on-disk copy no longer matches the - origin hash recorded in the manifest the last time it was synced — the exact - same test the sync loop uses to decide what to skip. This is the discovery - half of that behavior, so a user can find the names the ``~ N user-modified - (kept)`` notice only counts. - - Returns a list (sorted by name) of dicts: - ``{"name": str, "dest": Path, "bundled_src": Path}`` - where ``dest`` is the user's copy and ``bundled_src`` is the current stock - copy (so callers can diff or restore). - """ - manifest = _read_manifest() - if not manifest: - return [] - bundled_dir = _get_bundled_dir() - modified: List[dict] = [] - for skill_name, skill_dir in _discover_bundled_skills(bundled_dir): - origin_hash = manifest.get(skill_name, "") - # No entry, or a v1 entry not yet baselined (empty hash): not a tracked - # modification — the next sync handles it. - if not origin_hash: - continue - dest = _compute_relative_dest(skill_dir, bundled_dir) - if not dest.exists(): - continue - if _is_tracked_user_modification(origin_hash, _dir_hash(dest)): - modified.append( - {"name": skill_name, "dest": dest, "bundled_src": skill_dir} - ) - modified.sort(key=lambda e: e["name"]) - return modified - - -def _read_for_diff(path: Path) -> Tuple[Optional[bytes], Optional[str]]: - """Read a file once for diffing. - - Returns ``(raw_bytes, text)`` where ``text`` is ``None`` if the file is - binary; ``(None, None)`` if it could not be read. Returning the raw bytes - lets the caller compare binary files without re-reading them. - """ - try: - data = path.read_bytes() - except OSError: - return None, None - if b"\x00" in data: - return data, None - try: - return data, data.decode("utf-8") - except UnicodeDecodeError: - return data, None - - -def diff_bundled_skill(name: str) -> dict: - """Diff a user's copy of a bundled skill against the current stock version. - - Lets a user see exactly what diverged before deciding whether to keep their - edits or ``hermes skills reset`` back to upstream. - - Returns a dict: - ``ok`` (bool), ``name`` (str), ``found`` (bool — bundled source exists), - ``modified`` (bool), ``message`` (str), - ``diffs``: list of ``{"path": str, "status": str, "diff": str}`` where - status is one of ``modified`` / ``added`` (only in user copy) / - ``removed`` (only in bundled) / ``binary``. - """ - import difflib - - bundled_dir = _get_bundled_dir() - bundled_by_name = dict(_discover_bundled_skills(bundled_dir)) - bundled_src = bundled_by_name.get(name) - if bundled_src is None: - return { - "ok": False, - "name": name, - "found": False, - "modified": False, - "diffs": [], - "message": ( - f"'{name}' is not a tracked bundled skill (no stock version to " - f"diff against). Hub-installed skills use `hermes skills inspect`." - ), - } - dest = _compute_relative_dest(bundled_src, bundled_dir) - if not dest.exists(): - return { - "ok": False, - "name": name, - "found": True, - "modified": False, - "diffs": [], - "message": f"No local copy of '{name}' found at {dest}.", - } - - user_files = set(_skill_file_list(dest)) - stock_files = set(_skill_file_list(bundled_src)) - - diffs: List[dict] = [] - for rel in sorted(user_files | stock_files): - in_user = rel in user_files - in_stock = rel in stock_files - user_bytes, user_text = ( - _read_for_diff(dest / rel) if in_user else (None, None) - ) - stock_bytes, stock_text = ( - _read_for_diff(bundled_src / rel) if in_stock else (None, None) - ) - - if in_user and in_stock: - if user_text is None or stock_text is None: - # At least one side is binary — report only if bytes differ - # (reuse the bytes already read above, no second read). - if user_bytes != stock_bytes: - diffs.append( - {"path": rel, "status": "binary", "diff": ""} - ) - continue - if user_text == stock_text: - continue - text = "".join( - difflib.unified_diff( - stock_text.splitlines(keepends=True), - user_text.splitlines(keepends=True), - fromfile=f"stock/{rel}", - tofile=f"yours/{rel}", - ) - ) - diffs.append({"path": rel, "status": "modified", "diff": text}) - elif in_user: - diffs.append( - {"path": rel, "status": "added", "diff": f"+ only in your copy: {rel}"} - ) - else: - diffs.append( - {"path": rel, "status": "removed", "diff": f"- only in stock: {rel}"} - ) - - modified = bool(diffs) - return { - "ok": True, - "name": name, - "found": True, - "modified": modified, - "diffs": diffs, - "message": ( - f"'{name}' matches the stock version." - if not modified - else f"'{name}' differs from the stock version in {len(diffs)} file(s)." - ), - } - - -def set_bundled_skills_opt_out(enabled: bool) -> dict: - """Toggle the .no-bundled-skills opt-out marker for the active profile. - - When ``enabled`` is True, writes HERMES_HOME/.no-bundled-skills so the - installer, ``hermes update``, and any direct sync stop seeding bundled - skills. When False, removes the marker so seeding resumes on the next - sync. This is the on-disk-state half of ``hermes skills opt-out`` / - ``opt-in``; removal of already-present skills is a separate, explicit - step (see ``remove_pristine_bundled_skills``). - - Returns: - dict with keys: ok (bool), changed (bool), marker (str path), - message (str). - """ - marker = _hermes_home() / NO_BUNDLED_SKILLS_MARKER - existed = marker.exists() - try: - if enabled: - _hermes_home().mkdir(parents=True, exist_ok=True) - marker.write_text( - "This profile opted out of bundled-skill seeding " - "(`hermes skills opt-out`).\n" - "Delete this file to re-enable sync on the next `hermes update`.\n", - encoding="utf-8", - ) - changed = not existed - message = ( - "Opted out of bundled skills. Future install / update / sync " - "runs will not seed bundled skills into this profile." - if changed - else "Already opted out — marker was already present." - ) - else: - if existed: - marker.unlink() - changed = existed - message = ( - "Opted back in. The next `hermes update` (or `hermes skills " - "opt-in --sync`) will re-seed bundled skills." - if changed - else "Not opted out — no marker to remove." - ) - except OSError as e: - return { - "ok": False, "changed": False, "marker": str(marker), - "message": f"Could not update opt-out marker at {marker}: {e}", - } - return {"ok": True, "changed": changed, "marker": str(marker), "message": message} - - -def is_bundled_skills_opt_out() -> bool: - """Return True if the active profile carries the opt-out marker.""" - return (_hermes_home() / NO_BUNDLED_SKILLS_MARKER).exists() - - -def remove_pristine_bundled_skills(dry_run: bool = False) -> dict: - """Delete bundled skills that are present, manifest-tracked, AND unmodified. - - Safety is the whole point of this function. A skill on disk is removed - ONLY when all of these hold: - - it is recorded in the sync manifest (so it is genuinely a bundled - skill, not a hub-installed or hand-written one), AND - - it still exists in the bundled source (so we can hash-compare), AND - - its on-disk copy is byte-identical to the manifest origin hash - (so the user has not edited it). - - Anything user-modified, hub-installed, or locally authored is left - untouched and reported under ``skipped``. The manifest entry for each - removed skill is dropped so a later opt-in re-seed treats it as new. - - Args: - dry_run: When True, compute what would be removed without deleting. - - Returns: - dict with keys: ok (bool), removed (list[str]), - skipped (list[dict]) where each dict is - {name, reason}, dry_run (bool), message (str). - """ - manifest = _read_manifest() - bundled_dir = _get_bundled_dir() - bundled_by_name = dict(_discover_bundled_skills(bundled_dir)) - - removed: List[str] = [] - skipped: List[dict] = [] - - for name, origin_hash in sorted(manifest.items()): - src = bundled_by_name.get(name) - if src is None: - # Tracked but no longer bundled upstream — leave it; not ours to judge. - skipped.append({"name": name, "reason": "no bundled source (removed upstream)"}) - continue - dest = _compute_relative_dest(src, bundled_dir) - if not dest.exists(): - # Already gone from disk; just forget the stale manifest entry. - if not dry_run and name in manifest: - del manifest[name] - continue - on_disk = _dir_hash(dest) - if on_disk != origin_hash: - skipped.append({"name": name, "reason": "user-modified (kept)"}) - continue - # Pristine bundled copy — safe to remove. - if dry_run: - removed.append(name) - continue - try: - _rmtree_writable(dest) - except (OSError, IOError) as e: - skipped.append({"name": name, "reason": f"delete failed: {e}"}) - continue - if name in manifest: - del manifest[name] - removed.append(name) - - if not dry_run and removed: - _write_manifest(manifest) - - verb = "Would remove" if dry_run else "Removed" - message = f"{verb} {len(removed)} pristine bundled skill(s); kept {len(skipped)}." - return { - "ok": True, "removed": removed, "skipped": skipped, - "dry_run": dry_run, "message": message, - } +# Re-exported so ``from tools.skills_sync import X`` / ``patch("tools.skills_sync.X")`` keep working. +from tools.skills_sync_bundled_ops import ( # noqa: E402,F401 + _is_tracked_user_modification, _read_for_diff, diff_bundled_skill, list_user_modified_bundled_skills, + remove_pristine_bundled_skills, reset_bundled_skill, set_bundled_skills_opt_out, +) +from tools.skills_sync_optional import ( # noqa: E402,F401 + _backfill_optional_provenance, _content_hash, _index_installed_skill_dirs_by_name, _move_to_restore_backup, + _optional_skill_index, _read_hub_install_paths, _safe_rel_install_path, _skill_file_list, + restore_official_optional_skill, +) if __name__ == "__main__": print("Syncing bundled skills into ~/.hermes/skills/ ...") result = sync_skills(quiet=False) - parts = [ - f"{len(result['copied'])} new", - f"{len(result['updated'])} updated", - f"{result['skipped']} unchanged", - ] - if result["user_modified"]: - names = result["user_modified"] - MAX_SHOW = 5 - shown = ", ".join(names[:MAX_SHOW]) - if len(names) > MAX_SHOW: - shown += f", +{len(names) - MAX_SHOW} more" + parts = [f"{len(result['copied'])} new", f"{len(result['updated'])} updated", f"{result['skipped']} unchanged"] + names = result["user_modified"] + if names: + shown = ", ".join(names[:5]) + (f", +{len(names) - 5} more" if len(names) > 5 else "") parts.append(f"{len(names)} user-modified (kept): {shown}") if result["cleaned"]: parts.append(f"{len(result['cleaned'])} cleaned from manifest") diff --git a/tools/skills_sync_bundled_ops.py b/tools/skills_sync_bundled_ops.py new file mode 100644 index 0000000000..9e843a1d0b --- /dev/null +++ b/tools/skills_sync_bundled_ops.py @@ -0,0 +1,240 @@ +"""Bundled-skill maintenance ops: reset, diff, list-modified, opt-out, remove-pristine. + +Extracted from ``tools.skills_sync``. Profile-scoped paths and patchable helpers +(``_get_bundled_dir``, ``sync_skills``, ...) are resolved through ``_ss()`` at +call time so monkeypatching ``tools.skills_sync`` keeps working. +""" + +from pathlib import Path +from typing import List, Optional, Tuple + + +def _ss(): + from tools import skills_sync + + return skills_sync + + +def _is_tracked_user_modification(origin_hash: str, user_hash: str) -> bool: + """Whether an on-disk skill is a user modification ``hermes update`` keeps. Shared by + the sync loop and ``list_user_modified_bundled_skills`` so they never drift: needs a + recorded origin hash (un-baselined v1 entries don't count) AND differing content.""" + return bool(origin_hash) and user_hash != origin_hash + + +def _bundled_by_name(bundled_dir: Path) -> dict: + return dict(_ss()._discover_bundled_skills(bundled_dir)) + + +def reset_bundled_skill(name: str, restore: bool = False) -> dict: + """Reset a bundled skill's manifest tracking so future syncs work normally. + + An edited bundled skill stays ``user_modified`` forever — even after copying the + bundled version back — because the manifest holds the OLD origin hash; clearing + the entry breaks that loop. ``restore`` also deletes the user's copy so the next + sync re-copies bundled. Returns ``{ok, action, message, synced}``; action is + manifest_cleared / restored / not_in_manifest / bundled_missing / not_reset. + """ + ss = _ss() + manifest = ss._read_manifest() + bundled_dir = ss._get_bundled_dir() + bundled_by_name = _bundled_by_name(bundled_dir) + in_manifest = name in manifest + is_bundled = name in bundled_by_name + + def _fail(action: str, message: str) -> dict: + return {"ok": False, "action": action, "message": message, "synced": None} + + if not in_manifest and not is_bundled: + return _fail("not_in_manifest", f"'{name}' is not a tracked bundled skill. Nothing to reset. " + f"(Hub-installed skills use `hermes skills uninstall`.)") + + # Delete the user's copy BEFORE touching the manifest so a failed rmtree + # cannot leave the skill in a manifest-less limbo state. + deleted_user_copy = False + if restore: + if not is_bundled: + return _fail("bundled_missing", f"'{name}' has no bundled source — manifest entry preserved " + f"but cannot restore from bundled (skill was removed upstream).") + dest = ss._compute_relative_dest(bundled_by_name[name], bundled_dir) + if dest.exists(): + try: + ss._rmtree_writable(dest) + except (OSError, IOError) as e: + return _fail("not_reset", f"Could not delete user copy at {dest}: {e}. " + f"Manifest entry preserved — nothing was changed.") + deleted_user_copy = True + + if in_manifest: + del manifest[name] + ss._write_manifest(manifest) + synced = ss.sync_skills(quiet=True) + + if not restore: + action, message = "manifest_cleared", (f"Cleared manifest entry for '{name}'. Future `hermes update` runs " + f"will re-baseline against your current copy and accept upstream changes.") + elif deleted_user_copy: + action, message = "restored", f"Restored '{name}' from bundled source." + else: + action, message = "restored", f"Restored '{name}' (no prior user copy, re-copied from bundled)." + return {"ok": True, "action": action, "message": message, "synced": synced} + + +def list_user_modified_bundled_skills() -> List[dict]: + """Bundled skills ``hermes update`` keeps because the user edited them (same test + the sync loop uses). Name-sorted ``{"name", "dest", "bundled_src"}`` dicts.""" + ss = _ss() + manifest = ss._read_manifest() + if not manifest: + return [] + bundled_dir = ss._get_bundled_dir() + modified: List[dict] = [] + for skill_name, skill_dir in ss._discover_bundled_skills(bundled_dir): + origin_hash = manifest.get(skill_name, "") # empty = untracked/un-baselined v1: next sync handles it + dest = ss._compute_relative_dest(skill_dir, bundled_dir) + if origin_hash and dest.exists() and _is_tracked_user_modification(origin_hash, ss._dir_hash(dest)): + modified.append({"name": skill_name, "dest": dest, "bundled_src": skill_dir}) + return sorted(modified, key=lambda e: e["name"]) + + +def _read_for_diff(path: Path) -> Tuple[Optional[bytes], Optional[str]]: + """Read a file once for diffing: ``(raw_bytes, text)`` with ``text=None`` for + binary content, ``(None, None)`` if unreadable.""" + try: + data = path.read_bytes() + return data, (None if b"\x00" in data else data.decode("utf-8")) + except OSError: + return None, None + except UnicodeDecodeError: + return data, None + + +def diff_bundled_skill(name: str) -> dict: + """Diff a user's copy of a bundled skill against stock. Returns ``{ok, name, found, + modified, message, diffs}``; each diff is ``{"path", "status", "diff"}`` with status + modified / added (only in user copy) / removed (only in bundled) / binary.""" + import difflib + + from tools.skills_sync_optional import _skill_file_list + + ss = _ss() + + def _fail(found: bool, message: str) -> dict: + return {"ok": False, "name": name, "found": found, "modified": False, "diffs": [], "message": message} + + bundled_dir = ss._get_bundled_dir() + bundled_src = _bundled_by_name(bundled_dir).get(name) + if bundled_src is None: + return _fail(False, f"'{name}' is not a tracked bundled skill (no stock version to " + f"diff against). Hub-installed skills use `hermes skills inspect`.") + dest = ss._compute_relative_dest(bundled_src, bundled_dir) + if not dest.exists(): + return _fail(True, f"No local copy of '{name}' found at {dest}.") + + user_files = set(_skill_file_list(dest)) + stock_files = set(_skill_file_list(bundled_src)) + diffs: List[dict] = [] + for rel in sorted(user_files | stock_files): + if rel not in stock_files: + diffs.append({"path": rel, "status": "added", "diff": f"+ only in your copy: {rel}"}) + elif rel not in user_files: + diffs.append({"path": rel, "status": "removed", "diff": f"- only in stock: {rel}"}) + else: + user_bytes, user_text = _read_for_diff(dest / rel) + stock_bytes, stock_text = _read_for_diff(bundled_src / rel) + if user_text is None or stock_text is None: + # At least one side is binary — report only if the bytes differ. + if user_bytes != stock_bytes: + diffs.append({"path": rel, "status": "binary", "diff": ""}) + elif user_text != stock_text: + text = "".join(difflib.unified_diff( + stock_text.splitlines(keepends=True), user_text.splitlines(keepends=True), + fromfile=f"stock/{rel}", tofile=f"yours/{rel}", + )) + diffs.append({"path": rel, "status": "modified", "diff": text}) + + message = (f"'{name}' differs from the stock version in {len(diffs)} file(s)." if diffs + else f"'{name}' matches the stock version.") + return {"ok": True, "name": name, "found": True, "modified": bool(diffs), "diffs": diffs, "message": message} + + +_OPT_OUT_MESSAGES = { # (enabled, changed) -> message + (True, True): "Opted out of bundled skills. Future install / update / sync runs will not seed bundled skills into this profile.", + (True, False): "Already opted out — marker was already present.", + (False, True): "Opted back in. The next `hermes update` (or `hermes skills opt-in --sync`) will re-seed bundled skills.", + (False, False): "Not opted out — no marker to remove.", +} + + +def set_bundled_skills_opt_out(enabled: bool) -> dict: + """Toggle the .no-bundled-skills marker: the on-disk half of ``hermes skills + opt-out`` / ``opt-in`` that stops installer/update/sync seeding. Removing + already-present skills is a separate step (``remove_pristine_bundled_skills``). + Returns ``{ok, changed, marker, message}``.""" + ss = _ss() + marker = ss._hermes_home() / ss.NO_BUNDLED_SKILLS_MARKER + existed = marker.exists() + try: + if enabled: + ss._hermes_home().mkdir(parents=True, exist_ok=True) + marker.write_text( + "This profile opted out of bundled-skill seeding (`hermes skills opt-out`).\n" + "Delete this file to re-enable sync on the next `hermes update`.\n", + encoding="utf-8", + ) + elif existed: + marker.unlink() + except OSError as e: + return {"ok": False, "changed": False, "marker": str(marker), + "message": f"Could not update opt-out marker at {marker}: {e}"} + changed = enabled != existed + return {"ok": True, "changed": changed, "marker": str(marker), "message": _OPT_OUT_MESSAGES[(enabled, changed)]} + + +def remove_pristine_bundled_skills(dry_run: bool = False) -> dict: + """Delete bundled skills that are present, manifest-tracked, AND unmodified. + + Removed ONLY when in the sync manifest (genuinely bundled, not hub/hand-written), + still in the bundled source (hash-comparable), and byte-identical to the origin + hash; everything else lands in ``skipped``. Removed skills lose their manifest + entry so a later opt-in re-seed treats them as new. + Returns ``{ok, removed, skipped: [{name, reason}], dry_run, message}``. + """ + ss = _ss() + manifest = ss._read_manifest() + bundled_dir = ss._get_bundled_dir() + bundled_by_name = _bundled_by_name(bundled_dir) + + removed: List[str] = [] + skipped: List[dict] = [] + for name, origin_hash in sorted(manifest.items()): + src = bundled_by_name.get(name) + if src is None: + skipped.append({"name": name, "reason": "no bundled source (removed upstream)"}) + continue + dest = ss._compute_relative_dest(src, bundled_dir) + if not dest.exists(): + # Already gone from disk; just forget the stale manifest entry. + if not dry_run: + manifest.pop(name, None) + continue + if ss._dir_hash(dest) != origin_hash: + skipped.append({"name": name, "reason": "user-modified (kept)"}) + continue + if not dry_run: + try: + ss._rmtree_writable(dest) + except (OSError, IOError) as e: + skipped.append({"name": name, "reason": f"delete failed: {e}"}) + continue + manifest.pop(name, None) + removed.append(name) + + if not dry_run and removed: + ss._write_manifest(manifest) + + verb = "Would remove" if dry_run else "Removed" + return { + "ok": True, "removed": removed, "skipped": skipped, "dry_run": dry_run, + "message": f"{verb} {len(removed)} pristine bundled skill(s); kept {len(skipped)}.", + } diff --git a/tools/skills_sync_client.py b/tools/skills_sync_client.py index 1689edbe86..1ff1b45713 100644 --- a/tools/skills_sync_client.py +++ b/tools/skills_sync_client.py @@ -1,447 +1,179 @@ #!/usr/bin/env python3 -""" -Skill Sync client -- the low-level sync layer. +"""Skill Sync client -- the low-level sync layer. -This is the LOW-LEVEL sync layer. It builds content-addressed objects -(blob/tree/commit) from local skills, talks the sync wire contract to a sync -plane (push objects + CAS a ref, pull the owner's HEAD, three-way merge on a -409), and is driven by: +Builds content-addressed objects from local skills and talks the sync wire +contract (push objects + CAS a ref, pull the owner's HEAD, three-way merge on +a 409). Driven by the debounced push hook in ``skill_manage``, the periodic +``maybe_pull_skills`` at the curator tick sites, and the ``hermes sync`` CLI. +Lives under tools/ (NOT hermes_cli/) so it never imports the CLI at module +load. ``skills_sync_client_wire`` (object model + HTTP client + tree walks) and +``skills_sync_client_org`` (org-shared skills) are re-exported here. - * a debounced push hook in ``skill_manage`` (after the write-gate passes), - * a periodic pull hook (``maybe_pull_skills``) at the curator tick sites, - * the ``hermes sync status|pull|push|now`` CLI. +ACCESS GATE (pre-launch): sync is INERT unless the signed-in user is a Nous +admin, read off the ``tool_gateway_admin`` JWT claim. The claim name is NAS's +and misleading: it is the global portal-admin permission, NOT a tool-gateway +right. Replace with a real entitlement before shipping to users. -It lives beside ``tools/skills_sync.py`` (NOT under ``hermes_cli/``) so the -low-level sync layer never imports the CLI -- same rule the bundled-skills -sync module documents at ``skills_sync.py:43-50``. - -Contract: the Skill Sync wire contract (version 1, frozen -for Milestone 1). Endpoint shapes, object model, canonicalization, and status -codes below all trace to that document. - ---- ACCESS GATE (pre-launch) --------------------------------------------- -Client sync is INERT (no push, no pull, no-op) unless the signed-in user is a -**Nous admin**. We read that off the access token, which rides on the same -bearer ``resolve_nous_runtime_credentials()`` returns; we decode the JWT -payload (no signature verification -- the server re-verifies) and check the -claim before doing any sync work. - -NAMING: the claim on the wire is ``tool_gateway_admin``, which is misleading --- it is NOT a tool-gateway-specific right. NAS populates it from -``Permissions.ADMIN_ACCESS`` (access-token-issuer.ts), the same global portal -admin permission that guards ``/admin/*``; the claim is simply named for its -first consumer. We keep the wire name (other services read it) but call it -what it means everywhere on this side. - -This gate is pre-launch containment, not the shipping entitlement. Admin -status conflates "may administer Nous" with "has Skill Sync enabled", and has -no middle setting for a beta cohort -- opening it up would mean handing out -portal admin. Replace it with a real entitlement (a ``sync:*`` scope, a tier -check, or a per-cohort feature flag) before shipping to users. - ---- OPT-IN DEFAULT (M1-D, provisional) ----------------------------------- -Nothing syncs unless the user marks a skill for sync. The user's local intent -is toggled via ``hermes sync enable/disable`` (a ``sync`` flag on the skill's -``.usage.json`` sidecar, alongside ``pinned``/``created_by``), but the DURABLE, -CROSS-DEVICE opt-in state is a committed ``sync-manifest`` object in the sync -plane (design.md §2.8): a root-level blob in the tree at -``refs/user//HEAD`` recording per-skill ``{name, enabled}``. Push writes -the manifest from local intent; pull reconciles local intent FROM it, so a skill -opted in on one device becomes opted in on the others. The plane manifest is -authoritative; the local flag is just the editable intent. Only agent-created + -user-authored skills under ``~/.hermes/skills/`` are eligible; bundled and -hub-installed skills are excluded. +OPT-IN DEFAULT (provisional): nothing syncs unless the user marks a skill for +sync. Local intent is the ``sync`` flag in ``.usage.json``; the DURABLE +cross-device state is the committed ``sync-manifest`` blob in the plane. Only +agent-created + user-authored skills under ``~/.hermes/skills/`` are eligible. """ from __future__ import annotations -import hashlib import json import logging import os -import time -import stat as _stat -from datetime import datetime, timezone +import tempfile +from contextlib import suppress from pathlib import Path, PurePosixPath from typing import Any, Callable, Dict, List, Optional, Tuple +from tools.skills_sync_client_wire import ( # noqa: F401 (re-exports) + ARTIFACT_TYPE_SKILL, DEFAULT_MAX_OBJECT_BYTES, KIND_BLOB, KIND_COMMIT, KIND_TREE, MODE_DIR, + MODE_EXEC, MODE_FILE, SYNC_MANIFEST_ENTRY_NAME, SYNC_MANIFEST_TYPE, SYNC_MANIFEST_VERSION, + WIRE_VERSION, ObjectSet, SyncClient, SyncConflict, SyncError, _check_version, + assemble_root_from_skill_trees, build_commit, build_root_tree, build_sync_manifest_bytes, + build_tree, canonical_json_bytes, materialize_tree, merge_skill, nest_skill_tree, + parse_sync_manifest, read_manifest_of_root, read_ref_hash, root_tree_of_commit, + skill_trees_of_root, wire_address, +) + logger = logging.getLogger(__name__) -# Sync protocol constants -# Wire protocol version. The over-the-wire names below (the `hsp_version` -# capability field and the `x-hsp-object-type` response header) are part of -# the deployed server contract and are NOT renamed with the product — the -# user-facing feature is "Skill Sync"; these are protocol identifiers. -WIRE_VERSION = "1" -DEFAULT_MAX_OBJECT_BYTES = 26214400 # 25 MiB, mirrors capabilities default - -# Object kinds (sync contract) -KIND_BLOB = "blob" -KIND_TREE = "tree" -KIND_COMMIT = "commit" - -# Tree entry modes (sync contract) -MODE_FILE = "file" -MODE_EXEC = "exec" -MODE_DIR = "dir" - -ARTIFACT_TYPE_SKILL = "skill" - -# --------------------------------------------------------------------------- -# `sync-manifest` object convention (design notes). -# -# Per-skill sync opt-in ("this skill syncs / this one does not" -# opt-in state) is CONTENT inside the sync object model, NOT a device-local flag -# or a mutable preference table. An owner's synced set is a small committed blob -# named ``sync-manifest`` at the ROOT of the tree referenced by -# ``refs/user//HEAD``, recording per-skill ``{name, enabled}``. Toggling -# opt-in is a plain CAS ref update (upload the new manifest blob + root tree + -# commit, then CAS HEAD) — the same primitives push already uses. -# -# This makes opt-in durable and CROSS-DEVICE: device B learns which skills the -# user opted in on device A by reading the manifest on pull, rather than each -# device keeping its own local flag. The ``.usage.json`` ``sync`` flag is kept -# only as the local *intent* the user toggles via ``hermes sync enable`` — it is -# reconciled TO the manifest on pull and FROM it on push; the manifest in the -# plane is authoritative. -# -# MUST match gateway-gateway ``src/sync/manifest.ts`` byte-for-byte (the server -# reads + validates this exact shape). Entry name, ``type`` marker, ``version``, -# and the ``{name, enabled}`` skill shape are the shared contract. -# --------------------------------------------------------------------------- - -SYNC_MANIFEST_ENTRY_NAME = "sync-manifest" -SYNC_MANIFEST_TYPE = "sync-manifest" -SYNC_MANIFEST_VERSION = 1 +_merge_skill = merge_skill +_skill_trees_of_root = skill_trees_of_root -def build_sync_manifest_bytes(skills: Dict[str, bool]) -> bytes: - """Serialize the per-skill opt-in map into canonical ``sync-manifest`` bytes. +# Identity & access gate. The bearer comes from resolve_nous_runtime_credentials() +# (file lock, host allowlist, refresh -- not reimplemented); its payload is +# decoded unverified to read the gate claim. - ``skills`` maps skill name -> enabled. Emits the shape gateway-gateway's - ``parseSyncManifest`` validates: ``{type, version:1, skills:[{name,enabled}]}``. - Skill entries are sorted by name for a stable content address. - """ - manifest = { - "type": SYNC_MANIFEST_TYPE, - "version": SYNC_MANIFEST_VERSION, - "skills": [ - {"name": name, "enabled": bool(enabled)} - for name, enabled in sorted(skills.items()) - ], - } - return canonical_json_bytes(manifest) - - -def parse_sync_manifest(data: bytes) -> Optional[Dict[str, bool]]: - """Parse ``sync-manifest`` bytes into ``{name: enabled}``, or ``None`` if the - bytes are not a well-formed manifest. - - Strict (mirrors gateway-gateway ``parseSyncManifest``): an unknown ``type``, - a missing/!=1 ``version``, a non-array ``skills``, or a malformed skill entry - all reject rather than being coerced — a malformed manifest must not be - mistaken for "no skills opted in." - """ - try: - value = json.loads(data.decode("utf-8")) - except Exception: - return None - if not isinstance(value, dict): - return None - if value.get("type") != SYNC_MANIFEST_TYPE: - return None - if value.get("version") != SYNC_MANIFEST_VERSION: - return None - raw_skills = value.get("skills") - if not isinstance(raw_skills, list): - return None - out: Dict[str, bool] = {} - for raw in raw_skills: - if not isinstance(raw, dict): - return None - name = raw.get("name") - enabled = raw.get("enabled") - if not isinstance(name, str) or not name: - return None - if not isinstance(enabled, bool): - return None - out[name] = enabled - return out - - -# --------------------------------------------------------------------------- -# Content addressing -# -# The wire uses the FULL 64-hex sha256 digest. This is a DIFFERENT -# namespace from hermes-agent's local ``content_hash`` (skills_guard.py:846), -# which is a truncated 16-hex digest used for local dedup. They must never be -# conflated -- we compute full digests here. -# --------------------------------------------------------------------------- - -def wire_address(data: bytes) -> str: - """Return ``sha256:<64-hex>`` -- the wire address of ``data``.""" - return "sha256:" + hashlib.sha256(data).hexdigest() - - -def canonical_json_bytes(obj: Dict[str, Any]) -> bytes: - """Canonical JSON serialization for tree/commit hashing (sync contract). - - UTF-8, keys sorted lexicographically, no insignificant whitespace - (``separators=(",", ":")``), no trailing newline. Arrays must already be - in the contract-specified order by the caller (tree entries by ``name``, - commit ``parents`` in significance order). Both client and server MUST - produce byte-identical output or a push fails ``422 hash_mismatch``. - """ - return json.dumps( - obj, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - ).encode("utf-8") - - -# --------------------------------------------------------------------------- -# Identity & access gate -# -# We reuse resolve_nous_runtime_credentials() for the bearer (it honors the -# cross-process file lock + portal host allowlist and refreshes as needed -- -# we do NOT reimplement refresh). The returned api_key IS the JWT bearer; we -# decode its payload (unverified) to read the access-gate claim. -# --------------------------------------------------------------------------- - -# Dev-phase gate claim (NAS access-token-issuer.ts:312). Sync is inert unless -# the resolved token carries this claim === true. Remove when sync ships GA. -# Wire claim name is NAS's; it means "this user is a Nous admin" -# (populated from Permissions.ADMIN_ACCESS), NOT a tool-gateway right. +# Wire claim name is NAS's; it means "Nous admin" (Permissions.ADMIN_ACCESS). NOUS_ADMIN_CLAIM = "tool_gateway_admin" class SyncInertError(RuntimeError): - """Raised (and caught by the gate-and-swallow hooks) when sync must no-op: - - not logged in, no bearer, or the caller is not a Nous admin. - """ - - -def _decode_jwt_payload_unverified(token: str) -> Dict[str, Any]: - """Decode a JWT payload WITHOUT signature verification. - - Safe here: we never trust these claims for authz -- the server re-verifies - every call. We only read the dev-gate claim to decide whether to attempt - sync at all. Mirrors the diagnostic decode in - plugins/dashboard_auth/nous/__init__.py:463. - """ - try: - import jwt # PyJWT, a core dependency - - return jwt.decode( - token, - options={"verify_signature": False, "verify_exp": False}, - ) or {} - except Exception as e: - logger.debug("skills_sync_client: JWT payload decode failed: %s", e) - return {} + """Sync must no-op: not logged in, no bearer, or not a Nous admin. + Caught by the gate-and-swallow hooks.""" def resolve_identity() -> Dict[str, Any]: - """Resolve the Nous bearer + owner + dev-gate flag. + """Resolve ``{api_key, base_url, owner, nous_admin, claims}``; raises + SyncInertError if not logged in / no bearer. ``owner`` is advisory for local + ref naming only -- the server derives the real owner from the bearer. - Returns a dict: ``{api_key, base_url, owner, nous_admin, claims}``. - Raises :class:`SyncInertError` if not logged in / no bearer. - - ``owner`` is the token-verified subject; the server derives the real owner - from the bearer regardless (contract §0.4), so this is advisory for local - ref naming only. + The JWT payload is decoded WITHOUT signature verification. Safe: the claims + are never used for authz here (the server re-verifies); they only decide + whether to attempt sync at all. """ try: from hermes_cli.auth import resolve_nous_runtime_credentials - creds = resolve_nous_runtime_credentials() + creds = resolve_nous_runtime_credentials() or {} except Exception as e: raise SyncInertError(f"no Nous credentials: {e}") from e - api_key = (creds or {}).get("api_key") + api_key = creds.get("api_key") if not api_key: raise SyncInertError("no bearer token available") - claims = _decode_jwt_payload_unverified(api_key) - owner = ( - claims.get("sub") - or claims.get("privy_did") - or claims.get("tid") - or "unknown" - ) - nous_admin = claims.get(NOUS_ADMIN_CLAIM) is True + try: + import jwt # PyJWT, a core dependency + + claims = jwt.decode(api_key, options={"verify_signature": False, "verify_exp": False}) or {} + except Exception as e: + logger.debug("skills_sync_client: JWT payload decode failed: %s", e) + claims = {} + owner = claims.get("sub") or claims.get("privy_did") or claims.get("tid") or "unknown" return { "api_key": api_key, - "base_url": (creds or {}).get("base_url"), + "base_url": creds.get("base_url"), "owner": str(owner), - "nous_admin": nous_admin, + "nous_admin": claims.get(NOUS_ADMIN_CLAIM) is True, "claims": claims, } -def dev_gate_open() -> bool: - """Whether the access gate permits sync. Never raises.""" - try: - return bool(resolve_identity().get("nous_admin")) - except SyncInertError: - return False - except Exception as e: - logger.debug("skills_sync_client: dev_gate_open check failed: %s", e) - return False +# Configuration -- env-first so a Hermes Cloud instance can enable sync purely +# through environment variables. Every knob: HERMES_SYNC_ env -> config.yaml +# ``sync.`` -> built-in default (keys: base_url = the sync plane, NOT the +# inference URL; enabled; default_opt_in; org_auto_propose). - -# --------------------------------------------------------------------------- -# Sync-plane endpoint resolution -# -# The sync routes are mounted under /v1/sync/. The base URL defaults to the -# production plane, so a normal user configures nothing; config.yaml -# sync.base_url (or the HERMES_SYNC_BASE_URL bridge env) overrides it to point -# a dev/staging build at another plane. It is NOT the inference base_url. -# --------------------------------------------------------------------------- - -#: Production Skill Sync plane. Overridable per the resolution order below. +#: Production Skill Sync plane; a normal user configures nothing. DEFAULT_SYNC_BASE_URL = "https://gateway-gateway.nousresearch.com" -def resolve_sync_base_url() -> Optional[str]: - """Resolve the sync-plane base URL. - - Order: HERMES_SYNC_BASE_URL env bridge -> config.yaml ``sync.base_url`` -> - the production plane. Returns a base without a trailing slash (e.g. - ``https://host``); the ``/v1/sync/`` prefix is appended by the client. - - The production default means a normal user never configures a URL — the - env var and config key exist to point a dev/staging build at another - plane. Returns None only if the default is somehow blanked out. - """ - env = os.getenv("HERMES_SYNC_BASE_URL") - if env and env.strip(): - return env.strip().rstrip("/") - try: - # Lazy import: the low-level sync layer must not import the CLI at - # module load (skills_sync.py:43-50). A function-scoped import avoids - # the cycle -- same pattern agent/curator.py:141 uses for config. - from hermes_cli.config import load_config - - cfg = load_config() or {} - sync_cfg = cfg.get("sync") or {} - base = sync_cfg.get("base_url") - if isinstance(base, str) and base.strip(): - return base.strip().rstrip("/") - except Exception as e: - logger.debug("skills_sync_client: config sync.base_url read failed: %s", e) - return DEFAULT_SYNC_BASE_URL or None - - -# --------------------------------------------------------------------------- -# Sync feature configuration — env-first, so a Hermes Cloud instance can be set -# up to use sync BY DEFAULT purely through environment variables (no per-user -# config.yaml edit, no per-skill CLI call). Every knob follows the same -# precedence as base_url: the HERMES_SYNC_* env var wins, else config.yaml -# ``sync.*``, else a built-in default. -# -# HERMES_SYNC_BASE_URL -> sync.base_url (the sync plane URL) -# HERMES_SYNC_ENABLED -> sync.enabled (master on/off; default off) -# HERMES_SYNC_DEFAULT_OPT_IN -> sync.default_opt_in (personal sync policy; default false -# = opt-in. Set true to make -# every eligible skill sync -# without per-skill enable — -# the opt-OUT default a Cloud -# deployment wants.) -# --------------------------------------------------------------------------- - _TRUE = {"1", "true", "yes", "on"} _FALSE = {"0", "false", "no", "off", ""} +def _sync_config(key: str) -> Any: + """``sync.`` from config.yaml, or None. Lazy import: this layer must not + import the CLI at module load.""" + try: + from hermes_cli.config import load_config + + return ((load_config() or {}).get("sync") or {}).get(key) + except Exception as e: + logger.debug("skills_sync_client: config sync.%s read failed: %s", key, e) + return None + + +def resolve_sync_base_url() -> Optional[str]: + """HERMES_SYNC_BASE_URL -> ``sync.base_url`` -> production plane, without a + trailing slash (``/v1/sync/`` is appended by the client). None only if the + default is blanked out.""" + env = os.getenv("HERMES_SYNC_BASE_URL") + if env and env.strip(): + return env.strip().rstrip("/") + base = _sync_config("base_url") + if isinstance(base, str) and base.strip(): + return base.strip().rstrip("/") + return DEFAULT_SYNC_BASE_URL or None + + def _parse_bool(value: Any) -> Optional[bool]: - """Parse a config/env bool. Returns None if unrecognized (so callers can - fall through to the next precedence layer). Accepts real bools + strings.""" + """Parse a config/env bool; None if unrecognized so callers fall through to + the next precedence layer.""" if isinstance(value, bool): return value if value is None: return None s = str(value).strip().lower() - if s in _TRUE: - return True - if s in _FALSE: - return False - return None + return True if s in _TRUE else False if s in _FALSE else None def _sync_config_bool(env_var: str, config_key: str, *, default: bool) -> bool: - """Resolve a boolean sync knob: ``env_var`` -> ``sync.`` -> default.""" + """``env_var`` -> ``sync.`` -> default.""" env_val = _parse_bool(os.getenv(env_var)) if env_val is not None: return env_val - try: - from hermes_cli.config import load_config - - cfg = load_config() or {} - sync_cfg = cfg.get("sync") or {} - cfg_val = _parse_bool(sync_cfg.get(config_key)) - if cfg_val is not None: - return cfg_val - except Exception as e: - logger.debug("skills_sync_client: config sync.%s read failed: %s", config_key, e) - return default + cfg_val = _parse_bool(_sync_config(config_key)) + return default if cfg_val is None else cfg_val def sync_feature_enabled() -> bool: - """Whether the sync feature is turned on for this instance (env-first). - - ``HERMES_SYNC_ENABLED`` -> ``sync.enabled`` -> False. This is the master - switch a Hermes Cloud deployment sets to opt its instances into sync by - default. It is checked by the gate-and-swallow entrypoints IN ADDITION to - the Nous-admin token gate and a configured base URL — all three must hold for - background sync to run. - """ + """Master switch. Checked by the gate-and-swallow entrypoints IN ADDITION to + the Nous-admin gate and a configured base URL -- all three must hold.""" return _sync_config_bool("HERMES_SYNC_ENABLED", "enabled", default=False) def sync_org_auto_propose() -> bool: - """Whether an agent/user edit to an org skill is proposed automatically. - - ``HERMES_SYNC_ORG_AUTO_PROPOSE`` -> ``sync.org_auto_propose`` -> False. - - False (default): edits to an org-shared skill stay LOCAL until the user - runs ``hermes sync propose ``. The skill keeps working with the - edit applied; the organisation just doesn't see it yet. - - True: every local edit to an org skill is submitted to the org as a - proposal right away (an admin still approves it, unless the editor is an - admin). Suits a small, high-trust team that wants improvements to flow - back without anyone remembering to push them. - """ - return _sync_config_bool( - "HERMES_SYNC_ORG_AUTO_PROPOSE", "org_auto_propose", default=False - ) + """False (default): edits to an org skill stay LOCAL until ``hermes sync + propose``. True: every edit is proposed right away (an admin still approves + unless the editor is one) -- for small high-trust teams.""" + return _sync_config_bool("HERMES_SYNC_ORG_AUTO_PROPOSE", "org_auto_propose", default=False) def sync_default_opt_in() -> bool: - """The personal sync default opt-in policy (env-first). - - ``HERMES_SYNC_DEFAULT_OPT_IN`` -> ``sync.default_opt_in`` -> False. - - False (default): opt-IN — a skill syncs only after an explicit - ``hermes sync enable`` (or a plane manifest that opted it in). True: opt-OUT - — every sync-eligible skill is treated as opted in unless explicitly - disabled, which is the "your skills follow you with no setup" default a - Hermes Cloud deployment wants. Per the design notes, this default is - provisional and expected to flip; exposing it as env config lets the - operator choose per deployment without a protocol change. - """ + """False (default): opt-IN -- a skill syncs only after ``hermes sync enable`` + or a plane manifest opting it in. True: opt-OUT -- every eligible skill + syncs unless explicitly disabled (the Hermes Cloud default). Provisional.""" return _sync_config_bool("HERMES_SYNC_DEFAULT_OPT_IN", "default_opt_in", default=False) -# --------------------------------------------------------------------------- -# Local skill eligibility + the personal sync opt-in "sync" flag -# -# Only agent-created + user-authored skills under ~/.hermes/skills/ sync. -# Bundled (.bundled_manifest) and hub-installed skills are excluded. Sync is -# opt-in: a skill only syncs when its usage-sidecar carries ``sync: true``. -# --------------------------------------------------------------------------- +# Local skill eligibility + the personal opt-in flag def _skills_dir() -> Path: from hermes_constants import get_hermes_home @@ -449,15 +181,35 @@ def _skills_dir() -> Path: return get_hermes_home() / "skills" -def is_sync_eligible(skill_name: str) -> bool: - """Whether *skill_name* is a candidate for sync (before the opt-in check). +def _org_dir() -> Path: + """Local mirror root for org skills (read-only by convention).""" + return _skills_dir() / ORG_DIR_NAME - Eligible = present locally under ~/.hermes/skills/, NOT bundled, NOT - hub-installed, NOT an external-dir skill, and NOT under the org mirror - (``_org/`` — enterprise-managed content pulls from the org HEAD and must - never ride a personal push; the sync contract / the design notes). Mirrors the - exclusion logic used by the curator (tools/skill_usage.py). - """ + +def _rel_to_skills_dir(skill_dir: Path) -> Optional[Path]: + """*skill_dir* relative to ~/.hermes/skills/, or None if outside/unresolvable.""" + try: + return skill_dir.resolve().relative_to(_skills_dir().resolve()) + except (OSError, ValueError): + return None + + +def _skill_rel_path(skill_name: str) -> Optional[PurePosixPath]: + """The skill's path relative to ~/.hermes/skills/ (posix), or None.""" + try: + from tools.skill_usage import _find_skill_dir + except Exception: + return None + skill_dir = _find_skill_dir(skill_name) + rel = _rel_to_skills_dir(skill_dir) if skill_dir is not None else None + return PurePosixPath(rel.as_posix()) if rel is not None else None + + +def is_sync_eligible(skill_name: str) -> bool: + """Candidate for sync (before the opt-in check): present locally, NOT + bundled, NOT hub-installed, NOT external, and NOT under the ``_org/`` mirror + (enterprise content must never ride a personal push). Mirrors the curator's + exclusions (tools/skill_usage.py).""" try: from tools.skill_usage import is_bundled, is_hub_installed, _find_skill_dir from agent.skill_utils import is_external_skill_path @@ -466,80 +218,46 @@ def is_sync_eligible(skill_name: str) -> bool: if is_bundled(skill_name) or is_hub_installed(skill_name): return False skill_dir = _find_skill_dir(skill_name) - if skill_dir is None: + if skill_dir is None or is_external_skill_path(skill_dir): return False - if is_external_skill_path(skill_dir): - return False - try: - rel = skill_dir.resolve().relative_to(_skills_dir().resolve()) - if rel.parts and rel.parts[0] == ORG_DIR_NAME: - return False - except (OSError, ValueError): - pass - return True + rel = _rel_to_skills_dir(skill_dir) + return not (rel is not None and rel.parts and rel.parts[0] == ORG_DIR_NAME) def list_synced_skill_names() -> List[str]: - """Return the names of skills that should sync, honoring the opt-in policy. - - Two policies (``sync_default_opt_in()``, env-first — see that function): - - - **opt-in (default):** a skill syncs only when its usage record carries - ``sync: true`` AND it is eligible. Nothing syncs by default. - - **opt-out (Hermes Cloud "on by default"):** every *eligible* skill syncs - UNLESS its usage record explicitly carries ``sync: false``. This is what a - deployment sets (via ``HERMES_SYNC_DEFAULT_OPT_IN``) so a user's skills - follow them with no per-skill setup. - - Sorted, deduped. - """ + """Names of skills that should sync (sorted, deduped), per ``sync_default_opt_in()``: + opt-in -> only eligible skills whose usage record has ``sync: true``; + opt-out -> every eligible skill unless its record has ``sync: false``.""" try: from tools.skill_usage import load_usage except Exception: return [] usage = load_usage() or {} - + flags = {n: rec.get("sync") for n, rec in usage.items() if isinstance(rec, dict)} if sync_default_opt_in(): - # opt-OUT: all eligible skills except those explicitly turned off. - names = [] - for name in _all_local_skill_names(): - rec = usage.get(name) - if isinstance(rec, dict) and rec.get("sync") is False: - continue # explicit opt-out wins over the deployment default - if is_sync_eligible(name): - names.append(name) - return sorted(set(names)) - - # opt-IN (default): only explicitly-enabled eligible skills. - names = [] - for name, rec in usage.items(): - if isinstance(rec, dict) and rec.get("sync") is True and is_sync_eligible(name): - names.append(name) + names = [n for n in _all_local_skill_names() if flags.get(n) is not False and is_sync_eligible(n)] + else: + names = [n for n, f in flags.items() if f is True and is_sync_eligible(n)] return sorted(set(names)) def _all_local_skill_names() -> List[str]: - """Best-effort enumeration of every locally-present skill name (used by the - opt-out policy). A skill is any directory under ~/.hermes/skills/ containing - a ``SKILL.md``; the name is its frontmatter ``name`` (falling back to the - directory name). Eligibility (bundled/hub/external exclusion) is applied by - the caller via ``is_sync_eligible``. - """ + """Every locally-present skill name (a dir under ~/.hermes/skills/ with a + ``SKILL.md``; frontmatter ``name`` falling back to the dir name). Eligibility + is applied by the caller.""" names: List[str] = [] root = _skills_dir() try: - if not root.exists(): - return [] - for skill_md in root.rglob("SKILL.md"): + for skill_md in root.rglob("SKILL.md") if root.exists() else (): if skill_md.is_symlink(): continue - name: Optional[str] = None + name = skill_md.parent.name try: from tools.skill_usage import _read_skill_name - name = _read_skill_name(skill_md, skill_md.parent.name) + name = _read_skill_name(skill_md, name) except Exception: - name = skill_md.parent.name + pass if name: names.append(name) except OSError as e: @@ -547,142 +265,52 @@ def _all_local_skill_names() -> List[str]: return sorted(set(names)) -# --------------------------------------------------------------------------- -# Object building -- turn a skill directory into blob/tree/commit objects -# -# A skill dir becomes one tree (sync contract). Each file is a blob; each -# subdir a nested tree. The profile-root tree (the sync contract: "a tree whose -# entries are category trees") is built from the set of synced skill trees. -# --------------------------------------------------------------------------- - -class ObjectSet: - """Accumulates objects to push: hash -> (kind, bytes). - - Deduped by content address, so identical blobs across skills upload once. - """ - - def __init__(self) -> None: - self.objects: Dict[str, Tuple[str, bytes]] = {} - - def add(self, kind: str, data: bytes) -> str: - addr = wire_address(data) - self.objects.setdefault(addr, (kind, data)) - return addr - - def __len__(self) -> int: - return len(self.objects) +def _opted_in_rel_paths() -> List[str]: + """Relative posix paths of skills the user has opted into sync.""" + rels = (_skill_rel_path(name) for name in list_synced_skill_names()) + return [rel.as_posix() for rel in rels if rel is not None] -def _file_mode(path: Path) -> str: - """Return the tree mode for a regular file: ``exec`` if +x else ``file`` - (contract §2.3). No symlinks / other modes are emitted.""" +def _adopt_manifest_opt_ins(remote_manifest: Optional[Dict[str, bool]]) -> List[str]: + """Enable local sync intent for skills the plane manifest has enabled and that + are locally curation-eligible. Enables only -- a pull never silently disables. + Returns the adopted names; best-effort.""" + adopted: List[str] = [] + if not remote_manifest: + return adopted try: - if path.stat().st_mode & (_stat.S_IXUSR | _stat.S_IXGRP | _stat.S_IXOTH): - return MODE_EXEC - except OSError: - pass - return MODE_FILE + from tools.skill_usage import set_sync, is_curation_eligible, is_sync_enabled + + for sname, enabled in remote_manifest.items(): + if enabled and is_curation_eligible(sname) and not is_sync_enabled(sname): + set_sync(sname, True) + adopted.append(sname) + except Exception as e: + logger.debug("skills_sync_client: manifest opt-in reconcile failed: %s", e) + return adopted -def build_tree(dir_path: Path, objects: ObjectSet, *, max_object_bytes: int) -> str: - """Recursively build objects for *dir_path*; return the tree address. - - Regular files become blobs; subdirectories become nested trees. Symlinks, - sockets, and other special files are skipped (contract §2.3 security: no - symlinks). Blobs over *max_object_bytes* raise :class:`ValueError` so the - caller can surface / skip the artifact (contract §4.3 -> 413). - """ - entries: List[Dict[str, str]] = [] - for child in sorted(dir_path.iterdir(), key=lambda p: p.name): - if child.is_symlink(): - logger.debug("skills_sync_client: skipping symlink %s", child) - continue - if child.is_dir(): - sub_hash = build_tree(child, objects, max_object_bytes=max_object_bytes) - entries.append( - {"name": child.name, "kind": KIND_TREE, "hash": sub_hash, "mode": MODE_DIR} - ) - elif child.is_file(): - data = child.read_bytes() - if len(data) > max_object_bytes: - raise ValueError( - f"file {child} is {len(data)} bytes > max_object_bytes " - f"{max_object_bytes}" - ) - blob_hash = objects.add(KIND_BLOB, data) - entries.append( - { - "name": child.name, - "kind": KIND_BLOB, - "hash": blob_hash, - "mode": _file_mode(child), - } - ) - # else: skip special files - # Entries sorted by name (byte order) for canonicalization (sync contract). - entries.sort(key=lambda e: e["name"]) - tree_obj = {"type": KIND_TREE, "entries": entries} - return objects.add(KIND_TREE, canonical_json_bytes(tree_obj)) - - -def build_commit( - tree_hash: str, - parents: List[str], - *, - owner: str, - device: str, - message: str, - objects: ObjectSet, - ts: Optional[str] = None, -) -> str: - """Build a commit object (sync contract) and return its address. - - ``parents``: 0 for first commit, 1 for a normal edit, 2 for a merge commit - (order significant: parents[0] = base fast-forwarded from, parents[1] = - the other head being merged). - """ - commit_obj = { - "type": KIND_COMMIT, - "tree": tree_hash, - "parents": list(parents), - "author": {"owner": owner, "device": device}, - "ts": ts or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "message": message, - "artifact_type": ARTIFACT_TYPE_SKILL, - } - return objects.add(KIND_COMMIT, canonical_json_bytes(commit_obj)) - +# Device label (commit ``author.device``; advisory, never an auth input) def _default_device_label() -> str: - """A human-friendly default device label: the short hostname plus a short - random suffix for uniqueness (two machines can share a hostname). Falls back - to a bare uuid if the hostname is unavailable/unusable.""" + """Short hostname + short random suffix (two machines can share a hostname); + a bare uuid if the hostname is unusable.""" import socket import uuid - suffix = uuid.uuid4().hex[:6] try: host = socket.gethostname() or "" except OSError: host = "" - # Short hostname (drop domain), strip to a tidy slug; keep it readable. - short = host.split(".")[0].strip() - # Keep only sane chars so the label renders cleanly in the console. - short = "".join(c for c in short if c.isalnum() or c in "-_") or "" - return f"{short}-{suffix}" if short else uuid.uuid4().hex + short = "".join(c for c in host.split(".")[0].strip() if c.isalnum() or c in "-_") + return f"{short}-{uuid.uuid4().hex[:6]}" if short else uuid.uuid4().hex def stable_device_id() -> str: - """Return a stable per-device label for commit ``author.device`` (contract - -- advisory, never an auth input). Persisted under - ~/.hermes/skills/.sync_device_id. - - New devices are seeded with a HUMAN-FRIENDLY default (short hostname + a - short random suffix, e.g. ``bens-macbook-a1b2c3``) so the sync console shows - something recognizable instead of an opaque hash. Existing ``.sync_device_id`` - files are honored verbatim (backward-compatible — a machine keeps its id). - Use ``set_device_name()`` / ``hermes sync device --name`` to set an explicit - label.""" + """Stable per-device label, persisted at ~/.hermes/skills/.sync_device_id. + An existing file always wins. Otherwise seeded from HERMES_SYNC_DEVICE_NAME + (first use only; lets Hermes Cloud name hosted instances) or a friendly + default, then persisted so a later ``set_device_name()`` still wins.""" path = _skills_dir() / ".sync_device_id" try: if path.exists(): @@ -691,309 +319,76 @@ def stable_device_id() -> str: return val except OSError: pass - - # Hermes Cloud (and any templated deployment) can seed the label - # declaratively via HERMES_SYNC_DEVICE_NAME, so a hosted instance shows a - # recognizable name with no CLI call. Env seeds the FIRST-USE value only; it - # is then persisted, so a later `hermes sync device --name` (or editing the - # file) still wins on that device. An explicit file (above) always wins over - # the env. - import os - - env_name = (os.environ.get("HERMES_SYNC_DEVICE_NAME") or "").strip() - val = env_name if env_name else _default_device_label() + val = (os.environ.get("HERMES_SYNC_DEVICE_NAME") or "").strip() or _default_device_label() try: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(val, encoding="utf-8") + _write_device_id(val) except OSError as e: logger.debug("skills_sync_client: could not persist device id: %s", e) return val -def set_device_name(name: str) -> str: - """Set the human-friendly device label used for commit ``author.device``. +def _write_device_id(val: str) -> None: + path = _skills_dir() / ".sync_device_id" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(val, encoding="utf-8") - Writes the (trimmed) name to ~/.hermes/skills/.sync_device_id, overwriting - any previous value. The label is advisory metadata only — never an auth - input (contract §2.4) — so any non-empty string is accepted. Returns the - stored value. Raises ValueError on an empty name. - """ + +def set_device_name(name: str) -> str: + """Overwrite the device label with the trimmed *name*; any non-empty string + is accepted. Returns the stored value; ValueError on empty.""" cleaned = (name or "").strip() if not cleaned: raise ValueError("device name must be a non-empty string") - path = _skills_dir() / ".sync_device_id" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(cleaned, encoding="utf-8") + _write_device_id(cleaned) return cleaned -# --------------------------------------------------------------------------- -# Sync wire client -# -# Thin requests-based client for the endpoints in the sync contract- Uploads all -# new objects (batch), then CAS-es the ref. A 409 returns the actual head for -# the caller's three-way merge. Auth is the Nous bearer resolved above. -# --------------------------------------------------------------------------- +# Local sync STATE: the last HEAD we pushed/pulled + the root tree at that +# point (FULL-digest namespace). Distinct from the bundled manifest +# (skills_sync.py) and from the plane's `sync-manifest` object. Lives at +# ~/.hermes/skills/.sync_state; a legacy `.sync_manifest` is migrated on read. -class SyncError(RuntimeError): - """A non-recoverable wire error (4xx that the client can't retry).""" +_EMPTY_STATE: Dict[str, Any] = {"head": None, "skills": {}} - def __init__(self, message: str, *, status: Optional[int] = None): - super().__init__(message) - self.status = status - - -class SyncConflict(RuntimeError): - """CAS lost (409). NOT a rejection -- pushed objects are already durable. - - ``actual`` is the current head to merge against, or **None** when the ref - does not exist server-side (the server reports that as an empty string). - None means "there is nothing to merge against, retry as a create" -- it - must never be fetched as an object. - """ - - def __init__(self, actual: Optional[str]): - # Normalize here, not at the call site: the server sends "" for a - # non-existent ref, and every consumer must see that as None rather - # than an empty hash it might try to fetch. - self.actual: Optional[str] = actual or None - super().__init__( - f"CAS conflict; actual head {self.actual}" - if self.actual - else "CAS conflict; the ref does not exist yet" - ) - - -class SyncClient: - """Sync client bound to a base URL + bearer (routes under - ``/v1/sync/``).""" - - def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0): - self.base = base_url.rstrip("/") - self.api_key = api_key - self.timeout = timeout - import requests # core dependency - - self._session = requests.Session() - self._session.headers["Authorization"] = f"Bearer {api_key}" - - def _url(self, path: str) -> str: - return f"{self.base}/v1/sync/{path.lstrip('/')}" - - # -- capability & read ------------------------------------------------- - - def capabilities(self) -> Dict[str, Any]: - """GET /v1/sync/capabilities (sync contract). No auth required.""" - r = self._session.get(self._url("capabilities"), timeout=self.timeout) - if r.status_code != 200: - raise SyncError(f"capabilities failed: {r.status_code}", status=r.status_code) - return r.json() - - def get_refs(self, prefix: str, *, org_scope: bool = False) -> List[Dict[str, str]]: - """GET /v1/sync/refs?prefix=... (or the org route when ``org_scope``). - - Org refs live behind a SEPARATE endpoint, not behind a prefix filter on - the personal one: the personal route is hard-scoped to the token's own - owner, so asking it for ``refs/org//`` silently returns the - caller's personal refs instead of an error. Callers reading an org ref - MUST pass ``org_scope=True``. - """ - path = "org/refs" if org_scope else "refs" - params = None if org_scope else {"prefix": prefix} - r = self._session.get(self._url(path), params=params, timeout=self.timeout) - if r.status_code != 200: - raise SyncError(f"get_refs failed: {r.status_code}", status=r.status_code) - refs = (r.json() or {}).get("refs", []) - if org_scope: - # The org route returns the org's refs unfiltered; apply the - # prefix client-side so both modes have the same contract. - refs = [r_ for r_ in refs if str(r_.get("name", "")).startswith(prefix)] - return refs - - def get_object(self, obj_hash: str, *, org_scope: bool = False) -> Tuple[str, bytes]: - """GET /v1/sync/objects/:hash (or the org route when ``org_scope``). - - Kind comes from the object-type response header for tree/commit; a blob - (application/octet-stream) is returned as ``blob``. - - Org objects are stored under the ``org:`` scope key and are NOT - readable through the personal route (it scopes to the token's owner), - so walking an org commit requires ``org_scope=True`` on every hop. - """ - path = f"org/objects/{obj_hash}" if org_scope else f"objects/{obj_hash}" - r = self._session.get(self._url(path), timeout=self.timeout) - if r.status_code == 404: - raise SyncError(f"object {obj_hash} not found", status=404) - if r.status_code == 403: - raise SyncError(f"object {obj_hash} not readable", status=403) - if r.status_code != 200: - raise SyncError(f"get_object failed: {r.status_code}", status=r.status_code) - kind = r.headers.get("X-HSP-Object-Type") or KIND_BLOB - return kind, r.content - - def get_commit_json( - self, commit_hash: str, *, org_scope: bool = False - ) -> Dict[str, Any]: - """Fetch a commit object and parse its canonical JSON.""" - kind, data = self.get_object(commit_hash, org_scope=org_scope) - if kind != KIND_COMMIT: - raise SyncError(f"{commit_hash} is {kind}, expected commit") - return json.loads(data.decode("utf-8")) - - def get_tree_json( - self, tree_hash: str, *, org_scope: bool = False - ) -> Dict[str, Any]: - """Fetch a tree object and parse its canonical JSON.""" - kind, data = self.get_object(tree_hash, org_scope=org_scope) - if kind != KIND_TREE: - raise SyncError(f"{tree_hash} is {kind}, expected tree") - return json.loads(data.decode("utf-8")) - - # -- write ------------------------------------------------------------- - - def put_objects( - self, - objects: Dict[str, Tuple[str, bytes]], - *, - org_scope: bool = False, - ) -> Dict[str, Any]: - """POST /v1/sync/objects (sync contract). Batch multi-object upload. - - Contract §1 requires raw object bytes on the wire (NOT base64-in-JSON), - and specifies "a length-prefixed or multipart stream of - {hash, type, bytes}". We use multipart/form-data: one part per object, - the part's field name = the claimed ``sha256:`` hash, its - ``filename`` carries the object ``type`` (blob|tree|commit), and the - part body is the raw object bytes. The server recomputes each hash from - the received bytes and rejects the whole batch with 422 on mismatch. - Idempotent: a known hash is a no-op ``already_present``. - - M2 (contract §11.5): ``org_scope=True`` adds ``?scope=org`` so the - objects land in the ORG scope (org-readable; required before an org - CAS/propose). Gated server-side on the token's org_role claim. - - NOTE (framing choice within contract latitude): §4.2 says "length- - prefixed OR multipart"; this picks multipart/form-data with - (field=hash, filename=type, body=raw-bytes). The server strand must - parse the same framing -- flagged for cross-strand alignment. - """ - # (field_name, (filename, raw_bytes, content_type)) - files = [ - (h, (kind, data, "application/octet-stream")) - for h, (kind, data) in objects.items() - ] - r = self._session.post( - self._url("objects"), - files=files, - params={"scope": "org"} if org_scope else None, - timeout=self.timeout, - ) - if r.status_code == 413: - raise SyncError("object too large (413)", status=413) - if r.status_code == 422: - raise SyncError(f"hash_mismatch (422): {r.text}", status=422) - if r.status_code not in (200, 201): - raise SyncError(f"put_objects failed: {r.status_code}", status=r.status_code) - return r.json() if r.content else {} - - def cas_ref(self, name: str, from_hash: Optional[str], to_hash: str) -> Dict[str, Any]: - """POST /v1/sync/refs/:name -- atomic compare-and-swap (sync contract). - - Raises :class:`SyncConflict` (carrying the actual head) on 409. - - M2 (contract §11.5): a non-admin member's CAS on an org HEAD is never - rejected — the server converts it to a proposal and returns - ``202 {proposal_id, ref}``. Surfaced as - ``{"proposal_pending": True, ...}`` so callers can tell "merged" (200) - from "proposed, awaiting review" (202) without exceptions — a 202 is a - SUCCESS-shaped outcome, never to be presented as live (error table §5). - """ - r = self._session.post( - self._url(f"refs/{name}"), - json={"from": from_hash, "to": to_hash}, - timeout=self.timeout, - ) - if r.status_code == 202: - body = r.json() if r.content else {} - return {"proposal_pending": True, **body} - if r.status_code == 409: - # An EMPTY `actual` means the ref does not exist server-side (the - # CAS lost against "no head"), NOT that there is a commit to merge - # against. Callers must not try to fetch it as an object. - raise SyncConflict((r.json() or {}).get("actual", "")) - if r.status_code == 403: - raise SyncError("forbidden (403) -- owner/permission", status=403) - if r.status_code != 200: - raise SyncError(f"cas_ref failed: {r.status_code}", status=r.status_code) - return r.json() if r.content else {} - - -# --------------------------------------------------------------------------- -# Local sync STATE (client-local head bookkeeping, FULL-digest namespace) -# -# Records the last commit HEAD we pushed/pulled and, per synced skill, the tree -# hash of the on-disk content at that point. Distinct from the bundled manifest -# (skills_sync.py, truncated local content_hash namespace) AND from the -# `sync-manifest` OBJECT in the sync plane (the per-skill opt-in content). This -# is purely local reconciliation bookkeeping. Lives at -# ~/.hermes/skills/.sync_state as JSON. -# -# NOTE: renamed from `.sync_manifest` -> `.sync_state` to remove the collision -# with the plane `sync-manifest`. `read_sync_state` migrates an existing -# `.sync_manifest` on first read so no local head record is lost. -# --------------------------------------------------------------------------- def _sync_state_path() -> Path: return _skills_dir() / ".sync_state" -def _legacy_sync_state_path() -> Path: - return _skills_dir() / ".sync_manifest" +def _load_state_file(path: Path, what: str = "sync state read") -> Optional[Dict[str, Any]]: + """Parse a state file; None if missing / corrupt / not a dict.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + logger.debug("skills_sync_client: %s failed: %s", what, e) + return None + if not isinstance(data, dict): + return None + data.setdefault("head", None) + data.setdefault("skills", {}) + return data def read_sync_state() -> Dict[str, Any]: - """Read the local sync state. Returns a default on missing/corrupt. - - Shape: ``{"head": "sha256:...|null", "skills": {name: {tree, commit}}}``. - ``head`` is the last profile-root HEAD commit we reconciled with. - - Migrates a legacy ``.sync_manifest`` file (pre-rename) transparently: if the - new ``.sync_state`` is absent but the legacy file exists, it is read and - rewritten to the new path so an existing device keeps its head record. - """ + """``{"head": "sha256:...|null", "skills": {...}}``; a default on missing/corrupt. + If ``.sync_state`` is absent but the legacy ``.sync_manifest`` exists, it is + read, rewritten to the new path and removed, so no head record is lost.""" path = _sync_state_path() - if not path.exists(): - legacy = _legacy_sync_state_path() - if legacy.exists(): - try: - data = json.loads(legacy.read_text(encoding="utf-8")) - if isinstance(data, dict): - data.setdefault("head", None) - data.setdefault("skills", {}) - write_sync_state(data) # migrate to the new path - try: - legacy.unlink() - except OSError: - pass - return data - except (OSError, json.JSONDecodeError) as e: - logger.debug("skills_sync_client: legacy sync state migrate failed: %s", e) - return {"head": None, "skills": {}} - try: - data = json.loads(path.read_text(encoding="utf-8")) - if isinstance(data, dict): - data.setdefault("head", None) - data.setdefault("skills", {}) + if path.exists(): + return _load_state_file(path) or dict(_EMPTY_STATE) + legacy = _skills_dir() / ".sync_manifest" + if legacy.exists(): + data = _load_state_file(legacy, "legacy sync state migrate") + if data is not None: + write_sync_state(data) + with suppress(OSError): + legacy.unlink() return data - except (OSError, json.JSONDecodeError) as e: - logger.debug("skills_sync_client: sync state read failed: %s", e) - return {"head": None, "skills": {}} + return dict(_EMPTY_STATE) def write_sync_state(data: Dict[str, Any]) -> None: """Write the local sync state atomically. Best-effort.""" - import tempfile - path = _sync_state_path() try: path.parent.mkdir(parents=True, exist_ok=True) @@ -1005,101 +400,34 @@ def write_sync_state(data: Dict[str, Any]) -> None: os.fsync(f.fileno()) os.replace(tmp, path) except BaseException: - try: + with suppress(OSError): os.unlink(tmp) - except OSError: - pass raise except Exception as e: logger.debug("skills_sync_client: sync state write failed: %s", e) -# --------------------------------------------------------------------------- -# Tree materialization (pull) -- write a tree back to a skill directory -# --------------------------------------------------------------------------- - -def materialize_tree( - client: SyncClient, tree_hash: str, dest: Path, *, org_scope: bool = False -) -> None: - """Write the tree at *tree_hash* into *dest* (created if needed). - - Blobs become files (with +x restored for ``exec`` mode), nested trees - become subdirectories. Does NOT delete files absent from the tree -- the - caller decides removal semantics. Refuses path traversal via entry names. - """ - dest.mkdir(parents=True, exist_ok=True) - tree = client.get_tree_json(tree_hash, org_scope=org_scope) - for entry in tree.get("entries", []): - name = entry.get("name", "") - if not name or "/" in name or name in (".", ".."): - logger.warning("skills_sync_client: skipping unsafe tree entry %r", name) - continue - target = dest / name - kind = entry.get("kind") - if kind == KIND_TREE: - materialize_tree(client, entry["hash"], target, org_scope=org_scope) - elif kind == KIND_BLOB: - _, data = client.get_object(entry["hash"], org_scope=org_scope) - target.write_bytes(data) - if entry.get("mode") == MODE_EXEC: - try: - st = target.stat().st_mode - target.chmod(st | _stat.S_IXUSR | _stat.S_IXGRP | _stat.S_IXOTH) - except OSError: - pass +def _record_head(state: Dict[str, Any], head: str, root: str) -> None: + state["head"] = head + state["root"] = root + write_sync_state(state) -# --------------------------------------------------------------------------- -# Profile snapshot -- build the objects + per-skill tree map for a push -# -# The profile root is a tree whose entries mirror each synced skill's relative -# path under ~/.hermes/skills/ (the sync contract: "the profile root is a tree -# whose entries are category trees"). Only opted-in, eligible skills are -# included (personal sync opt-in + eligibility). -# --------------------------------------------------------------------------- - -def _skill_rel_path(skill_name: str) -> Optional[PurePosixPath]: - """Return the skill's path relative to ~/.hermes/skills/ (posix), or None.""" - try: - from tools.skill_usage import _find_skill_dir - except Exception: - return None - skill_dir = _find_skill_dir(skill_name) - if skill_dir is None: - return None - try: - rel = skill_dir.resolve().relative_to(_skills_dir().resolve()) - except (OSError, ValueError): - return None - return PurePosixPath(rel.as_posix()) - +# Profile snapshot -- the root tree mirrors each synced skill's relative path +# under ~/.hermes/skills/ (category dirs become intermediate trees). def snapshot_profile( skill_names: List[str], *, max_object_bytes: int = DEFAULT_MAX_OBJECT_BYTES ) -> Tuple[ObjectSet, str, Dict[str, str]]: - """Build all objects for *skill_names* + the profile-root tree. - - Returns ``(objects, root_tree_hash, skill_tree_map)`` where - ``skill_tree_map`` is ``{skill_name: tree_hash}``. Skills whose blobs - exceed *max_object_bytes* are skipped (surfaced via logger). - - The root tree nests category directories: a skill at ``devops/foo`` yields - a root entry ``devops`` (tree) containing ``foo`` (tree). Flat skills yield - a direct root entry. - - The root tree also carries a ``sync-manifest`` BLOB (design.md §2.8) - recording the per-skill opt-in state, so opt-in is durable + cross-device - rather than a device-local ``.usage.json`` flag. Every skill in - ``skill_names`` is recorded ``enabled: true`` (they ARE the opted-in set); - the manifest is the authoritative record the plane + other devices read. - """ + """Build all objects for *skill_names* + the profile-root tree; returns + ``(objects, root_tree_hash, {skill_name: tree_hash})``. Skills whose blobs + exceed *max_object_bytes* are skipped (logged). The root also carries the + ``sync-manifest`` blob listing every included skill as ``enabled: true``.""" from tools.skill_usage import _find_skill_dir objects = ObjectSet() skill_tree_map: Dict[str, str] = {} - # Nested dict representing the root: {name: {"__tree__": hash} | subdict} root: Dict[str, Any] = {} - for name in sorted(set(skill_names)): rel = _skill_rel_path(name) skill_dir = _find_skill_dir(name) @@ -1111,171 +439,48 @@ def snapshot_profile( logger.warning("skills_sync_client: skipping %s: %s", name, e) continue skill_tree_map[name] = tree_hash - # Insert into the nested root structure by relative path parts. - parts = list(rel.parts) - node = root - for part in parts[:-1]: - node = node.setdefault(part, {}) - node[parts[-1]] = {"__tree__": tree_hash} + nest_skill_tree(root, rel.parts, tree_hash) - # sync-manifest: record the opt-in state (the pushed set = enabled). - # Only skills that actually made it into the tree are recorded, keyed by the - # skill NAME (matching gateway-gateway's manifest shape + the read walk that - # enumerates skill subtrees by name). - manifest_map = {name: True for name in skill_tree_map} - manifest_hash = objects.add( - KIND_BLOB, build_sync_manifest_bytes(manifest_map) - ) - - root_hash = _build_root_tree(root, objects, manifest_hash=manifest_hash) - return objects, root_hash, skill_tree_map + manifest_hash = objects.add(KIND_BLOB, build_sync_manifest_bytes(dict.fromkeys(skill_tree_map, True))) + return objects, build_root_tree(root, objects, manifest_hash=manifest_hash), skill_tree_map -def _build_root_tree( - node: Dict[str, Any], objects: ObjectSet, *, manifest_hash: Optional[str] = None -) -> str: - """Recursively canonicalize the nested root structure into trees. - - ``manifest_hash`` (only passed at the top level) adds a root-level - ``sync-manifest`` BLOB entry (design.md §2.8) alongside the skill subtrees. - It cannot collide with a skill dir (skill entries are trees; this is a blob). - """ - entries: List[Dict[str, str]] = [] - for name, child in node.items(): - if isinstance(child, dict) and "__tree__" in child and len(child) == 1: - entries.append( - {"name": name, "kind": KIND_TREE, "hash": child["__tree__"], "mode": MODE_DIR} - ) - else: - sub_hash = _build_root_tree(child, objects) - entries.append( - {"name": name, "kind": KIND_TREE, "hash": sub_hash, "mode": MODE_DIR} - ) - if manifest_hash is not None: - entries.append( - { - "name": SYNC_MANIFEST_ENTRY_NAME, - "kind": KIND_BLOB, - "hash": manifest_hash, - "mode": MODE_FILE, - } - ) - entries.sort(key=lambda e: e["name"]) - tree_obj = {"type": KIND_TREE, "entries": entries} - return objects.add(KIND_TREE, canonical_json_bytes(tree_obj)) - - -# --------------------------------------------------------------------------- -# Ref naming (sync contract) -# --------------------------------------------------------------------------- +# Personal refs, push, pull def user_head_ref(owner: str) -> str: return f"refs/user/{owner}/HEAD" -def user_conflict_ref(owner: str, n: int) -> str: - return f"refs/user/{owner}/conflict/{n}" +def _personal_client( + identity: Optional[Dict[str, Any]], client: Optional[SyncClient] +) -> Tuple[Dict[str, Any], Optional[SyncClient]]: + """Resolve identity + client for a personal sync op. ``client`` is None when + no base URL is configured (callers return a no-op result).""" + identity = identity if identity is not None else resolve_identity() + if client is None: + base = resolve_sync_base_url() + client = SyncClient(base, identity["api_key"]) if base else None + return identity, client -def _root_tree_of_commit( - client: "SyncClient", commit_hash: str, *, org_scope: bool = False -) -> str: - """Return the tree hash referenced by a commit.""" - return client.get_commit_json(commit_hash, org_scope=org_scope)["tree"] +_NO_BASE_URL = {"ok": False, "reason": "no sync base url configured", "noop": True} -def _skill_trees_of_root( - client: "SyncClient", root_tree_hash: str, *, org_scope: bool = False -) -> Dict[str, str]: - """Flatten a profile-root tree into ``{posix_rel_path: skill_tree_hash}``. - - A skill tree is any tree containing a ``SKILL.md`` blob entry. We walk the - root tree; a subtree with a SKILL.md is treated as a skill leaf keyed by - its path, so category nesting is preserved. - """ - result: Dict[str, str] = {} - - def _walk(tree_hash: str, prefix: str) -> None: - tree = client.get_tree_json(tree_hash, org_scope=org_scope) - entries = tree.get("entries", []) - has_skill_md = any( - e.get("name") == "SKILL.md" and e.get("kind") == KIND_BLOB for e in entries - ) - if has_skill_md and prefix: - result[prefix] = tree_hash - return - for e in entries: - if e.get("kind") == KIND_TREE: - child_prefix = f"{prefix}/{e['name']}" if prefix else e["name"] - _walk(e["hash"], child_prefix) - - _walk(root_tree_hash, "") - return result - - -def read_manifest_of_root( - client: "SyncClient", root_tree_hash: str -) -> Optional[Dict[str, bool]]: - """Read the ``sync-manifest`` blob at the root of *root_tree_hash* into - ``{name: enabled}`` (design.md §2.8), or ``None`` if there is no manifest - entry / it is malformed. - - The manifest is a root-level BLOB entry named ``sync-manifest`` (never a - skill subtree). This is how a device learns the cross-device opt-in state - written by another device's push. - """ - try: - tree = client.get_tree_json(root_tree_hash) - except Exception as e: - logger.debug("skills_sync_client: manifest root read failed: %s", e) - return None - for e in tree.get("entries", []): - if e.get("name") == SYNC_MANIFEST_ENTRY_NAME and e.get("kind") == KIND_BLOB: - try: - _kind, data = client.get_object(e["hash"]) - except Exception as ex: - logger.debug("skills_sync_client: manifest blob fetch failed: %s", ex) - return None - return parse_sync_manifest(data) - return None - - -def _check_version(caps: Dict[str, Any]) -> None: - """Reject an incompatible server major version (sync contract).""" - ver = str(caps.get("hsp_version") or "") # wire field name - major = ver.split(".", 1)[0] - if major != WIRE_VERSION: - raise SyncError( - f"this server speaks sync version {ver!r}, but this Hermes speaks " - f"{WIRE_VERSION} — update Hermes to sync with it" - ) - - -# --------------------------------------------------------------------------- -# Push -# --------------------------------------------------------------------------- - def push_skills( - client: Optional["SyncClient"] = None, + client: Optional[SyncClient] = None, *, skill_names: Optional[List[str]] = None, identity: Optional[Dict[str, Any]] = None, message: str = "hermes skill sync", ) -> Dict[str, Any]: - """Push opted-in skills to the owner's HEAD (sync contract). - - Uploads all new objects, then CAS-es ``refs/user//HEAD``. On a 409, - fetches the actual head, three-way merges, and retries once (§4.4 / M1-C). - Returns a result dict; never raises for the inert / no-op cases. - """ - if identity is None: - identity = resolve_identity() - owner = identity["owner"] + """Push opted-in skills to ``refs/user//HEAD``: upload new objects, CAS + HEAD. A 409 with an actual head -> three-way merge + one retry; a 409 against + a NON-EXISTENT ref (stale local head, e.g. carried over from another plane) + -> redo the CAS as a create. Never raises for the inert / no-op cases.""" + identity, client = _personal_client(identity, client) if client is None: - base = resolve_sync_base_url() - if not base: - return {"ok": False, "reason": "no sync base url configured", "noop": True} - client = SyncClient(base, identity["api_key"]) + return dict(_NO_BASE_URL) + owner = identity["owner"] if skill_names is None: skill_names = list_synced_skill_names() @@ -1285,394 +490,192 @@ def push_skills( caps = client.capabilities() _check_version(caps) max_bytes = int(caps.get("max_object_bytes") or DEFAULT_MAX_OBJECT_BYTES) - objects, root_hash, _ = snapshot_profile(skill_names, max_object_bytes=max_bytes) - manifest = read_sync_state() - base_head = manifest.get("head") - - # Idempotency: if the profile-root tree is unchanged since our last push, - # there is nothing to propagate -- skip building an empty commit (contract - # objects are immutable, so an identical tree hash means identical content). - if base_head and manifest.get("root") == root_hash: + state = read_sync_state() + base_head = state.get("head") + # Idempotency: objects are immutable, so an unchanged root tree hash means + # identical content -- skip building an empty commit. + if base_head and state.get("root") == root_hash: return {"ok": True, "head": base_head, "reason": "unchanged", "noop": True} - device = stable_device_id() - parents = [base_head] if base_head else [] commit_hash = build_commit( - root_hash, parents, owner=owner, device=device, message=message, objects=objects + root_hash, [base_head] if base_head else [], owner=owner, device=stable_device_id(), + message=message, objects=objects, ) - client.put_objects(objects.objects) ref = user_head_ref(owner) - + result = {"ok": True, "head": commit_hash, "pushed_objects": len(objects)} try: client.cas_ref(ref, base_head, commit_hash) - manifest["head"] = commit_hash - manifest["root"] = root_hash - write_sync_state(manifest) - return {"ok": True, "head": commit_hash, "pushed_objects": len(objects)} except SyncConflict as conflict: - if not conflict.actual: - # The ref does not exist server-side: our `from` was a stale head - # (commonly a local state file carried over from another sync - # plane). There is nothing to merge — redo the CAS as a create. - client.cas_ref(ref, None, commit_hash) - manifest["head"] = commit_hash - manifest["root"] = root_hash - write_sync_state(manifest) - return { - "ok": True, - "head": commit_hash, - "pushed_objects": len(objects), - "recovered_stale_head": True, - } - return _resolve_push_conflict( - client, identity, conflict.actual, root_hash, commit_hash, - objects, skill_names, message, base_head, - ) + if conflict.actual: + return _resolve_push_conflict( + client, identity, conflict.actual, root_hash, commit_hash, objects, message, base_head + ) + client.cas_ref(ref, None, commit_hash) + result["recovered_stale_head"] = True + _record_head(state, commit_hash, root_hash) + return result -# --------------------------------------------------------------------------- -# Conflict resolution / three-way merge -# -# On a 409 the server hands back the actual head. We fetch it, three-way merge -# per skill against the base we forked from, reusing the origin/user/incoming -# decision semantics of skills_sync.py (_is_tracked_user_modification + -# the decision block at skills_sync.py:619-643): -# -# * base == ours == theirs -> nothing to do -# * ours == base, theirs moved -> take theirs (fast-forward incoming) -# * theirs == base, ours moved -> keep ours (our local edit) -# * both moved, ours == theirs -> converged; take either -# * both moved, differ -> TRUE OVERLAP -> conflict head -# -# Non-overlapping merges (each side changed a DIFFERENT skill) produce a merge -# commit (2 parents) and retry the CAS. A true overlap (both sides changed the -# SAME skill differently) is written to refs/user//conflict/ and -# surfaced for out-of-band resolution. -# --------------------------------------------------------------------------- - +# Three-way merge per skill against the base we forked from (see merge_skill): +# each side changing a DIFFERENT skill -> merge commit (2 parents) + CAS retry; +# both changing the SAME skill differently -> TRUE OVERLAP -> written to +# refs/user//conflict/ and surfaced for out-of-band resolution. def _resolve_push_conflict( - client: "SyncClient", + client: SyncClient, identity: Dict[str, Any], actual_head: str, our_root: str, our_commit: str, - objects: "ObjectSet", - skill_names: List[str], + objects: ObjectSet, message: str, base_head: Optional[str], ) -> Dict[str, Any]: owner = identity["owner"] - device = stable_device_id() - - theirs_root = _root_tree_of_commit(client, actual_head) - base_root = _root_tree_of_commit(client, base_head) if base_head else None - - ours_trees = _skill_trees_of_root(client, our_root) - theirs_trees = _skill_trees_of_root(client, theirs_root) - base_trees = _skill_trees_of_root(client, base_root) if base_root else {} + ours_trees = skill_trees_of_root(client, our_root) + theirs_trees = skill_trees_of_root(client, root_tree_of_commit(client, actual_head)) + base_trees = skill_trees_of_root(client, root_tree_of_commit(client, base_head)) if base_head else {} merged: Dict[str, str] = {} overlaps: List[str] = [] - all_paths = set(ours_trees) | set(theirs_trees) | set(base_trees) - for path in all_paths: - o = ours_trees.get(path) - t = theirs_trees.get(path) - b = base_trees.get(path) - decision = _merge_skill(b, o, t) + for path in set(ours_trees) | set(theirs_trees) | set(base_trees): + o, t = ours_trees.get(path), theirs_trees.get(path) + decision = merge_skill(base_trees.get(path), o, t) if decision == "overlap": overlaps.append(path) - # Keep OURS on the surfaced conflict head; theirs is retained - # server-side under the conflict ref for out-of-band resolution. - if o is not None: - merged[path] = o - elif decision == "ours" and o is not None: - merged[path] = o - elif decision == "theirs" and t is not None: - merged[path] = t - elif decision == "either": - merged[path] = o if o is not None else t # type: ignore[assignment] - # decision == "none": skill deleted on the winning side -> drop + # overlap keeps OURS on the surfaced conflict head (theirs stays + # server-side); "none" = deleted on the winning side -> drop. + pick = {"overlap": o, "ours": o, "theirs": t, "either": o if o is not None else t}.get(decision) + if pick is not None: + merged[path] = pick if overlaps: - # TRUE OVERLAP -> write a conflict head and surface it (personal sync). - n = _next_conflict_index(client, owner) - conflict_ref = user_conflict_ref(owner, n) + conflict_ref = f"refs/user/{owner}/conflict/{_next_conflict_index(client, owner)}" try: client.cas_ref(conflict_ref, None, our_commit) except SyncConflict: pass # someone else grabbed this index; the head still exists return { - "ok": False, - "conflict": True, - "conflict_ref": conflict_ref, - "overlapping_skills": sorted(overlaps), - "actual_head": actual_head, + "ok": False, "conflict": True, "conflict_ref": conflict_ref, + "overlapping_skills": sorted(overlaps), "actual_head": actual_head, "message": ( f"{len(overlaps)} skill(s) changed on both sides; wrote " f"{conflict_ref}. Resolve out-of-band (hermes sync / NAS UI)." ), } - # Non-overlap -> build a merge commit (parents: base->actual, ours) and - # retry the CAS against the actual head. + # Merge commit (parents: actual, ours); re-add our objects so the merge + # push is self-contained. merge_objects = ObjectSet() - # Re-add our objects so the merge push is self-contained (idempotent). - for h, (kind, data) in objects.objects.items(): - merge_objects.objects[h] = (kind, data) - merged_root = _assemble_root_from_skill_trees(client, merged, merge_objects) + merge_objects.objects.update(objects.objects) + merged_root = assemble_root_from_skill_trees(merged, merge_objects) merge_commit = build_commit( - merged_root, - [actual_head, our_commit], - owner=owner, - device=device, - message=f"merge: {message}", - objects=merge_objects, + merged_root, [actual_head, our_commit], owner=owner, device=stable_device_id(), + message=f"merge: {message}", objects=merge_objects, ) client.put_objects(merge_objects.objects) try: client.cas_ref(user_head_ref(owner), actual_head, merge_commit) except SyncConflict as c2: return { - "ok": False, - "conflict": True, + "ok": False, "conflict": True, "actual_head": c2.actual, "message": f"merge CAS lost again (head now {c2.actual}); retry sync.", - "actual_head": c2.actual, } - manifest = read_sync_state() - manifest["head"] = merge_commit - manifest["root"] = merged_root - write_sync_state(manifest) + _record_head(read_sync_state(), merge_commit, merged_root) return {"ok": True, "head": merge_commit, "merged": True} -def _merge_skill(base: Optional[str], ours: Optional[str], theirs: Optional[str]) -> str: - """Three-way decision for one skill's tree hash. - - Returns one of: ``ours``, ``theirs``, ``either``, ``overlap``, ``none``. - Mirrors the origin/user/incoming decision block of skills_sync.py:619-643: - a side "modified" the skill when its hash differs from the common base - (analogous to ``_is_tracked_user_modification(origin, current)``). - """ - if ours == theirs: - return "either" if ours is not None else "none" - ours_changed = ours != base - theirs_changed = theirs != base - if ours_changed and not theirs_changed: - return "ours" - if theirs_changed and not ours_changed: - return "theirs" - # both changed and differ - return "overlap" - - -def _assemble_root_from_skill_trees( - client: "SyncClient", skill_trees: Dict[str, str], objects: "ObjectSet" -) -> str: - """Build a profile-root tree object from ``{posix_rel_path: tree_hash}``. - - Rebuilds the intermediate category trees. The referenced skill trees are - assumed already durable (they came from either side of the merge); only - the new intermediate/root tree objects are added to *objects*. - """ - root: Dict[str, Any] = {} - for path, tree_hash in skill_trees.items(): - parts = PurePosixPath(path).parts - node = root - for part in parts[:-1]: - node = node.setdefault(part, {}) - node[parts[-1]] = {"__tree__": tree_hash} - return _build_root_tree(root, objects) - - -def _next_conflict_index(client: "SyncClient", owner: str) -> int: - """Pick the next free conflict ref index for the owner.""" +def _next_conflict_index(client: SyncClient, owner: str) -> int: + """Next free ``conflict/`` index for the owner.""" try: refs = client.get_refs(f"refs/user/{owner}/conflict/") except SyncError: return 1 - used = [] - for r in refs: - name = r.get("name", "") - tail = name.rsplit("/", 1)[-1] - if tail.isdigit(): - used.append(int(tail)) + used = [int(t) for t in (r.get("name", "").rsplit("/", 1)[-1] for r in refs) if t.isdigit()] return (max(used) + 1) if used else 1 -# --------------------------------------------------------------------------- -# Pull -# --------------------------------------------------------------------------- - def pull_skills( - client: Optional["SyncClient"] = None, - *, - identity: Optional[Dict[str, Any]] = None, + client: Optional[SyncClient] = None, *, identity: Optional[Dict[str, Any]] = None ) -> Dict[str, Any]: - """Pull the owner's HEAD and materialize opted-in skills to disk. - - Fetches ``refs/user//HEAD``; if it advanced past our recorded head, - walks the profile-root tree and writes each skill tree into - ~/.hermes/skills/. Only paths the user has opted into (``sync: true``) are - materialized, so a pull never resurrects a skill the user hasn't chosen. - Best-effort; returns a result dict. - """ - if identity is None: - identity = resolve_identity() - owner = identity["owner"] + """Pull the owner's HEAD and materialize opted-in skills under ~/.hermes/skills/ + if it advanced past our recorded head. Opt-in intent is first adopted FROM + the plane manifest, then only opted-in paths are written so a pull never + resurrects a skill the user hasn't chosen. Best-effort; returns a result dict.""" + identity, client = _personal_client(identity, client) if client is None: - base = resolve_sync_base_url() - if not base: - return {"ok": False, "reason": "no sync base url configured", "noop": True} - client = SyncClient(base, identity["api_key"]) + return dict(_NO_BASE_URL) + owner = identity["owner"] - caps = client.capabilities() - _check_version(caps) - - refs = client.get_refs(user_head_ref(owner)) - head = None - for r in refs: - if r.get("name") == user_head_ref(owner): - head = r.get("hash") - break + _check_version(client.capabilities()) + head = read_ref_hash(client, user_head_ref(owner)) if not head: return {"ok": True, "reason": "no remote HEAD yet", "noop": True} - manifest = read_sync_state() - if head == manifest.get("head"): + state = read_sync_state() + if head == state.get("head"): return {"ok": True, "reason": "already up to date", "head": head, "noop": True} - root_tree = _root_tree_of_commit(client, head) - remote_trees = _skill_trees_of_root(client, root_tree) - - # : reconcile local opt-in intent FROM the plane manifest, so a skill the - # user opted in on another device becomes opted in here too (opt-in is - # cross-device content, not a device-local flag). We only ADOPT enables from - # the manifest for skills present in the remote tree; we never silently - # disable a locally-enabled skill on pull (that stays the user's local call - # until their next push reconciles it). - reconciled_from_manifest: List[str] = [] - remote_manifest = read_manifest_of_root(client, root_tree) - if remote_manifest: - try: - from tools.skill_usage import set_sync, is_curation_eligible, is_sync_enabled - - for sname, enabled in remote_manifest.items(): - if not enabled: - continue - if not is_curation_eligible(sname): - continue - if not is_sync_enabled(sname): - set_sync(sname, True) - reconciled_from_manifest.append(sname) - except Exception as e: - logger.debug("skills_sync_client: manifest opt-in reconcile failed: %s", e) + root_tree = root_tree_of_commit(client, head) + remote_trees = skill_trees_of_root(client, root_tree) + adopted = _adopt_manifest_opt_ins(read_manifest_of_root(client, root_tree)) opted_in = set(_opted_in_rel_paths()) - updated = [] - for path, tree_hash in remote_trees.items(): - # Opt-in gate on pull: only materialize skills the user chose to sync - # (now including any adopted from the plane manifest above). - if opted_in and path not in opted_in: - continue - dest = _skills_dir() / path - materialize_tree(client, tree_hash, dest) - updated.append(path) + updated = [path for path in remote_trees if not opted_in or path in opted_in] + for path in updated: + materialize_tree(client, remote_trees[path], _skills_dir() / path) - manifest["head"] = head - write_sync_state(manifest) - return { - "ok": True, - "head": head, - "updated": sorted(updated), - "opt_in_adopted": sorted(reconciled_from_manifest), - } + state["head"] = head + write_sync_state(state) + return {"ok": True, "head": head, "updated": sorted(updated), "opt_in_adopted": sorted(adopted)} -def _opted_in_rel_paths() -> List[str]: - """Relative posix paths of skills the user has opted into sync.""" - paths = [] - for name in list_synced_skill_names(): - rel = _skill_rel_path(name) - if rel is not None: - paths.append(rel.as_posix()) - return paths +# Gated public entrypoints (gate-and-swallow, like the curator's +# maybe_run_curator): best-effort, never raise, return a result dict or None. - -# --------------------------------------------------------------------------- -# Gated public entrypoints (gate-and-swallow) -# -# maybe_pull_skills / maybe_push_skills clone the shape of the curator's -# maybe_run_curator (agent/curator.py:1998): best-effort, never raise, return -# a result dict or None. The access gate is checked first -- sync is inert -# (no push, no pull, no-op) unless the signed-in user is a Nous admin. -# --------------------------------------------------------------------------- - -def maybe_push_skills(*, message: str = "hermes skill sync") -> Optional[Dict[str, Any]]: - """Best-effort push if all gates pass. Returns a result dict or None. - Never raises. Called from the debounced skill_manage push hook.""" +def _gate_and_swallow(op: str, run: Callable[[Dict[str, Any]], Optional[Dict[str, Any]]]): + """Run *run(identity)* only if every background-sync gate holds (Nous admin, + feature on, base URL); None when inert or on any error.""" try: identity = resolve_identity() - if not identity.get("nous_admin"): - return None # access gate: inert unless the user is a Nous admin - if not sync_feature_enabled(): - return None # feature off for this instance (HERMES_SYNC_ENABLED) - if not resolve_sync_base_url(): + if not identity.get("nous_admin") or not sync_feature_enabled() or not resolve_sync_base_url(): return None - if not list_synced_skill_names(): - return None - return push_skills(identity=identity, message=message) + return run(identity) except Exception as e: - logger.debug("skills_sync_client: maybe_push_skills failed: %s", e, exc_info=True) + logger.debug("skills_sync_client: %s failed: %s", op, e, exc_info=True) return None +def maybe_push_skills(*, message: str = "hermes skill sync") -> Optional[Dict[str, Any]]: + """Best-effort push (debounced skill_manage hook). Never raises.""" + return _gate_and_swallow( + "maybe_push_skills", + lambda identity: push_skills(identity=identity, message=message) if list_synced_skill_names() else None, + ) + + def maybe_pull_skills() -> Optional[Dict[str, Any]]: - """Best-effort pull if all gates pass. Returns a result dict or None. - Never raises. Invoked at the curator tick sites (gateway housekeeping loop - + CLI startup).""" - try: - identity = resolve_identity() - if not identity.get("nous_admin"): - return None # access gate: inert unless the user is a Nous admin - if not sync_feature_enabled(): - return None # feature off for this instance (HERMES_SYNC_ENABLED) - if not resolve_sync_base_url(): - return None - return pull_skills(identity=identity) - except Exception as e: - logger.debug("skills_sync_client: maybe_pull_skills failed: %s", e, exc_info=True) - return None + """Best-effort pull (curator tick sites: gateway housekeeping + CLI startup). + Never raises.""" + return _gate_and_swallow("maybe_pull_skills", lambda identity: pull_skills(identity=identity)) def sync_status() -> Dict[str, Any]: - """Return a status snapshot for ``hermes sync status``. Never raises.""" + """Status snapshot for ``hermes sync status``. Never raises. + ``org_available`` False means the account isn't in a shared organisation + (the org workflow does not apply), not that anything is broken.""" status: Dict[str, Any] = { - "nous_admin": False, - "logged_in": False, - "feature_enabled": sync_feature_enabled(), - "default_opt_in": sync_default_opt_in(), - "base_url": resolve_sync_base_url(), - "opted_in_skills": [], - "local_head": None, - "owner": None, - # Org-shared skills. `org_available` is False for an account that - # isn't in a shared organisation — the org workflow does not apply, - # which is different from it being broken or misconfigured. - "org_available": False, - "org_id": None, - "org_role": None, - "org_skills": [], - # Org skills edited locally and not yet shared back. - "org_skills_modified": [], + "nous_admin": False, "logged_in": False, "feature_enabled": sync_feature_enabled(), + "default_opt_in": sync_default_opt_in(), "base_url": resolve_sync_base_url(), + "opted_in_skills": [], "local_head": None, "owner": None, "org_available": False, + "org_id": None, "org_role": None, "org_skills": [], "org_skills_modified": [], } + try: identity = resolve_identity() - status["logged_in"] = True - status["owner"] = identity.get("owner") - status["nous_admin"] = bool(identity.get("nous_admin")) + status.update(logged_in=True, owner=identity.get("owner"), nous_admin=bool(identity.get("nous_admin"))) except SyncInertError: pass except Exception as e: @@ -1684,12 +687,10 @@ def sync_status() -> Dict[str, Any]: pass try: org_identity = resolve_org_identity() - status["org_available"] = True - status["org_id"] = org_identity.get("org_id") - status["org_role"] = org_identity.get("org_role") - status["org_skills"] = list_org_skill_names() - status["org_skills_modified"] = list_locally_modified_org_skills( - status["org_id"] + status.update( + org_available=True, org_id=org_identity.get("org_id"), org_role=org_identity.get("org_role"), + org_skills=list_org_skill_names(), + org_skills_modified=list_locally_modified_org_skills(org_identity.get("org_id")), ) except SyncInertError: pass @@ -1698,490 +699,12 @@ def sync_status() -> Dict[str, Any]: return status -def list_org_skill_names() -> List[str]: - """Skill names present in the local org mirror (empty when none pulled).""" - names: List[str] = [] - try: - from agent.skill_utils import read_active_org_id - - org_id = read_active_org_id(_skills_dir()) - if not org_id: - return names - root = _org_dir() / org_id - if not root.is_dir(): - return names - for skill_md in root.rglob("SKILL.md"): - rel = skill_md.parent.relative_to(root) - if rel.parts: - names.append(str(rel).replace("\\", "/")) - except Exception as e: - logger.debug("skills_sync_client: org skill listing failed: %s", e) - return sorted(names) - - -# --------------------------------------------------------------------------- -# Org-shared skills (sync contract) — org pull + propose. -# -# Org skills live under a DISTINCT local namespace, ~/.hermes/skills/_org/ -# (the design notes: enterprise-managed skills are read-only to the runtime; a -# local edit is a personal fork of record until proposed). The org canonical -# set is `refs/org//HEAD` — the SAME object model as personal sync. -# -# PERSONAL-ORG GATE (the sync contract REFINED, Ben 2026-07-23): a personal org -# has NO org workflow. The discriminator travels in the token: NAS stamps the -# `org_role` claim ONLY for multi-member orgs. No claim ⇒ every org helper -# here is inert (org_sync_available() False; pull/propose raise SyncInertError) -# and the personal personal sync experience is untouched. -# -# `hermes sync propose` is the org sharing surface; proposal is -# intended to become largely automated later (curator/background hooks driving -# the same propose_skill() path). Keep this callable non-interactive. -# --------------------------------------------------------------------------- - -ORG_DIR_NAME = "_org" - - -def resolve_org_identity() -> Dict[str, Any]: - """Resolve identity + org context for org-skill operations. - - Returns ``resolve_identity()``'s dict extended with ``org_id`` and - ``org_role``. Raises :class:`SyncInertError` when the token carries no - ``org_role`` claim (personal org / issuer predates org support) — the - caller should treat org sync as unavailable, NOT as an error. - """ - identity = resolve_identity() - claims = identity.get("claims") or {} - org_id = claims.get("org_id") - org_role = claims.get("org_role") - if not org_id: - raise SyncInertError("no organisation associated with this account") - if not isinstance(org_role, str) or not org_role: - raise SyncInertError( - "this account isn't a member of a shared organisation" - ) - identity["org_id"] = str(org_id) - identity["org_role"] = org_role - return identity - - -def org_sync_available() -> bool: - """True iff this token can see the org-skill surface (multi-member org).""" - try: - resolve_org_identity() - return True - except Exception: - return False - - -# How many times a propose will re-splice onto a moved org HEAD before giving -# up. Small: contention means other members are actively proposing, and an -# unbounded loop would spin. -_ORG_CAS_MAX_ATTEMPTS = 5 - - -def _read_org_head(client: "SyncClient", org_id: str) -> Optional[str]: - """Current ``refs/org//HEAD``, or None if the org has no content. - - Reads through the ORG endpoint. The personal refs route is scoped to the - caller's own owner and answers an ``refs/org/...`` prefix with the caller's - PERSONAL refs, so a personal-route read here silently reports "no org head" - and every subsequent CAS races against a head it never saw. - """ - refs = client.get_refs(f"refs/org/{org_id}/", org_scope=True) - return next( - (r["hash"] for r in refs if r.get("name") == org_head_ref(org_id)), None - ) - - -def org_head_ref(org_id: str) -> str: - return f"refs/org/{org_id}/HEAD" - - -def _org_dir() -> Path: - """Local mirror root for org skills (read-only by convention ).""" - return _skills_dir() / ORG_DIR_NAME - - -def pull_org_skills( - client: Optional["SyncClient"] = None, - *, - identity: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """Pull the org canonical set into ``~/.hermes/skills/_org//``. - - Fast-forward only (design.md §2.6: no client merge on the org path): the - mirror is replaced with the org HEAD's content. Local edits under _org/ - are NOT merged — they are overwritten on pull; a member's change of record - is `propose_skill` (the fork lives in their personal skills, not _org/). - Returns {ok, org_id, head, updated} (updated = skill rel-paths written). - """ - identity = identity or resolve_org_identity() - if "org_id" not in identity: - raise SyncInertError("no organisation context available") - org_id = identity["org_id"] - if client is None: - base_url = resolve_sync_base_url() - if not base_url: - raise SyncInertError("no sync base URL configured") - client = SyncClient(base_url, identity["api_key"]) - - caps = client.capabilities() - _check_version(caps) - if "org" not in (caps.get("features") or []): - raise SyncInertError("this server does not support org-shared skills") - - head = _read_org_head(client, org_id) - # TOKEN-GATED resolution marker (agent/skill_utils.read_active_org_id): - # written HERE because this function only runs after resolve_org_identity - # verified the token's org_id + org_role. Discovery scans only the marked - # org's mirror, so a stale mirror from a previous org stops resolving the - # moment a pull runs under a different org — no manual cleanup. - _write_active_org_marker(org_id) - if not head: - return {"ok": True, "org_id": org_id, "head": None, "updated": []} - - head_commit = client.get_commit_json(head, org_scope=True) - root_tree = head_commit["tree"] - skill_trees = _skill_trees_of_root(client, root_tree, org_scope=True) - - dest_root = _org_dir() / org_id - updated: List[str] = [] - # Skills the user/agent has edited locally and upstream also changed. - # We do NOT overwrite them — the local work wins until the user resolves. - conflicted: List[str] = [] - baseline = _read_org_baseline(org_id) - for rel_path, tree_hash in sorted(skill_trees.items()): - dest = dest_root / PurePosixPath(rel_path) - try: - if dest.exists(): - # Local edits are protected: never clobber work the user or - # agent did in place. Skip the update and report it so they - # can resolve deliberately (propose the local version, or - # discard it and re-pull). - if org_skill_is_locally_modified(rel_path, org_id): - prev = baseline.get(rel_path) or {} - # Upstream also moved on => a real conflict the user must - # resolve. Upstream unchanged => their edit simply stands. - if prev.get("tree") != tree_hash: - conflicted.append(rel_path) - continue - import shutil - - shutil.rmtree(dest) - dest.mkdir(parents=True, exist_ok=True) - materialize_tree(client, tree_hash, dest, org_scope=True) - baseline[rel_path] = { - "fingerprint": _skill_dir_fingerprint(dest), - "tree": tree_hash, - } - updated.append(rel_path) - except Exception as e: - logger.warning( - "skills_sync_client: org skill materialize failed for %s: %s", - rel_path, - e, - ) - # Provenance sidecar for the load-time header (skill_view): the HEAD - # commit's author is TOKEN-VERIFIED at push time by the plane - # (author_mismatch guard, the sync plane) — trustworthy to display. - _write_org_provenance( - org_id, - { - "org_id": org_id, - "head": head, - "author_user_id": (head_commit.get("author") or {}).get("owner", ""), - "author_device": (head_commit.get("author") or {}).get("device", ""), - "ts": head_commit.get("ts", ""), - "skills": updated, - }, - ) - _write_org_baseline(org_id, baseline) - if conflicted: - logger.warning( - "skills_sync_client: %d org skill(s) have local edits AND upstream " - "changes; left untouched: %s", - len(conflicted), - ", ".join(conflicted), - ) - return { - "ok": True, - "org_id": org_id, - "head": head, - "updated": updated, - "conflicted": conflicted, - } - - -def _skill_dir_fingerprint(path: Path) -> str: - """Stable content hash of a materialized skill directory. - - Used to tell "the user/agent edited this org skill" from "this is exactly - what upstream shipped". Hashes every file's relative path + bytes, sorted, - so it is independent of filesystem ordering and mtimes. - """ - h = hashlib.sha256() - try: - for f in sorted(p for p in path.rglob("*") if p.is_file()): - h.update(str(f.relative_to(path)).replace("\\", "/").encode("utf-8")) - h.update(b"\0") - h.update(f.read_bytes()) - h.update(b"\0") - except OSError as e: - logger.debug("skills_sync_client: fingerprint failed for %s: %s", path, e) - return "" - return h.hexdigest() - - -def _org_baseline_path(org_id: str) -> Path: - """Sidecar recording the upstream fingerprint of each mirrored skill.""" - from agent.skill_utils import ORG_BASELINE_FILE - - return _org_dir() / org_id / ORG_BASELINE_FILE - - -def _read_org_baseline(org_id: str) -> Dict[str, Any]: - try: - return json.loads(_org_baseline_path(org_id).read_text(encoding="utf-8")) - except Exception: - return {} - - -def _write_org_baseline(org_id: str, baseline: Dict[str, Any]) -> None: - try: - p = _org_baseline_path(org_id) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(baseline, indent=2, sort_keys=True), encoding="utf-8") - except Exception as e: - logger.debug("skills_sync_client: baseline write failed: %s", e) - - -def org_skill_is_locally_modified(skill_rel_path: str, org_id: str) -> bool: - """True when the local copy of an org skill differs from what upstream sent.""" - dest = _org_dir() / org_id / PurePosixPath(skill_rel_path) - if not dest.is_dir(): - return False - entry = _read_org_baseline(org_id).get(skill_rel_path) or {} - recorded = entry.get("fingerprint") if isinstance(entry, dict) else entry - if not recorded: - # No baseline recorded (pre-existing mirror) — treat as unmodified so - # we don't cry wolf; the next pull records one. - return False - return _skill_dir_fingerprint(dest) != recorded - - -def list_locally_modified_org_skills(org_id: Optional[str] = None) -> List[str]: - """Org skills with local edits that upstream has not seen.""" - try: - from agent.skill_utils import read_active_org_id - - org_id = org_id or read_active_org_id(_skills_dir()) - if not org_id: - return [] - baseline = _read_org_baseline(org_id) - return sorted( - rel for rel in baseline if org_skill_is_locally_modified(rel, org_id) - ) - except Exception as e: - logger.debug("skills_sync_client: modified-scan failed: %s", e) - return [] - - -def _write_active_org_marker(org_id: str) -> None: - """Record which org's mirror may resolve (best-effort, never raises).""" - try: - from agent.skill_utils import ORG_ACTIVE_MARKER - - root = _org_dir() - root.mkdir(parents=True, exist_ok=True) - (root / ORG_ACTIVE_MARKER).write_text(org_id, encoding="utf-8") - except Exception as e: - logger.debug("skills_sync_client: active-org marker write failed: %s", e) - - -def _write_org_provenance(org_id: str, data: Dict[str, Any]) -> None: - """Persist the org HEAD provenance sidecar (best-effort, never raises).""" - try: - from agent.skill_utils import ORG_PROVENANCE_FILE - - dest = _org_dir() / org_id - dest.mkdir(parents=True, exist_ok=True) - (dest / ORG_PROVENANCE_FILE).write_text( - json.dumps(data, indent=2), encoding="utf-8" - ) - except Exception as e: - logger.debug("skills_sync_client: org provenance write failed: %s", e) - - -def propose_skill( - skill_name: str, - client: Optional["SyncClient"] = None, - *, - identity: Optional[Dict[str, Any]] = None, - message: Optional[str] = None, -) -> Dict[str, Any]: - """Propose a local skill's current content to the org canonical set. - - Snapshots the LOCAL (personal) skill directory as an org-scoped commit - layered on the current org HEAD tree (splice/replace that one skill - subtree), uploads the objects with ``?scope=org``, then CAS-es the org - HEAD (contract §11.5): - - - ADMIN/OWNER token → the server merges directly → ``{ok, merged: True}``. - - MEMBER token → the server converts to a proposal (202) → - ``{ok, proposal_pending: True, proposal_id, ref}``. NEVER presented as - live/merged. - - Non-interactive by design — an automated submitter (curator hook) drives - this exact function later (Ben's automation trajectory). - """ - identity = identity or resolve_org_identity() - org_id = identity["org_id"] - if client is None: - base_url = resolve_sync_base_url() - if not base_url: - raise SyncInertError("no sync base URL configured") - client = SyncClient(base_url, identity["api_key"]) - - caps = client.capabilities() - _check_version(caps) - if "org" not in (caps.get("features") or []): - raise SyncInertError("this server does not support org-shared skills") - max_bytes = int(caps.get("max_object_bytes") or DEFAULT_MAX_OBJECT_BYTES) - - # Locate the local skill directory (personal namespace, NOT _org/). - rel = _skill_rel_path(skill_name) - if rel is None: - raise SyncError(f"skill '{skill_name}' not found under the skills dir") - skill_dir = _skills_dir() / rel - if not (skill_dir / "SKILL.md").exists(): - raise SyncError(f"skill '{skill_name}' has no SKILL.md") - - # Build the proposed skill tree. - objects = ObjectSet() - skill_tree = build_tree(skill_dir, objects, max_object_bytes=max_bytes) - - # Base = current org HEAD (None for the org's first content). The proposed - # root is HEAD's skill-tree map with this one skill spliced in — proposals - # are per-skill deltas, never a wholesale replace of the org set. - # - # Wrapped in a bounded retry: between reading HEAD and the CAS, another - # member's propose (or an admin merge) can advance it. The server answers - # 409 with the new head; we re-splice this one skill onto THAT head and try - # again rather than surfacing a raw conflict. Re-splicing (not replaying - # the old root) is what keeps the other member's skill from being dropped. - attempts = 0 - while True: - attempts += 1 - base_head = _read_org_head(client, org_id) - if base_head: - base_root = _root_tree_of_commit(client, base_head, org_scope=True) - skill_map = _skill_trees_of_root(client, base_root, org_scope=True) - else: - skill_map = {} - skill_map[str(rel)] = skill_tree - - root_hash = _assemble_root_from_skill_trees(client, skill_map, objects) - commit_hash = build_commit( - root_hash, - [base_head] if base_head else [], - owner=identity["owner"], - device=stable_device_id(), - message=message or f"propose {skill_name}", - objects=objects, - ) - - client.put_objects(objects.objects, org_scope=True) - try: - result = client.cas_ref(org_head_ref(org_id), base_head, commit_hash) - break - except SyncConflict as conflict: - if attempts >= _ORG_CAS_MAX_ATTEMPTS: - raise SyncError( - "the organisation's skills changed while this was being " - f"proposed, and {attempts} attempts to catch up all lost " - "the race — run the command again", - status=409, - ) from conflict - logger.debug( - "propose_skill: org HEAD moved (actual=%r), re-splicing (attempt %d)", - conflict.actual, - attempts, - ) - continue - - if result.get("proposal_pending"): - return { - "ok": True, - "proposal_pending": True, - "proposal_id": result.get("proposal_id"), - "ref": result.get("ref"), - "commit": commit_hash, - "org_id": org_id, - } - return { - "ok": True, - "merged": True, - "head": result.get("hash", commit_hash), - "commit": commit_hash, - "org_id": org_id, - } - - -def maybe_pull_org_skills() -> Optional[Dict[str, Any]]: - """Best-effort org pull if all gates pass. Never raises; None when inert. - - Gates (all must hold): logged in, org_role claim present (multi-member - org), feature enabled, base URL configured. Personal orgs are inert here - by construction — resolve_org_identity raises SyncInertError without the - claim. - - Marker hygiene: when the token VERIFIABLY lacks the org claim (logged in, - personal org / left the org), the active-org marker is cleared so - previously-mirrored org skills stop resolving. When we simply cannot - resolve identity (offline, logged out), the marker is left alone — - offline grace keeps already-pulled org skills working. - """ - try: - identity = resolve_org_identity() - except SyncInertError: - # Distinguish "verifiably personal/left-org" from "can't tell". - try: - base_identity = resolve_identity() - claims = base_identity.get("claims") or {} - if not claims.get("org_role"): - _clear_active_org_marker() - except Exception: - pass # offline/logged out — keep offline grace - return None - except Exception as e: - logger.debug( - "skills_sync_client: maybe_pull_org_skills inert/failed: %s", e - ) - return None - try: - if not sync_feature_enabled(): - return None - if not resolve_sync_base_url(): - return None - return pull_org_skills(identity=identity) - except Exception as e: - logger.debug( - "skills_sync_client: maybe_pull_org_skills inert/failed: %s", e - ) - return None - - -def _clear_active_org_marker() -> None: - """Remove the active-org marker (org skills stop resolving).""" - try: - from agent.skill_utils import ORG_ACTIVE_MARKER - - marker = _org_dir() / ORG_ACTIVE_MARKER - if marker.exists(): - marker.unlink() - logger.info( - "skills_sync_client: cleared active-org marker " - "(token has no org workflow); org skills no longer resolve" - ) - except Exception as e: - logger.debug("skills_sync_client: marker clear failed: %s", e) +# Org-shared skills live in skills_sync_client_org; imported last because that +# module reads this module's state lazily. +from tools.skills_sync_client_org import ( # noqa: E402,F401 (re-exports) + ORG_DIR_NAME, _ORG_CAS_MAX_ATTEMPTS, _clear_active_org_marker, _org_baseline_path, + _read_org_baseline, _read_org_head, _skill_dir_fingerprint, _write_active_org_marker, + _write_org_baseline, _write_org_provenance, list_locally_modified_org_skills, + list_org_skill_names, maybe_pull_org_skills, org_head_ref, org_skill_is_locally_modified, + propose_skill, pull_org_skills, resolve_org_identity, +) diff --git a/tools/skills_sync_client_org.py b/tools/skills_sync_client_org.py new file mode 100644 index 0000000000..3994a1c6e8 --- /dev/null +++ b/tools/skills_sync_client_org.py @@ -0,0 +1,381 @@ +"""Org-shared skills: org pull + propose (``~/.hermes/skills/_org//``). + +Org skills live under a DISTINCT local namespace (read-only to the runtime; a +local edit is a personal fork until proposed). The canonical set is +``refs/org//HEAD`` -- the SAME object model as personal sync. +PERSONAL-ORG GATE: NAS stamps ``org_role`` ONLY for multi-member orgs; no +claim => pull/propose raise SyncInertError and personal sync is untouched. +``propose_skill`` must stay non-interactive (automation will drive it). +Module state (``_skills_dir``, ``_org_dir``, base URL, device id) stays in +``tools.skills_sync_client`` and is read lazily so tests can monkeypatch it. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import shutil +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Dict, List, Optional + +from tools.skills_sync_client_wire import ( + DEFAULT_MAX_OBJECT_BYTES, ObjectSet, SyncClient, SyncConflict, SyncError, _check_version, + assemble_root_from_skill_trees, build_commit, build_tree, materialize_tree, read_ref_hash, + root_tree_of_commit, skill_trees_of_root, +) + +logger = logging.getLogger("tools.skills_sync_client") + +ORG_DIR_NAME = "_org" + +# Propose re-splices onto a moved org HEAD at most this many times. Small: +# contention means other members are actively proposing; unbounded would spin. +_ORG_CAS_MAX_ATTEMPTS = 5 + + +def _ssc(): + from tools import skills_sync_client + + return skills_sync_client + + +def org_head_ref(org_id: str) -> str: + return f"refs/org/{org_id}/HEAD" + + +def resolve_org_identity() -> Dict[str, Any]: + """``resolve_identity()`` extended with ``org_id`` + ``org_role``. + + Raises SyncInertError when the token carries no ``org_role`` claim (personal + org / issuer predates org support): org sync is unavailable, NOT an error. + """ + from tools.skills_sync_client import SyncInertError, resolve_identity + + identity = resolve_identity() + claims = identity.get("claims") or {} + org_id = claims.get("org_id") + org_role = claims.get("org_role") + if not org_id: + raise SyncInertError("no organisation associated with this account") + if not isinstance(org_role, str) or not org_role: + raise SyncInertError("this account isn't a member of a shared organisation") + identity["org_id"] = str(org_id) + identity["org_role"] = org_role + return identity + + +def _org_client(identity: Optional[Dict[str, Any]], client: Optional[SyncClient]): + """Resolve (identity, client, caps) for an org operation; raises SyncInertError + when the base URL is missing or the server lacks the ``org`` feature.""" + ssc = _ssc() + identity = identity or resolve_org_identity() + if client is None: + base_url = ssc.resolve_sync_base_url() + if not base_url: + raise ssc.SyncInertError("no sync base URL configured") + client = SyncClient(base_url, identity["api_key"]) + caps = client.capabilities() + _check_version(caps) + if "org" not in (caps.get("features") or []): + raise ssc.SyncInertError("this server does not support org-shared skills") + return identity, client, caps + + +def _read_org_head(client: SyncClient, org_id: str) -> Optional[str]: + """Current org HEAD, or None. MUST read through the ORG endpoint.""" + return read_ref_hash(client, org_head_ref(org_id), org_scope=True) + + +# Local mirror sidecars + +def _mirror_root(org_id: str) -> Path: + return _ssc()._org_dir() / org_id + + +def _write_sidecar(what: str, path_fn: Callable[[], Path], text: str) -> None: + """Best-effort sidecar write (path resolution included); never raises.""" + try: + path = path_fn() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + except Exception as e: + logger.debug("skills_sync_client: %s write failed: %s", what, e) + + +def _skill_dir_fingerprint(path: Path) -> str: + """Stable content hash of a materialized skill dir (sorted relative path + + bytes, independent of filesystem order and mtimes). "" on read failure.""" + h = hashlib.sha256() + try: + for f in sorted(p for p in path.rglob("*") if p.is_file()): + h.update(str(f.relative_to(path)).replace("\\", "/").encode("utf-8")) + h.update(b"\0") + h.update(f.read_bytes()) + h.update(b"\0") + except OSError as e: + logger.debug("skills_sync_client: fingerprint failed for %s: %s", path, e) + return "" + return h.hexdigest() + + +def _sidecar_path(org_id: Optional[str], const: str) -> Path: + """``/>`` (org-level when org_id is None).""" + import agent.skill_utils as sku + + return (_mirror_root(org_id) if org_id else _ssc()._org_dir()) / getattr(sku, const) + + +def _org_baseline_path(org_id: str) -> Path: + """Sidecar recording the upstream fingerprint of each mirrored skill.""" + return _sidecar_path(org_id, "ORG_BASELINE_FILE") + + +def _read_org_baseline(org_id: str) -> Dict[str, Any]: + try: + return json.loads(_org_baseline_path(org_id).read_text(encoding="utf-8")) + except Exception: + return {} + + +def _write_org_baseline(org_id: str, baseline: Dict[str, Any]) -> None: + _write_sidecar( + "baseline", lambda: _org_baseline_path(org_id), json.dumps(baseline, indent=2, sort_keys=True) + ) + + +def _write_org_provenance(org_id: str, data: Dict[str, Any]) -> None: + _write_sidecar( + "org provenance", lambda: _sidecar_path(org_id, "ORG_PROVENANCE_FILE"), json.dumps(data, indent=2) + ) + + +def _write_active_org_marker(org_id: str) -> None: + """Record which org's mirror may resolve (agent/skill_utils.read_active_org_id).""" + _write_sidecar("active-org marker", lambda: _sidecar_path(None, "ORG_ACTIVE_MARKER"), org_id) + + +def _clear_active_org_marker() -> None: + """Remove the active-org marker so org skills stop resolving.""" + try: + marker = _sidecar_path(None, "ORG_ACTIVE_MARKER") + if marker.exists(): + marker.unlink() + logger.info( + "skills_sync_client: cleared active-org marker " + "(token has no org workflow); org skills no longer resolve" + ) + except Exception as e: + logger.debug("skills_sync_client: marker clear failed: %s", e) + + +def org_skill_is_locally_modified(skill_rel_path: str, org_id: str) -> bool: + """True when the local copy of an org skill differs from what upstream sent. + No recorded baseline (pre-existing mirror) => unmodified; the next pull + records one.""" + dest = _mirror_root(org_id) / PurePosixPath(skill_rel_path) + if not dest.is_dir(): + return False + entry = _read_org_baseline(org_id).get(skill_rel_path) or {} + recorded = entry.get("fingerprint") if isinstance(entry, dict) else entry + return bool(recorded) and _skill_dir_fingerprint(dest) != recorded + + +def list_locally_modified_org_skills(org_id: Optional[str] = None) -> List[str]: + """Org skills with local edits that upstream has not seen.""" + try: + from agent.skill_utils import read_active_org_id + + org_id = org_id or read_active_org_id(_ssc()._skills_dir()) + if not org_id: + return [] + return sorted(rel for rel in _read_org_baseline(org_id) if org_skill_is_locally_modified(rel, org_id)) + except Exception as e: + logger.debug("skills_sync_client: modified-scan failed: %s", e) + return [] + + +def list_org_skill_names() -> List[str]: + """Skill names present in the local org mirror (empty when none pulled).""" + names: List[str] = [] + try: + from agent.skill_utils import read_active_org_id + + org_id = read_active_org_id(_ssc()._skills_dir()) + root = _mirror_root(org_id) if org_id else None + if root and root.is_dir(): + for skill_md in root.rglob("SKILL.md"): + rel = skill_md.parent.relative_to(root) + if rel.parts: + names.append(str(rel).replace("\\", "/")) + except Exception as e: + logger.debug("skills_sync_client: org skill listing failed: %s", e) + return sorted(names) + + +# Pull / propose + +def pull_org_skills( + client: Optional[SyncClient] = None, *, identity: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """Pull the org canonical set into the local mirror (fast-forward only; no + client merge on the org path). A mirrored skill with LOCAL edits is never + clobbered: it is skipped, and reported in ``conflicted`` when upstream also + moved; the member's change of record is ``propose_skill``. + Returns ``{ok, org_id, head, updated, conflicted}``.""" + identity = identity or resolve_org_identity() + if "org_id" not in identity: + raise _ssc().SyncInertError("no organisation context available") + identity, client, _caps = _org_client(identity, client) + org_id = identity["org_id"] + + head = _read_org_head(client, org_id) + # Token-gated marker: written HERE because this runs only after the token's + # org_id + org_role were verified. A stale mirror from a previous org stops + # resolving the moment a pull runs under a different org. + _write_active_org_marker(org_id) + if not head: + return {"ok": True, "org_id": org_id, "head": None, "updated": []} + + head_commit = client.get_commit_json(head, org_scope=True) + skill_trees = skill_trees_of_root(client, head_commit["tree"], org_scope=True) + + dest_root = _mirror_root(org_id) + updated: List[str] = [] + conflicted: List[str] = [] + baseline = _read_org_baseline(org_id) + for rel_path, tree_hash in sorted(skill_trees.items()): + dest = dest_root / PurePosixPath(rel_path) + try: + if dest.exists(): + if org_skill_is_locally_modified(rel_path, org_id): + if (baseline.get(rel_path) or {}).get("tree") != tree_hash: + conflicted.append(rel_path) + continue + shutil.rmtree(dest) + dest.mkdir(parents=True, exist_ok=True) + materialize_tree(client, tree_hash, dest, org_scope=True) + baseline[rel_path] = {"fingerprint": _skill_dir_fingerprint(dest), "tree": tree_hash} + updated.append(rel_path) + except Exception as e: + logger.warning("skills_sync_client: org skill materialize failed for %s: %s", rel_path, e) + # Provenance for the skill_view header: the HEAD author is token-verified + # by the plane at push time, so it is trustworthy to display. + author = head_commit.get("author") or {} + _write_org_provenance(org_id, { + "org_id": org_id, "head": head, "author_user_id": author.get("owner", ""), + "author_device": author.get("device", ""), "ts": head_commit.get("ts", ""), "skills": updated, + }) + _write_org_baseline(org_id, baseline) + if conflicted: + logger.warning( + "skills_sync_client: %d org skill(s) have local edits AND upstream " + "changes; left untouched: %s", len(conflicted), ", ".join(conflicted), + ) + return {"ok": True, "org_id": org_id, "head": head, "updated": updated, "conflicted": conflicted} + + +def propose_skill( + skill_name: str, + client: Optional[SyncClient] = None, + *, + identity: Optional[Dict[str, Any]] = None, + message: Optional[str] = None, +) -> Dict[str, Any]: + """Propose a local (personal) skill's content to the org canonical set. + + Snapshots the skill dir as an org-scoped commit splicing that ONE skill + subtree into the current org HEAD (proposals are per-skill deltas, never a + wholesale replace), uploads with ``?scope=org``, then CAS-es the org HEAD: + ADMIN/OWNER -> server merges -> ``{ok, merged: True}``; MEMBER -> 202 + proposal -> ``{ok, proposal_pending: True, proposal_id, ref}``, never + presented as live. + + If HEAD moves between read and CAS, the skill is re-spliced onto the NEW + head (not replayed from the old root, which would drop the other member's + skill) up to ``_ORG_CAS_MAX_ATTEMPTS`` times. + """ + ssc = _ssc() + identity, client, caps = _org_client(identity, client) + org_id = identity["org_id"] + max_bytes = int(caps.get("max_object_bytes") or DEFAULT_MAX_OBJECT_BYTES) + + rel = ssc._skill_rel_path(skill_name) + if rel is None: + raise SyncError(f"skill '{skill_name}' not found under the skills dir") + skill_dir = ssc._skills_dir() / rel + if not (skill_dir / "SKILL.md").exists(): + raise SyncError(f"skill '{skill_name}' has no SKILL.md") + + objects = ObjectSet() + skill_tree = build_tree(skill_dir, objects, max_object_bytes=max_bytes) + + for attempt in range(1, _ORG_CAS_MAX_ATTEMPTS + 1): + base_head = _read_org_head(client, org_id) + skill_map = ( + skill_trees_of_root(client, root_tree_of_commit(client, base_head, org_scope=True), org_scope=True) + if base_head + else {} + ) + skill_map[str(rel)] = skill_tree + root_hash = assemble_root_from_skill_trees(skill_map, objects) + commit_hash = build_commit( + root_hash, [base_head] if base_head else [], owner=identity["owner"], + device=ssc.stable_device_id(), message=message or f"propose {skill_name}", objects=objects, + ) + client.put_objects(objects.objects, org_scope=True) + try: + result = client.cas_ref(org_head_ref(org_id), base_head, commit_hash) + break + except SyncConflict as conflict: + if attempt >= _ORG_CAS_MAX_ATTEMPTS: + raise SyncError( + "the organisation's skills changed while this was being " + f"proposed, and {attempt} attempts to catch up all lost " + "the race — run the command again", + status=409, + ) from conflict + logger.debug( + "propose_skill: org HEAD moved (actual=%r), re-splicing (attempt %d)", conflict.actual, attempt + ) + + if result.get("proposal_pending"): + return { + "ok": True, "proposal_pending": True, "proposal_id": result.get("proposal_id"), + "ref": result.get("ref"), "commit": commit_hash, "org_id": org_id, + } + return { + "ok": True, "merged": True, "head": result.get("hash", commit_hash), + "commit": commit_hash, "org_id": org_id, + } + + +def maybe_pull_org_skills() -> Optional[Dict[str, Any]]: + """Best-effort org pull if all gates hold (logged in, org_role claim, + feature enabled, base URL). Never raises; None when inert. + + Marker hygiene: when the token VERIFIABLY lacks the org claim (personal org + / left the org) the active-org marker is cleared so mirrored org skills stop + resolving. When identity cannot be resolved at all (offline, logged out) the + marker is left alone -- offline grace keeps pulled org skills working. + """ + ssc = _ssc() + try: + identity = resolve_org_identity() + except ssc.SyncInertError: + try: + if not (ssc.resolve_identity().get("claims") or {}).get("org_role"): + _clear_active_org_marker() + except Exception: + pass + return None + except Exception as e: + logger.debug("skills_sync_client: maybe_pull_org_skills inert/failed: %s", e) + return None + try: + if not ssc.sync_feature_enabled() or not ssc.resolve_sync_base_url(): + return None + return pull_org_skills(identity=identity) + except Exception as e: + logger.debug("skills_sync_client: maybe_pull_org_skills inert/failed: %s", e) + return None diff --git a/tools/skills_sync_client_wire.py b/tools/skills_sync_client_wire.py new file mode 100644 index 0000000000..ea1bf5bf19 --- /dev/null +++ b/tools/skills_sync_client_wire.py @@ -0,0 +1,439 @@ +"""Skill Sync wire model: content-addressed objects, the HTTP client, tree walks. + +Everything here is independent of local skill state (no ``~/.hermes`` reads); +``tools/skills_sync_client.py`` orchestrates push/pull on top of it and +re-exports these names. + +Wire contract (version 1). ``hsp_version`` / ``X-HSP-Object-Type`` are deployed +protocol identifiers and are NOT renamed with the product name "Skill Sync". +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import stat as _stat +from contextlib import suppress +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger("tools.skills_sync_client") + +WIRE_VERSION = "1" +DEFAULT_MAX_OBJECT_BYTES = 26214400 # 25 MiB, mirrors capabilities default + +KIND_BLOB = "blob" +KIND_TREE = "tree" +KIND_COMMIT = "commit" + +MODE_FILE = "file" +MODE_EXEC = "exec" +MODE_DIR = "dir" + +ARTIFACT_TYPE_SKILL = "skill" +_EXEC_BITS = _stat.S_IXUSR | _stat.S_IXGRP | _stat.S_IXOTH + +# `sync-manifest`: per-skill opt-in is CONTENT in the object model, not a +# device-local flag -- a root-level blob in the tree at refs/user//HEAD +# recording {name, enabled}. The plane manifest is authoritative; the local +# `.usage.json` `sync` flag is only the editable intent (reconciled FROM it on +# pull, TO it on push). Shape MUST match gateway-gateway src/sync/manifest.ts. +SYNC_MANIFEST_ENTRY_NAME = "sync-manifest" +SYNC_MANIFEST_TYPE = "sync-manifest" +SYNC_MANIFEST_VERSION = 1 + + +# Content addressing. The wire uses the FULL 64-hex sha256 -- a different +# namespace from the truncated 16-hex local `content_hash` (skills_guard.py). + +def wire_address(data: bytes) -> str: + """Return ``sha256:<64-hex>`` -- the wire address of ``data``.""" + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def canonical_json_bytes(obj: Dict[str, Any]) -> bytes: + """Canonical JSON for tree/commit hashing: UTF-8, sorted keys, no whitespace, + no trailing newline. Arrays must already be in contract order (tree entries + by name, commit parents by significance). Client and server MUST produce + byte-identical output or a push fails ``422 hash_mismatch``.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def build_sync_manifest_bytes(skills: Dict[str, bool]) -> bytes: + """Serialize ``{name: enabled}`` into canonical ``sync-manifest`` bytes + (entries sorted by name for a stable content address).""" + return canonical_json_bytes({ + "type": SYNC_MANIFEST_TYPE, "version": SYNC_MANIFEST_VERSION, + "skills": [{"name": name, "enabled": bool(enabled)} for name, enabled in sorted(skills.items())], + }) + + +def parse_sync_manifest(data: bytes) -> Optional[Dict[str, bool]]: + """Parse ``sync-manifest`` bytes into ``{name: enabled}``, or None if malformed. + + Strict (mirrors gateway-gateway ``parseSyncManifest``): unknown type, version + != 1, non-list skills, or a malformed entry all reject -- a malformed + manifest must not be mistaken for "no skills opted in". + """ + try: + value = json.loads(data.decode("utf-8")) + except Exception: + return None + if ( + not isinstance(value, dict) + or value.get("type") != SYNC_MANIFEST_TYPE + or value.get("version") != SYNC_MANIFEST_VERSION + or not isinstance(value.get("skills"), list) + ): + return None + out: Dict[str, bool] = {} + for raw in value["skills"]: + if not isinstance(raw, dict): + return None + name, enabled = raw.get("name"), raw.get("enabled") + if not isinstance(name, str) or not name or not isinstance(enabled, bool): + return None + out[name] = enabled + return out + + +# Object building + +class ObjectSet: + """Objects to push, ``hash -> (kind, bytes)``, deduped by content address.""" + + def __init__(self) -> None: + self.objects: Dict[str, Tuple[str, bytes]] = {} + + def add(self, kind: str, data: bytes) -> str: + addr = wire_address(data) + self.objects.setdefault(addr, (kind, data)) + return addr + + def __len__(self) -> int: + return len(self.objects) + + +def _entry(name: str, kind: str, hash_: str, mode: str) -> Dict[str, str]: + return {"name": name, "kind": kind, "hash": hash_, "mode": mode} + + +def _add_tree(entries: List[Dict[str, str]], objects: ObjectSet) -> str: + """Canonicalize *entries* (sorted by name, byte order) into a tree object.""" + entries.sort(key=lambda e: e["name"]) + return objects.add(KIND_TREE, canonical_json_bytes({"type": KIND_TREE, "entries": entries})) + + +def _file_mode(path: Path) -> str: + """``exec`` if +x else ``file``. No symlink / other modes are emitted.""" + with suppress(OSError): + if path.stat().st_mode & _EXEC_BITS: + return MODE_EXEC + return MODE_FILE + + +def build_tree(dir_path: Path, objects: ObjectSet, *, max_object_bytes: int) -> str: + """Recursively build objects for *dir_path*; return the tree address. Files + -> blobs, subdirs -> nested trees; symlinks/special files are skipped + (contract: no symlinks). A blob over *max_object_bytes* raises ValueError + so the caller can surface / skip the artifact (server -> 413).""" + entries: List[Dict[str, str]] = [] + for child in sorted(dir_path.iterdir(), key=lambda p: p.name): + if child.is_symlink(): + logger.debug("skills_sync_client: skipping symlink %s", child) + elif child.is_dir(): + sub_hash = build_tree(child, objects, max_object_bytes=max_object_bytes) + entries.append(_entry(child.name, KIND_TREE, sub_hash, MODE_DIR)) + elif child.is_file(): + data = child.read_bytes() + if len(data) > max_object_bytes: + raise ValueError( + f"file {child} is {len(data)} bytes > max_object_bytes {max_object_bytes}" + ) + entries.append(_entry(child.name, KIND_BLOB, objects.add(KIND_BLOB, data), _file_mode(child))) + return _add_tree(entries, objects) + + +def build_commit( + tree_hash: str, parents: List[str], *, owner: str, device: str, message: str, + objects: ObjectSet, ts: Optional[str] = None, +) -> str: + """Build a commit object and return its address. + + ``parents``: 0 for the first commit, 1 for an edit, 2 for a merge (order + significant: parents[0] = base fast-forwarded from, parents[1] = other head). + """ + return objects.add(KIND_COMMIT, canonical_json_bytes({ + "type": KIND_COMMIT, "tree": tree_hash, "parents": list(parents), + "author": {"owner": owner, "device": device}, + "ts": ts or datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "message": message, "artifact_type": ARTIFACT_TYPE_SKILL, + })) + + +def build_root_tree(node: Dict[str, Any], objects: ObjectSet, *, manifest_hash: Optional[str] = None) -> str: + """Canonicalize a nested ``{name: {"__tree__": hash} | subdict}`` root into trees. + + ``manifest_hash`` (top level only) adds the root ``sync-manifest`` BLOB entry. + It cannot collide with a skill dir: skill entries are trees, this is a blob. + """ + entries: List[Dict[str, str]] = [] + for name, child in node.items(): + if isinstance(child, dict) and "__tree__" in child and len(child) == 1: + entries.append(_entry(name, KIND_TREE, child["__tree__"], MODE_DIR)) + else: + entries.append(_entry(name, KIND_TREE, build_root_tree(child, objects), MODE_DIR)) + if manifest_hash is not None: + entries.append(_entry(SYNC_MANIFEST_ENTRY_NAME, KIND_BLOB, manifest_hash, MODE_FILE)) + return _add_tree(entries, objects) + + +def nest_skill_tree(root: Dict[str, Any], rel_parts: Tuple[str, ...], tree_hash: str) -> None: + """Insert a skill tree leaf into the nested root structure by path parts.""" + node = root + for part in rel_parts[:-1]: + node = node.setdefault(part, {}) + node[rel_parts[-1]] = {"__tree__": tree_hash} + + +def assemble_root_from_skill_trees(skill_trees: Dict[str, str], objects: ObjectSet) -> str: + """Build a profile-root tree from ``{posix_rel_path: tree_hash}``. + + The skill trees are assumed already durable (they came from either side of + a merge / the org HEAD); only the new intermediate/root trees are added. + """ + root: Dict[str, Any] = {} + for path, tree_hash in skill_trees.items(): + nest_skill_tree(root, PurePosixPath(path).parts, tree_hash) + return build_root_tree(root, objects) + + +# HTTP client (routes under /v1/sync/) + +class SyncError(RuntimeError): + """A non-recoverable wire error (4xx the client can't retry).""" + + def __init__(self, message: str, *, status: Optional[int] = None): + super().__init__(message) + self.status = status + + +class SyncConflict(RuntimeError): + """CAS lost (409). NOT a rejection -- pushed objects are already durable. + + ``actual`` is the current head to merge against, or None when the ref does + not exist server-side (the server sends ""). None means "retry as a create"; + it must never be fetched as an object -- normalized here, not per call site. + """ + + def __init__(self, actual: Optional[str]): + self.actual: Optional[str] = actual or None + super().__init__( + f"CAS conflict; actual head {self.actual}" + if self.actual + else "CAS conflict; the ref does not exist yet" + ) + + +def _check_version(caps: Dict[str, Any]) -> None: + """Reject an incompatible server major version.""" + ver = str(caps.get("hsp_version") or "") # wire field name + if ver.split(".", 1)[0] != WIRE_VERSION: + raise SyncError( + f"this server speaks sync version {ver!r}, but this Hermes speaks " + f"{WIRE_VERSION} — update Hermes to sync with it" + ) + + +class SyncClient: + """Sync client bound to a base URL + Nous bearer. + + Org refs/objects live behind SEPARATE ``org/`` routes, not a prefix filter: + the personal routes are hard-scoped to the token's owner and would silently + answer an org query with personal data. Callers reading org content MUST + pass ``org_scope=True`` on every hop. + """ + + def __init__(self, base_url: str, api_key: str, *, timeout: float = 30.0): + self.base = base_url.rstrip("/") + self.api_key = api_key + self.timeout = timeout + import requests # core dependency + + self._session = requests.Session() + self._session.headers["Authorization"] = f"Bearer {api_key}" + + def _url(self, path: str) -> str: + return f"{self.base}/v1/sync/{path.lstrip('/')}" + + @staticmethod + def _check(r, op: str, ok=(200,), errors: Optional[Dict[int, str]] = None) -> None: + """Raise SyncError unless the status is in *ok*; *errors* maps specific + statuses to their message, anything else gets ``" failed: "``.""" + if r.status_code in ok: + return + msg = (errors or {}).get(r.status_code) + raise SyncError(msg or f"{op} failed: {r.status_code}", status=r.status_code) + + def capabilities(self) -> Dict[str, Any]: + """GET capabilities (no auth required).""" + r = self._session.get(self._url("capabilities"), timeout=self.timeout) + self._check(r, "capabilities") + return r.json() + + def get_refs(self, prefix: str, *, org_scope: bool = False) -> List[Dict[str, str]]: + """GET refs?prefix=... (or org/refs, filtered client-side by *prefix*).""" + path = "org/refs" if org_scope else "refs" + params = None if org_scope else {"prefix": prefix} + r = self._session.get(self._url(path), params=params, timeout=self.timeout) + self._check(r, "get_refs") + refs = (r.json() or {}).get("refs", []) + if org_scope: + refs = [r_ for r_ in refs if str(r_.get("name", "")).startswith(prefix)] + return refs + + def get_object(self, obj_hash: str, *, org_scope: bool = False) -> Tuple[str, bytes]: + """GET objects/:hash -> ``(kind, bytes)``. Kind comes from the object-type + header; a blob (octet-stream) is returned as ``blob``.""" + path = f"org/objects/{obj_hash}" if org_scope else f"objects/{obj_hash}" + r = self._session.get(self._url(path), timeout=self.timeout) + self._check( + r, "get_object", + errors={404: f"object {obj_hash} not found", 403: f"object {obj_hash} not readable"}, + ) + return r.headers.get("X-HSP-Object-Type") or KIND_BLOB, r.content + + def _get_json_of_kind(self, obj_hash: str, expected: str, org_scope: bool) -> Dict[str, Any]: + kind, data = self.get_object(obj_hash, org_scope=org_scope) + if kind != expected: + raise SyncError(f"{obj_hash} is {kind}, expected {expected}") + return json.loads(data.decode("utf-8")) + + def get_commit_json(self, commit_hash: str, *, org_scope: bool = False) -> Dict[str, Any]: + return self._get_json_of_kind(commit_hash, KIND_COMMIT, org_scope) + + def get_tree_json(self, tree_hash: str, *, org_scope: bool = False) -> Dict[str, Any]: + return self._get_json_of_kind(tree_hash, KIND_TREE, org_scope) + + def put_objects(self, objects: Dict[str, Tuple[str, bytes]], *, org_scope: bool = False) -> Dict[str, Any]: + """POST objects -- batch upload as multipart/form-data: field name = the + claimed ``sha256:``, ``filename`` = object type, body = raw bytes + (the contract requires raw bytes, not base64-in-JSON). The server + recomputes every hash and rejects the whole batch with 422 on mismatch; + known hashes are idempotent no-ops. ``org_scope`` adds ``?scope=org`` so + objects land org-readable (required before an org CAS/propose).""" + files = [(h, (kind, data, "application/octet-stream")) for h, (kind, data) in objects.items()] + r = self._session.post( + self._url("objects"), files=files, params={"scope": "org"} if org_scope else None, timeout=self.timeout + ) + self._check( + r, "put_objects", ok=(200, 201), + errors={413: "object too large (413)", 422: f"hash_mismatch (422): {r.text}"}, + ) + return r.json() if r.content else {} + + def cas_ref(self, name: str, from_hash: Optional[str], to_hash: str) -> Dict[str, Any]: + """POST refs/:name -- atomic compare-and-swap. Raises SyncConflict on 409. + + A non-admin member's CAS on an org HEAD is converted server-side to a + proposal (202) and surfaced as ``{"proposal_pending": True, ...}``: a + SUCCESS-shaped outcome that must never be presented as live/merged. + """ + r = self._session.post( + self._url(f"refs/{name}"), json={"from": from_hash, "to": to_hash}, timeout=self.timeout + ) + if r.status_code == 202: + return {"proposal_pending": True, **(r.json() if r.content else {})} + if r.status_code == 409: + # "" actual = the ref does not exist server-side (SyncConflict -> None). + raise SyncConflict((r.json() or {}).get("actual", "")) + self._check(r, "cas_ref", errors={403: "forbidden (403) -- owner/permission"}) + return r.json() if r.content else {} + + +# Reading remote trees + +def read_ref_hash(client: SyncClient, ref: str, *, org_scope: bool = False) -> Optional[str]: + """Hash of *ref* (queried with itself as prefix), or None if absent.""" + refs = client.get_refs(ref, org_scope=org_scope) + return next((r.get("hash") for r in refs if r.get("name") == ref), None) + + +def root_tree_of_commit(client: SyncClient, commit_hash: str, *, org_scope: bool = False) -> str: + return client.get_commit_json(commit_hash, org_scope=org_scope)["tree"] + + +def skill_trees_of_root(client: SyncClient, root_tree_hash: str, *, org_scope: bool = False) -> Dict[str, str]: + """Flatten a profile-root tree into ``{posix_rel_path: skill_tree_hash}``. A + skill tree is any subtree containing a ``SKILL.md`` blob, keyed by its path + so category nesting is preserved.""" + result: Dict[str, str] = {} + + def _walk(tree_hash: str, prefix: str) -> None: + entries = client.get_tree_json(tree_hash, org_scope=org_scope).get("entries", []) + if prefix and any(e.get("name") == "SKILL.md" and e.get("kind") == KIND_BLOB for e in entries): + result[prefix] = tree_hash + return + for e in entries: + if e.get("kind") == KIND_TREE: + _walk(e["hash"], f"{prefix}/{e['name']}" if prefix else e["name"]) + + _walk(root_tree_hash, "") + return result + + +def read_manifest_of_root(client: SyncClient, root_tree_hash: str) -> Optional[Dict[str, bool]]: + """``{name: enabled}`` from the root-level ``sync-manifest`` blob, or None if + absent/malformed. This is how a device learns another device's opt-ins.""" + try: + tree = client.get_tree_json(root_tree_hash) + except Exception as e: + logger.debug("skills_sync_client: manifest root read failed: %s", e) + return None + for e in tree.get("entries", []): + if e.get("name") == SYNC_MANIFEST_ENTRY_NAME and e.get("kind") == KIND_BLOB: + try: + _kind, data = client.get_object(e["hash"]) + except Exception as ex: + logger.debug("skills_sync_client: manifest blob fetch failed: %s", ex) + return None + return parse_sync_manifest(data) + return None + + +def materialize_tree(client: SyncClient, tree_hash: str, dest: Path, *, org_scope: bool = False) -> None: + """Write the tree at *tree_hash* into *dest* (created if needed): blobs -> + files (+x restored for ``exec``), trees -> subdirectories. Does NOT delete + files absent from the tree (caller decides). Refuses path traversal.""" + dest.mkdir(parents=True, exist_ok=True) + for entry in client.get_tree_json(tree_hash, org_scope=org_scope).get("entries", []): + name = entry.get("name", "") + if not name or "/" in name or name in (".", ".."): + logger.warning("skills_sync_client: skipping unsafe tree entry %r", name) + continue + target = dest / name + kind = entry.get("kind") + if kind == KIND_TREE: + materialize_tree(client, entry["hash"], target, org_scope=org_scope) + elif kind == KIND_BLOB: + _, data = client.get_object(entry["hash"], org_scope=org_scope) + target.write_bytes(data) + if entry.get("mode") == MODE_EXEC: + with suppress(OSError): + target.chmod(target.stat().st_mode | _EXEC_BITS) + + +def merge_skill(base: Optional[str], ours: Optional[str], theirs: Optional[str]) -> str: + """Three-way decision for one skill's tree hash: ``ours`` / ``theirs`` / + ``either`` / ``overlap`` / ``none``. A side "modified" the skill when its + hash differs from the common base (same semantics as skills_sync.py's + origin/user/incoming block).""" + if ours == theirs: + return "either" if ours is not None else "none" + if theirs == base: # only we moved + return "ours" + if ours == base: # only they moved + return "theirs" + return "overlap" diff --git a/tools/skills_sync_optional.py b/tools/skills_sync_optional.py new file mode 100644 index 0000000000..91d0ef0ad7 --- /dev/null +++ b/tools/skills_sync_optional.py @@ -0,0 +1,262 @@ +"""Official optional-skill provenance: hub-lock backfill and restore. + +Extracted from ``tools.skills_sync``. Profile-scoped paths and patchable +helpers are resolved through ``_ss()`` at call time so tests and multi-profile +runtimes that patch ``tools.skills_sync`` globals keep working. +""" + +import json +import logging +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Dict, Iterator, List, Optional, Set, Tuple + +from agent.skill_utils import is_excluded_skill_path +from utils import atomic_write_text + +logger = logging.getLogger("tools.skills_sync") + + +def _ss(): + from tools import skills_sync + + return skills_sync + + +def _content_hash(directory: Path) -> str: + """Same hash style the skills hub lock uses; hashing is provenance metadata + only, so fall back to the local MD5 if guard deps are unavailable.""" + try: + from tools.skills_guard import content_hash + + return content_hash(directory) + except Exception: + return _ss()._dir_hash(directory) + + +def _safe_rel_install_path(path: Path, base: Path) -> str: + """Return a normalized relative POSIX path, rejecting traversal/absolute paths.""" + posix = path.relative_to(base).as_posix() + pure = PurePosixPath(posix) + parts = [part for part in pure.parts if part not in {"", "."}] + if pure.is_absolute() or not parts or ".." in parts: + raise ValueError(f"Unsafe optional skill path: {posix}") + return "/".join(parts) + + +def _skill_file_list(skill_dir: Path) -> List[str]: + """List files inside a skill directory in lock-file format.""" + return [f.relative_to(skill_dir).as_posix() for f in sorted(skill_dir.rglob("*")) if f.is_file()] + + +def _hub_lock_path() -> Path: + return _ss()._skills_dir() / ".hub" / "lock.json" + + +def _load_hub_lock() -> Optional[dict]: + """Parse the skills-hub lock; None when missing or unreadable.""" + try: + return json.loads(_hub_lock_path().read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return None + + +def _hub_lock_entries(data: Optional[dict]) -> List[dict]: + return [e for e in ((data or {}).get("installed") or {}).values() if isinstance(e, dict)] + + +def _read_hub_install_paths() -> Set[str]: + """Install paths recorded in the hub lock, as POSIX strings. Hub-installed skills + are owned by the hub, never by bundled sync: rename recovery must not move them even + when content matches a bundled origin hash, or the lock's ``install_path`` dangles.""" + return {str(e["install_path"]).strip("/") for e in _hub_lock_entries(_load_hub_lock()) if e.get("install_path")} + + +def _write_hub_lock(lock_path: Path, data: dict) -> None: + """Atomic write so a crash mid-write can't wipe all provenance (the + JSONDecodeError fallback in the reader resets ``installed`` to empty).""" + atomic_write_text(lock_path, json.dumps(data, indent=2, ensure_ascii=False) + "\n", tmp_prefix=".lock_") + + +def _iter_optional_skills(optional_dir: Path, *, root_relative: bool) -> Iterator[Tuple[Path, Path, str]]: + """Yield ``(skill_md, src, install_path)`` for every safe official optional skill.""" + for skill_md in sorted(optional_dir.rglob("SKILL.md")): + if root_relative and is_excluded_skill_path(skill_md.relative_to(optional_dir), root=optional_dir): + continue + if not root_relative and is_excluded_skill_path(skill_md): + continue + try: + yield skill_md, skill_md.parent, _safe_rel_install_path(skill_md.parent, optional_dir) + except ValueError as e: + logger.debug("Skipping optional skill with unsafe path %s: %s", skill_md.parent, e) + + +def _optional_skill_index() -> Dict[str, Tuple[str, str, Path]]: + """Official optional skills keyed by BOTH folder name and frontmatter name, so callers + may pass either the hub-lock slug or the user-facing name. Values are + ``(folder_name, install_path, source_dir)``.""" + ss = _ss() + optional_dir = ss._get_optional_dir() + index: Dict[str, Tuple[str, str, Path]] = {} + if not optional_dir.exists(): + return index + for skill_md, src, install_path in _iter_optional_skills(optional_dir, root_relative=True): + value = (src.name, install_path, src) + index[src.name] = value + index[ss._read_skill_name(skill_md, src.name)] = value + return index + + +def _move_to_restore_backup(path: Path, backup_root: Path) -> str: + """Move an existing skill directory into a restore backup, preserving rel path.""" + rel = path.relative_to(_ss()._skills_dir()) + target = backup_root / rel + suffix = 0 + while target.exists(): + suffix += 1 + target = (backup_root / rel).with_name(f"{rel.name}-{suffix}") + _ss()._move_dir(path, target) + return rel.as_posix() + + +def _find_active_copies(folder_name: str, src_frontmatter: str, dest: Path) -> List[Path]: + """Active copies of an official skill (by frontmatter name or folder slug), + even when the curator moved it into another category; excludes ``dest``.""" + ss = _ss() + names = {folder_name, src_frontmatter} + return [ + md.parent + for md in ss._iter_active_skill_mds(sort=True) + if md.parent != dest and (md.parent.name == folder_name or ss._read_skill_name(md, md.parent.name) in names) + ] + + +def restore_official_optional_skill(name: str, *, restore: bool = False) -> dict: + """Restore one or all official optional skills from repo source. ``restore=False`` + only performs exact-match provenance backfill; ``restore=True`` repairs mutated / + reorganized skills by backing up matching active copies and copying the official + source into its canonical path.""" + ss = _ss() + + def _fail(message: str) -> dict: + return {"ok": False, "message": message, "restored": [], "backfilled": [], "backed_up": []} + + index = _optional_skill_index() + if not index: + return _fail("No official optional skills directory found.") + if name in {"all", "*"}: + targets = sorted(set(index.values()), key=lambda item: item[1]) + elif name in index: + targets = [index[name]] + else: + return _fail(f"Official optional skill not found: {name}") + + restored: List[str] = [] + backed_up: List[str] = [] + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + backup_root = ss._skills_dir() / ".restore-backups" / f"official-optional-{timestamp}" + + for folder_name, install_path, src in targets if restore else []: + dest = ss._skills_dir() / Path(*install_path.split("/")) + canonical_ok = dest.exists() and ss._dir_hash(dest) == ss._dir_hash(src) + src_frontmatter = ss._read_skill_name(src / "SKILL.md", folder_name) + for match in _find_active_copies(folder_name, src_frontmatter, dest): + if match.exists(): + backed_up.append(_move_to_restore_backup(match, backup_root)) + if dest.exists() and not canonical_ok: + backed_up.append(_move_to_restore_backup(dest, backup_root)) + if not dest.exists(): + ss._copy_dir(src, dest) + restored.append(folder_name) + + return { + "ok": True, "message": "Official optional skill repair complete.", "restored": restored, + "backfilled": _backfill_optional_provenance(quiet=True), "backed_up": backed_up, + "backup_dir": str(backup_root) if backed_up else "", + } + + +def _index_installed_skill_dirs_by_name() -> Dict[str, List[Path]]: + """Index installed skills by directory name with one active-tree scan, + skipping anything that resolves outside the skills tree (symlinks/external).""" + ss = _ss() + index: Dict[str, List[Path]] = {} + root = ss._skills_dir().resolve() + for skill_md in ss._iter_active_skill_mds(): + try: + skill_md.parent.resolve().relative_to(root) + except (OSError, ValueError): + continue + index.setdefault(skill_md.parent.name, []).append(skill_md.parent) + return index + + +def _relocated_dest(src_name: str, index: Dict[str, List[Path]]) -> Optional[Tuple[Path, str]]: + """The active tree may hold a skill under a DIFFERENT category path than the + repo (upstream reorganizes; the installed copy keeps its old location). Fall + back to a UNIQUE same-directory-name match — an ambiguous name gives no basis + to pick one. Returns ``(dest, install_path)`` or None.""" + candidates = index.get(src_name, []) + if len(candidates) != 1: + return None + dest = candidates[0] + try: + return dest, _safe_rel_install_path(dest, _ss()._skills_dir()) + except ValueError as e: + logger.debug("Skipping relocated optional skill %s: %s", dest, e) + return None + + +def _backfill_optional_provenance(quiet: bool = False) -> List[str]: + """Mark already-present official optional skills as hub-installed: skills that used + to be bundled (or were hand-copied) and now live under optional-skills/ get official + provenance when byte-identical to the source. Modified/local skills are left alone.""" + ss = _ss() + optional_dir = ss._get_optional_dir() + if not optional_dir.exists(): + return [] + + data = _load_hub_lock() + if data is None: + data = {"version": 1, "installed": {}} + installed = data.setdefault("installed", {}) + existing_paths = {entry.get("install_path") for entry in _hub_lock_entries(data)} + + backfilled: List[str] = [] + installed_dir_index: Optional[Dict[str, List[Path]]] = None + for _skill_md, src, install_path in _iter_optional_skills(optional_dir, root_relative=False): + lock_name = src.name + if lock_name in installed or install_path in existing_paths: + continue + dest = ss._skills_dir() / Path(*install_path.split("/")) + if not dest.is_dir(): + if installed_dir_index is None: + installed_dir_index = _index_installed_skill_dirs_by_name() + found = _relocated_dest(src.name, installed_dir_index) + if found is None: + continue + dest, install_path = found # still requires a byte-identical hash below + if install_path in existing_paths or ss._dir_hash(dest) != ss._dir_hash(src): + continue + + timestamp = datetime.now(timezone.utc).isoformat() + installed[lock_name] = { + "source": "official", + "identifier": f"official/{install_path}", + "trust_level": "builtin", + "scan_verdict": "backfilled", + "content_hash": _content_hash(dest), + "install_path": install_path, + "files": _skill_file_list(dest), + "metadata": {"backfilled_from": "optional-skills"}, + "installed_at": timestamp, + "updated_at": timestamp, + } + existing_paths.add(install_path) + backfilled.append(lock_name) + if not quiet: + print(f" = {lock_name} (official optional provenance backfilled)") + + if backfilled: + _write_hub_lock(_hub_lock_path(), data) + return backfilled