diff --git a/tools/spill_safety.py b/tools/spill_safety.py index 34eeb2ed51..37f12c08af 100644 --- a/tools/spill_safety.py +++ b/tools/spill_safety.py @@ -1,11 +1,10 @@ -"""Symlink-safe creation helpers for spill/cache files under ``~/.hermes``, where a -plain ``open(path, "w")`` would follow a pre-planted symlink onto ``~/.bashrc`` etc. -New files use ``O_CREAT | O_EXCL`` (fails on ANY existing path, even a dangling -link); overwrites ``lstat`` + ``unlink`` first (removes the link, never its target) -then create exclusively, so the pair can't be raced. ``private=True`` (default) = -``0o700`` dirs / ``0o600`` files for spills that may hold pre-redaction secrets; -``private=False`` keeps umask perms for cache dirs bind-mounted into remote backends -(``credential_files._CACHE_DIRS``). Disk failures raise ``OSError`` to the caller.""" +"""Symlink-safe creation helpers for spill/cache files under ``~/.hermes``, where a plain +``open(path, "w")`` would follow a pre-planted symlink onto ``~/.bashrc`` etc. New files use +``O_CREAT | O_EXCL`` (fails on ANY existing path, even a dangling link); overwrites ``lstat`` + +``unlink`` first (removes the link, never its target) then create exclusively, so the pair +can't be raced. ``private=True`` (default) = ``0o700`` dirs / ``0o600`` files for spills that +may hold pre-redaction secrets; ``private=False`` keeps umask perms for cache dirs bind-mounted +into remote backends (``credential_files._CACHE_DIRS``). Disk failures raise ``OSError``.""" from __future__ import annotations diff --git a/tools/todo_tool.py b/tools/todo_tool.py index 560c956461..33b2138038 100644 --- a/tools/todo_tool.py +++ b/tools/todo_tool.py @@ -71,15 +71,9 @@ class TodoStore: cur["parent"] = parent else: cur.pop("parent", None) - # Rebuild preserving original order for existing items. - seen = set() - rebuilt = [] - for item in self._items: - current = existing.get(item["id"], item) - if current["id"] not in seen: - rebuilt.append(current) - seen.add(current["id"]) - self._items = self._normalize_order(rebuilt) + # Rebuild preserving original order for existing items (first occurrence wins). + rebuilt = {item["id"]: existing.get(item["id"], item) for item in self._items} + self._items = self._normalize_order(list(rebuilt.values())) def read(self) -> List[Dict[str, str]]: return [item.copy() for item in self._items] @@ -108,12 +102,9 @@ class TodoStore: if not self._items: return None children: Dict[str, List[Dict[str, str]]] = {} - roots: List[Dict[str, str]] = [] for item in self._items: if item.get("parent"): children.setdefault(item["parent"], []).append(item) - else: - roots.append(item) def render(item: Dict[str, str], depth: int, out: List[str]) -> bool: kid_lines: List[str] = [] @@ -129,8 +120,9 @@ class TodoStore: return keep lines = [TODO_INJECTION_HEADER] - for item in roots: - render(item, 0, lines) + for item in self._items: + if not item.get("parent"): + render(item, 0, lines) return "\n".join(lines) if len(lines) > 1 else None @staticmethod diff --git a/tools/tool_backend_helpers.py b/tools/tool_backend_helpers.py index 24249aa6be..978e009860 100644 --- a/tools/tool_backend_helpers.py +++ b/tools/tool_backend_helpers.py @@ -216,9 +216,9 @@ def selection_exists(section: str) -> bool: # Backends that once shipped in-tree but were removed; a config still pointing at one would -# otherwise fail silently at the FIRST tool call with a generic "no registered provider has -# that name". Consulted by the startup config check and selection_error(). Add removals -# here, never as one-off string checks: "web": {"": "the backend was removed"} +# otherwise fail silently at the FIRST tool call with a generic "no registered provider has that +# name". Used by the startup config check and selection_error(); add removals here, never as +# one-off string checks: "web": {"": "the backend was removed in vX (...)"} REMOVED_BACKENDS: Dict[str, Dict[str, str]] = {} diff --git a/tools/tool_output_limits.py b/tools/tool_output_limits.py index 4b56e2c746..550cc52a5a 100644 --- a/tools/tool_output_limits.py +++ b/tools/tool_output_limits.py @@ -56,13 +56,6 @@ def _reset_tool_output_limits_cache() -> None: _cached_limits = None -def get_max_bytes() -> int: - return get_tool_output_limits()["max_bytes"] - - -def get_max_lines() -> int: - return get_tool_output_limits()["max_lines"] - - -def get_max_line_length() -> int: - return get_tool_output_limits()["max_line_length"] +def get_max_bytes() -> int: return get_tool_output_limits()["max_bytes"] +def get_max_lines() -> int: return get_tool_output_limits()["max_lines"] +def get_max_line_length() -> int: return get_tool_output_limits()["max_line_length"] diff --git a/tools/tool_result_storage.py b/tools/tool_result_storage.py index 3fc19e209c..9921ab5106 100644 --- a/tools/tool_result_storage.py +++ b/tools/tool_result_storage.py @@ -1,10 +1,9 @@ -"""Tool result persistence -- preserves large outputs instead of truncating. -Layers against context overflow: (1) per-tool caps inside each tool; (2) -``maybe_persist_tool_result`` — output over the tool's threshold is persisted and replaced -by a preview + path. Canonical home is ALWAYS host-side ``$HERMES_HOME/cache/spillover/ -{id}.txt`` (works for sessions that never ran a terminal); remote backends get the -translated in-sandbox path (probed for readability) else a copy in the sandbox temp dir. -(3) ``enforce_turn_budget``.""" +"""Tool result persistence -- preserves large outputs instead of truncating. Layers against +context overflow: (1) per-tool caps inside each tool; (2) ``maybe_persist_tool_result`` — +output over the tool's threshold is persisted and replaced by a preview + path; canonical home +is ALWAYS host-side ``$HERMES_HOME/cache/spillover/{id}.txt`` (works for sessions that never +ran a terminal), remote backends get the translated in-sandbox path (probed for readability) +else a copy in the sandbox temp dir; (3) ``enforce_turn_budget``.""" import hashlib import logging diff --git a/tools/tool_search.py b/tools/tool_search.py index 6279e1edcb..511c668d00 100644 --- a/tools/tool_search.py +++ b/tools/tool_search.py @@ -1,14 +1,14 @@ -"""Progressive tool disclosure ("tool search"): MCP/plugin tools and a curated set -of event-triggered core tools are replaced in the model-visible array by three -bridge tools — tool_search / tool_describe / tool_call. Invariants: core tools -(``toolsets._HERMES_CORE_TOOLS``) and session-gated GUI toolsets never defer unless -named in ``defer``; ANY deferrable tool activates the bridge (the listing scales -with budget, not activation); the catalog is stateless — rebuilt from the live -tool-defs every assembly (a session-keyed one drifts and silently drops tools); -bridge calls route through ``model_tools.handle_function_call`` (same guardrails).""" +"""Progressive tool disclosure ("tool search"): MCP/plugin tools and a curated set of +event-triggered core tools are replaced in the model-visible array by three bridge tools — +tool_search / tool_describe / tool_call. Invariants: core tools (``toolsets._HERMES_CORE_TOOLS``) +and session-gated GUI toolsets never defer unless named in ``defer``; ANY deferrable tool +activates the bridge (the listing scales with budget, not activation); the catalog is +stateless — rebuilt from the live tool-defs every assembly (a session-keyed one drifts and +silently drops tools); bridge calls route through ``model_tools.handle_function_call``.""" from __future__ import annotations +import functools import json import logging import math @@ -20,13 +20,12 @@ from tools.registry import tool_error from tools.tool_search_catalog import ( # noqa: F401 — re-exported public/test names BRIDGE_TOOL_NAMES, CHARS_PER_TOKEN, TOOL_CALL_NAME, TOOL_DESCRIBE_NAME, TOOL_SEARCH_NAME, CatalogEntry, _corpus_stats, _entry_search_text, _fn, _listing_group_label, - _registry_entry, _short_desc, _stem, _tokenize, build_catalog, + _registry_entry, _registry_toolset, _short_desc, _stem, _tokenize, build_catalog, build_catalog_listing_with_form, search_catalog) from tools.tool_search_validation import validate_deferred_call_args # noqa: F401 logger = logging.getLogger("tools.tool_search") -_MAX_QUERIES_PER_CALL = 10 # bound the work one bridge call can request -_MAX_DESCRIBE_NAMES_PER_CALL = 10 +_MAX_QUERIES_PER_CALL = _MAX_DESCRIBE_NAMES_PER_CALL = 10 # bound the work one bridge call requests @dataclass(frozen=True) @@ -51,9 +50,8 @@ class ToolSearchConfig: def from_raw(cls, raw: Any) -> "ToolSearchConfig": """Build from a raw dict / legacy bool / None; every field is clamped and unknown values fall back to safe defaults — a config typo must not break the agent.""" - if not isinstance(raw, dict): - return cls(enabled="off" if raw is False else "auto", threshold_pct=5.0, - search_default_limit=5, max_search_limit=25) + if not isinstance(raw, dict): # legacy bool / None + raw = {"enabled": "off" if raw is False else "auto"} max_search_limit = _clamped_int(raw.get("max_search_limit"), 25, 1, 50) defer_raw = raw.get("defer") return cls( @@ -105,12 +103,8 @@ def _config_from_loader(loader_name: str) -> ToolSearchConfig: return ToolSearchConfig.from_raw(None) -def load_config() -> ToolSearchConfig: - return _config_from_loader("load_config") - - -def load_config_readonly() -> ToolSearchConfig: # no copy of the cached full config - return _config_from_loader("load_config_readonly") +load_config = functools.partial(_config_from_loader, "load_config") +load_config_readonly = functools.partial(_config_from_loader, "load_config_readonly") # no copy def _core_tool_names() -> frozenset[str]: @@ -149,11 +143,9 @@ def is_deferrable_tool_name(name: str, defer_tools: Optional[frozenset] = None) return True if name in _core_tool_names(): return False - entry = _registry_entry(name) - try: - return entry.toolset.startswith("mcp-") or entry.toolset not in _DIRECT_SURFACE_TOOLSETS - except Exception: # unregistered, or malformed entry (no str toolset): never deferrable - return False + toolset = _registry_toolset(name) # None (unregistered/malformed) never defers + return toolset is not None and ( + toolset.startswith("mcp-") or toolset not in _DIRECT_SURFACE_TOOLSETS) def _tool_def_names(tool_defs: Iterable[Dict[str, Any]]) -> Iterable[str]: @@ -248,10 +240,9 @@ def _search_description(deferred_count: int, listing: Optional[str], listing_for def bridge_tool_schemas(deferred_count: int, listing: Optional[str] = None, listing_form: str = "") -> List[Dict[str, Any]]: - """Bridge tool schemas injected in place of deferred tools. Kept short — every byte is - paid on every turn. ``listing`` is embedded in the tool_search description; per-tool - ``listing_form``s say "skip search when you see the exact name", "groups" says which - domains exist and that search is mandatory.""" + """Bridge tool schemas injected in place of deferred tools; kept short — every byte is paid + every turn. ``listing`` is embedded in the tool_search description; per-tool forms say + "skip search when you see the exact name", "groups" says search is mandatory.""" return [ _bridge_schema( TOOL_SEARCH_NAME, @@ -353,10 +344,10 @@ def is_bridge_tool(name: str) -> bool: def _shared_tool_record(entry: CatalogEntry) -> Dict[str, Any]: """One record for the shared ``tools`` map (per-query groups carry names only); ``required`` lets the model attempt a trivial call without a ``tool_describe`` round-trip.""" - schema = entry.schema if isinstance(entry.schema, dict) else {} - fn = schema.get("function") - params = fn.get("parameters") if isinstance(fn, dict) else None - required = params.get("required") if isinstance(params, dict) else None + try: + required = entry.schema["function"]["parameters"]["required"] + except (TypeError, KeyError, AttributeError): + required = [] return {"source": entry.source, "source_name": entry.source_name, "description": (entry.description or "")[:400], # cap chatty MCP descriptions "required": [r[:64] for r in (required if isinstance(required, list) else []) @@ -398,9 +389,8 @@ def dispatch_tool_search(args: Dict[str, Any], *, current_tool_defs: List[Dict[s required}}}``. ``limit`` applies PER QUERY; empty groups get ``available_sources`` + ``hint`` so a lexical miss is not mistaken for a missing capability.""" config = config or load_config() - queries, err = _string_list_arg( - args, "queries", dedupe=False, max_items=_MAX_QUERIES_PER_CALL, - retry_hint="Retry with fewer, more targeted queries.") + queries, err = _string_list_arg(args, "queries", dedupe=False, max_items=_MAX_QUERIES_PER_CALL, + retry_hint="Retry with fewer, more targeted queries.") if err: return err raw_limit = args.get("limit") @@ -501,9 +491,9 @@ def resolve_underlying_call(args: Dict[str, Any]) -> Tuple[Optional[str], Dict[s __all__ = [ "TOOL_SEARCH_NAME", "TOOL_DESCRIBE_NAME", "TOOL_CALL_NAME", "BRIDGE_TOOL_NAMES", - "ToolSearchConfig", "CatalogEntry", "AssemblyResult", "load_config", - "is_deferrable_tool_name", "classify_tools", "estimate_tokens_from_schemas", - "should_activate", "build_catalog", "build_catalog_listing_with_form", "listing_token_budget", - "search_catalog", "bridge_tool_schemas", "assemble_tool_defs", "is_bridge_tool", - "dispatch_tool_search", "dispatch_tool_describe", "resolve_underlying_call", - "scoped_deferrable_names", "validate_deferred_call_args"] + "ToolSearchConfig", "CatalogEntry", "AssemblyResult", "load_config", "is_deferrable_tool_name", + "classify_tools", "estimate_tokens_from_schemas", "should_activate", "build_catalog", + "build_catalog_listing_with_form", "listing_token_budget", "search_catalog", + "bridge_tool_schemas", "assemble_tool_defs", "is_bridge_tool", "dispatch_tool_search", + "dispatch_tool_describe", "resolve_underlying_call", "scoped_deferrable_names", + "validate_deferred_call_args"] diff --git a/tools/tool_search_catalog.py b/tools/tool_search_catalog.py index 67d68af9c3..06b305d25e 100644 --- a/tools/tool_search_catalog.py +++ b/tools/tool_search_catalog.py @@ -36,22 +36,18 @@ class CatalogEntry: _TOKEN_RE = re.compile(r"[A-Za-z0-9]+") -# Snowball stemmers carry mutable parsing state and bridge dispatch runs on parallel -# tool-call threads: one stemmer per thread, created lazily. _thread_local = threading.local() -def _stemmer() -> Any: - st = getattr(_thread_local, "stemmer", None) - if st is None: - st = _thread_local.stemmer = snowballstemmer.stemmer("english") - return st - - @functools.lru_cache(maxsize=16384) def _stem(token: str) -> str: - """Stem one token, memoized across stateless catalog rebuilds.""" - return _stemmer().stemWord(token) + """Stem one token, memoized across stateless catalog rebuilds. Snowball stemmers carry + mutable parsing state and bridge dispatch runs on parallel tool-call threads, so the + stemmer is one-per-thread, created lazily.""" + st = getattr(_thread_local, "stemmer", None) + if st is None: + st = _thread_local.stemmer = snowballstemmer.stemmer("english") + return st.stemWord(token) def _tokenize(text: str) -> List[str]: @@ -75,6 +71,12 @@ def _registry_entry(name: str) -> Any: return None +def _registry_toolset(name: str) -> Optional[str]: + """Toolset of a registered tool; None when unregistered or malformed (no str toolset).""" + toolset = getattr(_registry_entry(name), "toolset", None) + return toolset if isinstance(toolset, str) else None + + def _entry_search_text(td: Dict[str, Any], source_label: str = "") -> str: """Search-text blob: split name words + source label + description + top-level parameter names (schema bodies are noise with no recall gain). The ``mcp__`` prefix is dropped — it @@ -92,11 +94,10 @@ def _entry_search_text(td: Dict[str, Any], source_label: str = "") -> str: def _classify_source(name: str) -> Tuple[str, str]: """Return (source_kind, source_name) for a registered tool name.""" - entry = _registry_entry(name) - try: - return ("mcp" if entry.toolset.startswith("mcp-") else "plugin", entry.toolset) - except Exception: # unregistered, or malformed entry (no str toolset) + toolset = _registry_toolset(name) + if toolset is None: return ("other", "") + return ("mcp" if toolset.startswith("mcp-") else "plugin", toolset) def build_catalog(tool_defs: List[Dict[str, Any]]) -> List[CatalogEntry]: @@ -143,9 +144,7 @@ def _corpus_stats(catalog: List[CatalogEntry]) -> _CorpusStats: """Compute the BM25 statistics shared by every query over a catalog.""" doc_lengths = [len(entry._tokens) for entry in catalog] avg_dl = sum(doc_lengths) / max(len(doc_lengths), 1) - doc_freq: Dict[str, int] = Counter() - for entry in catalog: - doc_freq.update(set(entry._tokens)) + doc_freq = Counter(tok for entry in catalog for tok in set(entry._tokens)) return doc_lengths, avg_dl, dict(doc_freq), len(catalog) @@ -201,14 +200,12 @@ def _listing_group_label(source_name: str) -> str: def build_catalog_listing_with_form( deferrable: List[Dict[str, Any]], *, max_tokens: int = 4000) -> Tuple[Optional[str], str]: - """Render the skills-style deferred-catalog manifest: ``- name: short desc`` lines grouped - under a heading per source (MCP server / plugin toolset). Returns ``(text, form)``; form is - ``"full"``, ``"names"``, ``"mixed"`` (oversized servers collapsed to a name + count line, - small ones keep per-tool lines), ``"groups"`` (every server summarized) or ``"none"`` + """Render the deferred-catalog manifest: ``- name: short desc`` lines grouped per source. + Returns ``(text, form)``; form is ``"full"``, ``"names"``, ``"mixed"`` (oversized servers + collapsed to a name + count line), ``"groups"`` (every server summarized) or ``"none"`` (over budget even summarized -> text is None). Ordering is deterministic (sorted groups - and tools) so the block is byte-stable across assemblies — the request prefix stays - cacheable. Degradation is PER SERVER, largest first: one huge server must not cost a - small co-attached server its listing.""" + and tools) so the block is byte-stable — the request prefix stays cacheable. Degradation + is PER SERVER, largest first: one huge server must not cost a small one its listing.""" groups: Dict[str, List[Tuple[str, str]]] = {} for td in deferrable: fn = _fn(td) diff --git a/tools/tool_search_validation.py b/tools/tool_search_validation.py index 4ed31b0e10..0c83f219a4 100644 --- a/tools/tool_search_validation.py +++ b/tools/tool_search_validation.py @@ -31,9 +31,7 @@ def _schema_for_local_validation(node: Any) -> Any: return normalized schema_type = normalized.get("type") if isinstance(schema_type, str): - if schema_type != "null": - normalized["type"] = [schema_type, "null"] - return normalized + schema_type = [schema_type] if isinstance(schema_type, list): if "null" not in schema_type: normalized["type"] = [*schema_type, "null"] diff --git a/tools/tour_tool.py b/tools/tour_tool.py index cab47f6da1..c54ee77a25 100644 --- a/tools/tour_tool.py +++ b/tools/tour_tool.py @@ -1,10 +1,9 @@ -"""Guided tour (highlight + narrate UI elements) in the Hermes desktop GUI. Generic: -the agent discovers targets (``action="targets"``), then highlights one step at a -time (``show``) or hands over a step list the user pages (``start``). Round-trips -through the gateway blocking-prompt bridge (``tour.request``/``tour.respond``) so the -agent learns whether the selector matched. Lives in ``desktop_ui`` and withdraws -itself when tours are off: a tour takes the whole screen, so "off" must mean the -model is never told the tool exists rather than offered a call that fails.""" +"""Guided tour (highlight + narrate UI elements) in the Hermes desktop GUI: the agent discovers +targets (``action="targets"``), then highlights one step at a time (``show``) or hands over a +step list the user pages (``start``). Round-trips through the gateway blocking-prompt bridge +(``tour.request``/``tour.respond``) so the agent learns whether the selector matched. Lives in +``desktop_ui`` and withdraws itself when tours are off: a tour takes the whole screen, so "off" +must mean the model is never told the tool exists rather than offered a call that fails.""" import json from typing import Callable, Optional