hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

refactor(tools): unify web_tools debug-error/merge helpers, drop __main__ demo, compact web cache/policy/rescue

Browse Source
This commit is contained in:
Teknium
2026-09-02 22:10:50 -07:00
parent 113f04616b
commit 151119e61c
6 changed files with 215 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
tools/web_result_cache.py
View File

@@ -1,17 +1,13 @@
"""Result caching for web_search / web_extract.
Two caches, both TTL-bounded (default 20 min, ``web.cache_ttl_minutes``;
disable with ``web.cache_enabled: false``). Only successful responses cache.
* **Search memo** — in-memory, per-process, single-flighted: concurrent
identical queries share one paid request. Limits are bucketed to
10/20/50/100 so near-identical requests share an entry; callers slice.
* **Extract cache** — disk-backed under ``cache/web`` (cross-process) with a
JSON sidecar index: URL digest → (file, fetched_at, title). Hits re-run the
normal truncate pipeline with the caller's char_limit.
Lives here, not in generic tool dispatch, so hits sit *after* every safety
check (secret-in-URL, SSRF, policy) and skip only the vendor call.
Two caches, both TTL-bounded (default 20 min, ``web.cache_ttl_minutes``; disable with
``web.cache_enabled: false``). Only successful responses cache.
* **Search memo** — in-memory, per-process, single-flighted: concurrent identical queries share
one paid request. Limits are bucketed to 10/20/50/100 so near-identical requests share an entry.
* **Extract cache** — disk-backed under ``cache/web`` (cross-process) with a JSON sidecar index:
URL digest → (file, fetched_at, title). Hits re-run the normal truncate pipeline.
Lives here, not in generic tool dispatch, so hits sit *after* every safety check (secret-in-URL,
SSRF, policy) and skip only the vendor call.
"""
import hashlib
@@ -62,10 +58,7 @@ def ttl_seconds() -> float:
def bucket_limit(limit: int) -> int:
"""Round a requested result count up to the nearest bucket."""
for b in _LIMIT_BUCKETS:
if limit <= b:
return b
return _LIMIT_BUCKETS[-1]
return next((b for b in _LIMIT_BUCKETS if limit <= b), _LIMIT_BUCKETS[-1])
def normalize_query(query: str) -> str:
@@ -95,9 +88,8 @@ def _deep_copy(response: dict) -> dict:
class SearchMemo:
"""TTL memo + single-flight coalescer for search responses.
Thread-safe: the parallel tool-dispatch pool and subagents share this
process, so identical queries genuinely race. Per-key locks make the
losers wait for (and share) the winner's response.
Thread-safe: the parallel tool-dispatch pool and subagents share this process, so identical
queries genuinely race. Per-key locks make the losers wait for (and share) the winner's response.
"""
def __init__(self) -> None:
@@ -125,9 +117,7 @@ class SearchMemo:
def store(self, provider: str, query: str, limit: int, response: dict) -> None:
"""Cache a SUCCESSFUL response for the bucketed key."""
if not cache_enabled():
return
if not isinstance(response, dict) or not response.get("success"):
if not cache_enabled() or not isinstance(response, dict) or not response.get("success"):
return
key = self._key(provider, query, limit)
with self._store_lock:
@@ -142,16 +132,12 @@ class SearchMemo:
with self._store_lock:
lock = self._key_locks.get(key)
if lock is None:
# Bound the lock table, but never evict a HELD lock: dropping
# one lets a concurrent identical request mint a fresh lock and
# issue a duplicate paid call. locked() is a safe snapshot under
# _store_lock because holders already have their reference.
# Bound the lock table, but never evict a HELD lock: dropping one lets a concurrent
# identical request mint a fresh lock and issue a duplicate paid call. locked() is a
# safe snapshot under _store_lock because holders already have their reference.
if len(self._key_locks) > 256:
self._key_locks = {
k: v for k, v in self._key_locks.items() if v.locked()
}
lock = threading.Lock()
self._key_locks[key] = lock
self._key_locks = {k: v for k, v in self._key_locks.items() if v.locked()}
lock = self._key_locks[key] = threading.Lock()
return lock
def clear(self) -> None:
@@ -216,15 +202,11 @@ def _save_index(index: dict) -> None:
return
try:
if len(index) > _INDEX_MAX_ENTRIES:
newest = sorted(
index.items(),
key=lambda kv: kv[1].get("fetched_at", 0),
reverse=True,
)[:_INDEX_MAX_ENTRIES]
index = dict(newest)
# Per-process tmp name: CLI, gateway, cron, and subagents all write this
# index; a shared tmp name would let concurrent writers truncate each
# other. os.replace is atomic, so the worst outcome is a lost insert.
newest = sorted(index.items(), key=lambda kv: kv[1].get("fetched_at", 0), reverse=True)
index = dict(newest[:_INDEX_MAX_ENTRIES])
# Per-process tmp name: CLI, gateway, cron, and subagents all write this index; a shared tmp
# name would let concurrent writers truncate each other. os.replace is atomic, so the worst
# outcome is a lost insert.
tmp = path.with_suffix(f".tmp.{os.getpid()}")
tmp.write_text(json.dumps(index), encoding="utf-8")
tmp.replace(path)
@@ -275,9 +257,7 @@ def _is_cache_exempt_host(url: str) -> bool:
if not isinstance(patterns, (list, tuple)) or not patterns:
return False
host = _url_host(url)
if not host:
return False
return any(_host_matches_pattern(host, str(p)) for p in patterns)
return bool(host) and any(_host_matches_pattern(host, str(p)) for p in patterns)
except Exception: # noqa: BLE001 — config problems never break tools
return False
@@ -285,16 +265,15 @@ def _is_cache_exempt_host(url: str) -> bool:
def _is_local_dev_url(url: str) -> bool:
"""True for loopback/private/LAN URLs — never cached.
Private-address pages are the user's own fast-changing dev servers;
freshness is the point of fetching them. Hostname heuristics only, no DNS:
this is a freshness decision, not a security boundary (SSRF enforcement
lives in tools/url_safety.py, which blocks these by default anyway).
Private-address pages are the user's own fast-changing dev servers; freshness is the point of
fetching them. Hostname heuristics only, no DNS: this is a freshness decision, not a security
boundary (SSRF enforcement lives in tools/url_safety.py, which blocks these by default anyway).
"""
try:
host = _url_host(url).lower()
if not host:
return True # unparseable → don't cache
if host == "localhost" or host.endswith(".localhost") or host.endswith(".local"):
if host == "localhost" or host.endswith((".localhost", ".local")):
return True
if "." not in host and ":" not in host:
return True # single-label LAN name, not public DNS
@@ -303,10 +282,7 @@ def _is_local_dev_url(url: str) -> bool:
ip = ipaddress.ip_address(host)
except ValueError:
return False # public DNS name
return bool(
ip.is_private or ip.is_loopback or ip.is_link_local
or ip.is_reserved or ip.is_unspecified
)
return bool(ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified)
except Exception: # noqa: BLE001 — on doubt, don't cache
return True
@@ -316,20 +292,13 @@ def _cacheable(url: str) -> bool:
return cache_enabled() and not (_is_local_dev_url(url) or _is_cache_exempt_host(url))
def extract_cache_get(
url: str,
format: Optional[str] = None,
provider: str = "",
) -> Optional[dict]:
def extract_cache_get(url: str, format: Optional[str] = None, provider: str = "") -> Optional[dict]:
"""Return {'url','title','content'} for a fresh cached page, else None."""
if not _cacheable(url):
return None
with _index_lock:
index = _load_index()
entry = index.get(_url_digest(url, format, provider))
if not entry:
return None
if (time.time() - float(entry.get("fetched_at", 0))) >= ttl_seconds():
entry = _load_index().get(_url_digest(url, format, provider))
if not entry or (time.time() - float(entry.get("fetched_at", 0))) >= ttl_seconds():
return None
try:
file_path = Path(entry["file"])
@@ -341,22 +310,10 @@ def extract_cache_get(
except Exception: # noqa: BLE001 — evicted/pruned file == miss
return None
logger.info("web_extract cache hit: %s", url)
return {
"url": url,
"title": entry.get("title", ""),
"content": content,
"error": None,
"cached": True,
}
return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True}
def extract_cache_put(
url: str,
content: str,
title: str = "",
format: Optional[str] = None,
provider: str = "",
) -> None:
def extract_cache_put(url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = "") -> None:
"""Store one successful extraction's full clean text for TTL reuse.
Pages over the truncate-store ceiling are not cached: serving a capped copy
@@ -376,10 +333,7 @@ def extract_cache_put(
with _index_lock:
index = _load_index()
index[_url_digest(url, format, provider)] = {
"url": url,
"file": str(file_path),
"title": title or "",
"fetched_at": time.time(),
"url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),
}
_save_index(index)
except Exception as exc: # noqa: BLE001 — cache writes are best-effort

390
tools/web_tools.py
View File

@@ -6,37 +6,25 @@ capability via ``web.search_backend`` / ``web.extract_backend``). Every vendor
implementation lives in ``plugins/web/<vendor>/provider.py`` and registers with
``agent.web_search_registry``; this module owns selection, safety gates,
caching, keyless rescue, and the truncate-and-store result pipeline.
Debug: ``WEB_TOOLS_DEBUG=true`` writes ``logs/web_tools_debug_<UUID>.json``.
"""
import json
import logging
import os
from typing import List, Dict, Any, Optional, TYPE_CHECKING
from typing import List, Dict, Any, Optional
import httpx # noqa: F401 — kept at module top so tests can patch tools.web_tools.httpx
# Vendor helpers re-exported so external code and unit-test patches of
# ``tools.web_tools.<name>`` keep working after the plugin migration.
if TYPE_CHECKING:
from firecrawl import Firecrawl # noqa: F401 — type hints only
from plugins.web.firecrawl.provider import ( # noqa: F401 — backward-compat names
Firecrawl,
_firecrawl_backend_help_suffix,
_get_firecrawl_client,
_get_firecrawl_gateway_url,
_is_tool_gateway_ready,
check_firecrawl_api_key,
Firecrawl, _firecrawl_backend_help_suffix, _get_firecrawl_client, _get_firecrawl_gateway_url,
_is_tool_gateway_ready, check_firecrawl_api_key,
)
from plugins.web.tavily.provider import ( # noqa: F401 — backward-compat names
_normalize_tavily_documents,
_normalize_tavily_search_results,
_tavily_request,
)
from plugins.web.parallel.provider import ( # noqa: F401 — backward-compat names
_get_async_parallel_client,
_get_parallel_client,
_normalize_tavily_documents, _normalize_tavily_search_results, _tavily_request,
)
from plugins.web.parallel.provider import _get_async_parallel_client, _get_parallel_client # noqa: F401
from plugins.web.exa.provider import _get_exa_client # noqa: F401
# Per-vendor client cache slots. Plugins read/write these via tools.web_tools so
@@ -49,50 +37,26 @@ _exa_client: Optional[Any] = None
from tools.debug_helpers import DebugSession
from tools.managed_tool_gateway import ( # noqa: F401 — backward-compat names for tests
build_vendor_gateway_url,
peek_nous_access_token as _peek_nous_access_token,
read_nous_access_token as _read_nous_access_token,
resolve_managed_tool_gateway,
build_vendor_gateway_url, resolve_managed_tool_gateway,
peek_nous_access_token as _peek_nous_access_token, read_nous_access_token as _read_nous_access_token,
)
from tools.tool_backend_helpers import ( # noqa: F401
managed_nous_tools_enabled,
nous_tool_gateway_unavailable_message,
prefers_gateway,
managed_nous_tools_enabled, nous_tool_gateway_unavailable_message, prefers_gateway,
)
from tools.url_safety import async_is_safe_url
from tools.web_tools_rescue import ( # noqa: F401 — re-exported (tests patch tools.web_tools.<name>)
_keyless_rescue_enabled,
_policy_blocked_result,
_rescue_eligible,
_rescue_extract,
_rescue_search,
_keyless_rescue_enabled, _policy_blocked_result, _rescue_eligible, _rescue_extract, _rescue_search,
)
from tools.web_tools_truncate import ( # noqa: F401 — re-exported (tests + web_result_cache import via tools.web_tools)
DEFAULT_EXTRACT_CHAR_LIMIT,
MAX_STORED_TEXT_CHARS,
_clamp_char_limit,
_effective_char_limit,
_get_extract_char_limit,
_store_full_text,
_trim_results,
_truncate_results,
_truncate_with_footer,
DEFAULT_EXTRACT_CHAR_LIMIT, MAX_STORED_TEXT_CHARS, _clamp_char_limit, _effective_char_limit,
_get_extract_char_limit, _store_full_text, _trim_results, _truncate_results, _truncate_with_footer,
convert_base64_images_to_links,
)
from tools.web_tools_extract import ( # noqa: F401 — re-exported
_EXTRACT_BACKENDS_HINT,
_NO_RESULT_ERROR,
_disabled_plugin_error,
_extract_error_json,
_extract_safe_urls,
_no_provider_error,
_resolve_extract_provider,
_result_entry,
_strict_selection_error,
_validate_extract_urls,
_web_extract_url,
_EXTRACT_BACKENDS_HINT, _NO_RESULT_ERROR, _disabled_plugin_error, _extract_error_json, _extract_safe_urls,
_merge_in_order, _no_provider_error, _resolve_extract_provider, _result_entry, _strict_selection_error,
_validate_extract_urls, _web_extract_url,
)
import sys
logger = logging.getLogger(__name__)
@@ -100,14 +64,9 @@ logger = logging.getLogger(__name__)
# ─── Backend Selection ────────────────────────────────────────────────────────
def _env_value(name: str) -> str:
"""Resolve ``name`` via the Hermes config-aware env layer, then process env.
Values set through ``hermes config set`` / ``hermes tools`` must be honored
by autodetect and ``check_web_api_key()``, not just raw exports.
"""
"""Resolve ``name`` via the Hermes config-aware env layer (``hermes config set`` values), then process env."""
try:
from hermes_cli.config import get_env_value
val = get_env_value(name)
except Exception:
val = None
@@ -134,10 +93,9 @@ def _configured_backend(key: str = "backend") -> str:
return (_load_web_config().get(key) or "").lower().strip()
# Built-in backends probed by the hardcoded checks in _BUILTIN_AVAILABILITY.
# Any other name is a plugin-registered provider resolved via the registry's
# ``is_available()``. Includes ``xai`` (probed via has_xai_credentials(), not a
# registered provider) even though the registry's _LEGACY_PREFERENCE omits it —
# Built-in backends probed by the hardcoded checks in _BUILTIN_AVAILABILITY. Any other name is a
# plugin-registered provider resolved via the registry's ``is_available()``. Includes ``xai`` (probed via
# has_xai_credentials(), not a registered provider) even though the registry's _LEGACY_PREFERENCE omits it —
# if xai ever ships as a registered provider, drop it here.
_LEGACY_WEB_BACKENDS = frozenset(
{"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "keenable"}
@@ -150,7 +108,6 @@ def _registered_web_provider(backend: str):
return None
try:
from agent.web_search_registry import get_provider
return get_provider(backend)
except Exception as exc: # noqa: BLE001 — registry optional; never fatal
logger.debug("web provider registry lookup failed for %r: %s", backend, exc)
@@ -165,26 +122,14 @@ def _probe(provider, method: str, context: str = "") -> Optional[bool]:
try:
return bool(getattr(provider, method)())
except Exception as exc: # noqa: BLE001 — a broken provider is "unavailable"
logger.debug(
"web provider %r.%s() raised%s: %s",
getattr(provider, "name", provider), method, context, exc,
)
logger.debug("web provider %r.%s() raised%s: %s", getattr(provider, "name", provider), method, context, exc)
return None
def _registered_web_provider_available(backend: str):
"""``is_available()`` of a registered provider, or ``None`` when unregistered (caller falls through)."""
provider = _registered_web_provider(backend)
if provider is None:
return None
return _probe(provider, "is_available") or False
def _list_registered_web_providers():
"""All plugin-registered web providers (empty list on failure)."""
try:
from agent.web_search_registry import list_providers
return list_providers()
except Exception as exc: # noqa: BLE001 — registry optional; never fatal
logger.debug("web provider registry list failed: %s", exc)
@@ -194,29 +139,26 @@ def _list_registered_web_providers():
def _get_backend() -> str:
"""Shared web backend name.
A stored ``web.backend`` is returned as-is — no availability probe, no
fallback — so a broken selection surfaces the vendor's honest error rather
than silently rerouting. The autodetect ladder runs ONLY when no web
selection has ever been stored.
A stored ``web.backend`` is returned as-is — no availability probe, no fallback — so a broken
selection surfaces the vendor's honest error rather than silently rerouting. The autodetect
ladder runs ONLY when no web selection has ever been stored.
"""
configured = _configured_backend()
if configured:
# "nous" (managed subscription) is serviced by the firecrawl provider,
# whose client resolver routes it through the managed Tool Gateway.
from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER
return "firecrawl" if configured == NOUS_MANAGED_PROVIDER else configured
from tools.tool_backend_helpers import selection_exists
if selection_exists("web"):
# Selection exists (use_gateway / per-capability keys) but no shared
# name: keep the firecrawl default rather than credential-laddering.
return "firecrawl"
# Never-configured install. Explicit user credentials beat the managed-
# gateway probe (a Nous OAuth token's tier may not grant web access, and the
# gateway then fails at runtime with no fallback). Free tiers trail paid.
# Never-configured install. Explicit user credentials beat the managed-gateway probe (a Nous
# OAuth token's tier may not grant web access, and the gateway then fails at runtime with no
# fallback). Free tiers trail paid.
backend_candidates = (
("tavily", _has_env("TAVILY_API_KEY")),
("exa", _has_env("EXA_API_KEY")),
@@ -232,19 +174,16 @@ def _get_backend() -> str:
if available:
return backend
# Plugin-contributed providers (built-ins are covered above). We already
# hold the provider object, so probe it directly instead of re-looking-up.
# Plugin-contributed providers (built-ins are covered above); probe the held object directly.
for provider in _list_registered_web_providers():
if provider.name not in _LEGACY_WEB_BACKENDS and _probe(provider, "is_available"):
return provider.name
# Keyless free tier — strictly last so it never pre-empts a keyed backend.
# Discovery must run first: reachable from contexts that haven't loaded
# plugins (subprocess agent runs, delegate children, scripts).
# Keyless free tier — strictly last so it never pre-empts a keyed backend. Discovery must run
# first: reachable from contexts that haven't loaded plugins (subprocess runs, delegate children).
try:
_ensure_web_plugins_loaded()
from agent.web_search_registry import _keyless_preference, _keyless_tier_enabled
if _keyless_tier_enabled():
for name in _keyless_preference():
provider = _registered_web_provider(name)
@@ -256,26 +195,18 @@ def _get_backend() -> str:
return "firecrawl" # default (backward compat)
def _get_capability_backend(capability: str) -> str:
"""``web.{capability}_backend`` if stored (strict, no probe), else ``_get_backend()``."""
return _configured_backend(f"{capability}_backend") or _get_backend()
def _get_search_backend() -> str:
"""Backend for web_search: ``web.search_backend`` > ``web.backend`` > autodetect."""
return _get_capability_backend("search")
"""Backend for web_search: ``web.search_backend`` (strict, no probe) > ``web.backend`` > autodetect."""
return _configured_backend("search_backend") or _get_backend()
def _get_extract_backend() -> str:
"""Backend for web_extract: ``web.extract_backend`` > ``web.backend`` > autodetect."""
return _get_capability_backend("extract")
"""Backend for web_extract: ``web.extract_backend`` (strict, no probe) > ``web.backend`` > autodetect."""
return _configured_backend("extract_backend") or _get_backend()
def _tavily_explicitly_configured() -> bool:
return any(
_configured_backend(key) == "tavily"
for key in ("backend", "search_backend", "extract_backend")
)
return any(_configured_backend(key) == "tavily" for key in ("backend", "search_backend", "extract_backend"))
def _xai_available() -> bool:
@@ -297,9 +228,8 @@ def _ddgs_package_importable() -> bool:
return False
# Availability probes for the built-in backends (see _LEGACY_WEB_BACKENDS).
# Lambdas so tests patching module-level helpers (e.g. _ddgs_package_importable,
# check_firecrawl_api_key) are honored at call time.
# Availability probes for the built-in backends (see _LEGACY_WEB_BACKENDS). Lambdas so tests patching
# module-level helpers (e.g. _ddgs_package_importable, check_firecrawl_api_key) are honored at call time.
_BUILTIN_AVAILABILITY = {
"exa": lambda: _has_env("EXA_API_KEY"),
"parallel": lambda: _has_env("PARALLEL_API_KEY"),
@@ -316,14 +246,14 @@ _BUILTIN_AVAILABILITY = {
def _is_backend_available(backend: str) -> bool:
"""True when *backend* is usable — the single availability chokepoint.
Non-legacy names delegate to the registered provider's ``is_available()``;
built-ins use the cheap hardcoded probes.
Non-legacy names delegate to the registered provider's ``is_available()`` (unregistered names
fall through); built-ins use the cheap hardcoded probes.
"""
backend = (backend or "").lower().strip()
if backend not in _LEGACY_WEB_BACKENDS:
registered = _registered_web_provider_available(backend)
if registered is not None:
return registered
provider = _registered_web_provider(backend)
if provider is not None:
return _probe(provider, "is_available") or False
probe = _BUILTIN_AVAILABILITY.get(backend)
return probe() if probe else False
@@ -331,25 +261,17 @@ def _is_backend_available(backend: str) -> bool:
def _web_requires_env() -> list[str]:
"""Tool-registry metadata env vars for the web backends.
Gateway vars are always listed: gating them on ``managed_nous_tools_enabled()``
cost a synchronous portal HTTP refresh at every CLI startup. Contract: set var
-> tool sees it; not-logged-in users simply lack the vars, so extras are harmless.
Gateway vars are always listed: gating them on ``managed_nous_tools_enabled()`` cost a synchronous
portal HTTP refresh at every CLI startup. Contract: set var -> tool sees it; not-logged-in users
simply lack the vars, so extras are harmless.
"""
return [
"EXA_API_KEY",
"PARALLEL_API_KEY",
"TAVILY_API_KEY",
"KEENABLE_API_KEY",
"FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL",
"FIRECRAWL_GATEWAY_URL",
"TOOL_GATEWAY_DOMAIN",
"TOOL_GATEWAY_SCHEME",
"EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY",
"FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME",
"TOOL_GATEWAY_USER_TOKEN",
]
# Truncate-and-store pipeline lives in tools/web_tools_truncate.py (re-imported above).
_debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
@@ -358,14 +280,12 @@ _debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
def _ensure_web_plugins_loaded() -> None:
"""Idempotently run plugin discovery so the web registry is populated.
Dispatch is reachable from contexts that never triggered discovery
(subprocess agent runs, delegate children, scripts); without it the
registry is empty and a configured backend yields a misleading
"No web ... provider configured" error.
Dispatch is reachable from contexts that never triggered discovery (subprocess agent runs,
delegate children, scripts); without it the registry is empty and a configured backend yields
a misleading "No web ... provider configured" error.
"""
try:
from hermes_cli.plugins import _ensure_plugins_discovered
_ensure_plugins_discovered()
except Exception as exc: # noqa: BLE001
# Warning, not debug: a broken plugin import is otherwise invisible.
@@ -377,12 +297,20 @@ def _finish_debug(call_name: str, debug_call_data: dict) -> None:
_debug.save()
def _debug_error(call_name: str, debug_call_data: dict, error_msg: str) -> str:
"""Record *error_msg* in the debug session and return the ``tool_error`` envelope for it."""
logger.debug("%s", error_msg)
debug_call_data["error"] = error_msg
_finish_debug(call_name, debug_call_data)
return tool_error(error_msg)
def web_search_tool(query: str, limit: int = 5) -> str:
"""Search the web via the configured backend.
Returns a JSON string ``{"success": bool, "data": {"web": [{"title", "url",
"description", "position"}, ...]}}`` (metadata only — use web_extract_tool
for page content) or ``{"success": false, "error": ...}``.
Returns a JSON string ``{"success": bool, "data": {"web": [{"title", "url", "description",
"position"}, ...]}}`` (metadata only — use web_extract_tool for page content) or
``{"success": false, "error": ...}``.
"""
try:
limit = int(limit)
@@ -405,23 +333,16 @@ def web_search_tool(query: str, limit: int = 5) -> str:
# Sync only — every provider's search() is sync.
_ensure_web_plugins_loaded()
from agent.web_search_registry import (
get_active_search_provider,
get_provider as _wsp_get_provider,
)
from agent.web_search_registry import get_active_search_provider, get_provider as _wsp_get_provider
backend = _get_search_backend()
provider = _wsp_get_provider(backend) if backend else None
if provider is None or not provider.supports_search():
from tools.tool_backend_helpers import selection_exists
if provider is None and backend and selection_exists("web"):
error_text = _strict_selection_error("search", backend)
debug_call_data["error"] = error_text
error_text = debug_call_data["error"] = _strict_selection_error("search", backend)
_finish_debug("web_search_tool", debug_call_data)
return json.dumps(
{"success": False, "error": error_text}, indent=2, ensure_ascii=False
)
return json.dumps({"success": False, "error": error_text}, indent=2, ensure_ascii=False)
# Never-configured install: legacy availability-walked autodetect.
provider = get_active_search_provider()
@@ -429,15 +350,11 @@ def web_search_tool(query: str, limit: int = 5) -> str:
response_data = {
"success": False,
"error": _no_provider_error(
"search",
"No web search provider configured. Run `hermes tools` to set one up.",
"search", "No web search provider configured. Run `hermes tools` to set one up."
),
}
else:
logger.info(
"Web search via %s: '%s' (limit: %d)",
provider.name, query, limit,
)
logger.info("Web search via %s: '%s' (limit: %d)", provider.name, query, limit)
response_data = _memoized_search(provider, query, limit)
debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", []))
@@ -447,20 +364,16 @@ def web_search_tool(query: str, limit: int = 5) -> str:
return result_json
except Exception as e:
error_msg = f"Error searching web: {str(e)}"
logger.debug("%s", error_msg)
debug_call_data["error"] = error_msg
_finish_debug("web_search_tool", debug_call_data)
return tool_error(error_msg)
return _debug_error("web_search_tool", debug_call_data, f"Error searching web: {str(e)}")
def _memoized_search(provider, query: str, limit: int) -> dict:
"""TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py).
Sits after every safety/config check. The provider is asked for the
BUCKETED count so near-identical limits share an entry; the caller's count
is sliced out. Only successful, non-rescued responses are cached — caching
a rescue would make the one-shot ring fallback sticky for a whole TTL.
Sits after every safety/config check. The provider is asked for the BUCKETED count so
near-identical limits share an entry; the caller's count is sliced out. Only successful,
non-rescued responses are cached — caching a rescue would make the one-shot ring fallback
sticky for a whole TTL.
"""
from tools.web_result_cache import bucket_limit, search_memo, slice_search_response
@@ -473,9 +386,7 @@ def _memoized_search(provider, query: str, limit: int) -> dict:
raise
return _rescue_search(provider.name, str(exc), query, fetch_limit), True
if not resp.get("success") and _rescue_eligible(provider):
return _rescue_search(
provider.name, str(resp.get("error", "")), query, fetch_limit
), True
return _rescue_search(provider.name, str(resp.get("error", "")), query, fetch_limit), True
return resp, False
response_data = search_memo.lookup(provider.name, query, limit)
@@ -490,19 +401,13 @@ def _memoized_search(provider, query: str, limit: int) -> dict:
return slice_search_response(response_data, limit)
async def web_extract_tool(
urls: List[Any],
format: str = None,
char_limit: Optional[int] = None,
) -> str:
async def web_extract_tool(urls: List[Any], format: str = None, char_limit: Optional[int] = None) -> str:
"""Extract clean page content (no LLM) from URLs via the configured backend.
Pages over ``char_limit`` (default web.extract_char_limit or 15000) are
head+tail truncated with a footer pointing at the stored full text. Inline
base64 images become ``[IMAGE: alt]`` placeholders. URLs carrying secrets
are refused before any fetch; private-network URLs are blocked per entry.
Returns a JSON string with a ``results`` list of ``url``/``title``/``content``/``error``.
Pages over ``char_limit`` (default web.extract_char_limit or 15000) are head+tail truncated with
a footer pointing at the stored full text. Inline base64 images become ``[IMAGE: alt]``
placeholders. URLs carrying secrets are refused before any fetch; private-network URLs are
blocked per entry. Returns a JSON string with a ``results`` list of url/title/content/error.
"""
normalized_urls, normalized_indices, invalid_urls, blocked = _validate_extract_urls(urls)
if blocked is not None:
@@ -523,17 +428,14 @@ async def web_extract_tool(
logger.info("Extracting content from %d URL(s)", len(normalized_urls))
# SSRF protection — filter private/internal URLs before any backend.
safe_urls = []
safe_indices = []
safe_urls, safe_indices = [], []
ssrf_blocked: Dict[int, Dict[str, Any]] = {}
for index, url in zip(normalized_indices, normalized_urls):
if not await async_is_safe_url(url):
ssrf_blocked[index] = _result_entry(
url, "Blocked: URL targets a private or internal network address"
)
else:
if await async_is_safe_url(url):
safe_urls.append(url)
safe_indices.append(index)
else:
ssrf_blocked[index] = _result_entry(url, "Blocked: URL targets a private or internal network address")
if not safe_urls:
results = []
@@ -548,33 +450,19 @@ async def web_extract_tool(
# Reconstruct input order across invalid, blocked, and provider entries
# (providers preserve the order of the safe URL list they receive).
if invalid_urls or ssrf_blocked:
safe_results = {
index: (
results[position]
if position < len(results)
else _result_entry(safe_urls[position], _NO_RESULT_ERROR)
)
for position, index in enumerate(safe_indices)
}
by_index = {**safe_results, **ssrf_blocked, **invalid_urls}
results = [by_index[index] for index in range(len(urls))]
results = _merge_in_order(len(urls), {**ssrf_blocked, **invalid_urls}, safe_indices, safe_urls, results)
response = {"results": results}
pages_extracted = len(response.get('results', []))
logger.info("Extracted content from %d pages", pages_extracted)
debug_call_data["pages_extracted"] = pages_extracted
debug_call_data["original_response_size"] = len(json.dumps(response))
logger.info("Extracted content from %d pages", len(results))
debug_call_data["pages_extracted"] = len(results)
debug_call_data["original_response_size"] = len(json.dumps({"results": results}))
debug_call_data["processing_applied"].append("truncate_and_store")
_truncate_results(response.get("results", []), _effective_char_limit(char_limit), debug_call_data)
trimmed_response = {"results": _trim_results(response.get("results", []))}
if trimmed_response.get("results") == []:
_truncate_results(results, _effective_char_limit(char_limit), debug_call_data)
trimmed = _trim_results(results)
if not trimmed:
result_json = tool_error("Content was inaccessible or not found")
else:
result_json = json.dumps(trimmed_response, indent=2, ensure_ascii=False)
result_json = json.dumps({"results": trimmed}, indent=2, ensure_ascii=False)
# Belt-and-suspenders sweep over the serialized JSON in case a provider
# tucked a base64 blob somewhere unexpected (e.g. metadata).
@@ -586,20 +474,16 @@ async def web_extract_tool(
return cleaned_result
except Exception as e:
error_msg = f"Error extracting content: {str(e)}"
logger.debug("%s", error_msg)
debug_call_data["error"] = error_msg
_finish_debug("web_extract_tool", debug_call_data)
return tool_error(error_msg)
return _debug_error("web_extract_tool", debug_call_data, f"Error extracting content: {str(e)}")
def _provider_is_ready(provider) -> bool:
"""True when *provider* is keyed-available OR keyless-capable, without raising.
``get_active_*_provider()`` returns an explicitly configured backend even
when ``is_available()`` is False (so dispatch can emit a precise error), so
readiness gates (tool check_fn, ``hermes doctor``) must probe for real.
Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig.
``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()``
is False (so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes
doctor``) must probe for real. Keyless mode (Exa/Parallel free tier) is a working state, not a
misconfig.
"""
if provider is None:
return False
@@ -615,9 +499,8 @@ def _provider_is_ready(provider) -> bool:
def check_web_api_key() -> bool:
"""``check_fn`` gate for web_search / web_extract: is any web backend available?
A plugin-registered provider reporting ``is_available()`` must light the
tools up even with no built-in credentials; resolution funnels through
:func:`_is_backend_available`.
A plugin-registered provider reporting ``is_available()`` must light the tools up even with no
built-in credentials; resolution funnels through :func:`_is_backend_available`.
"""
configured = _configured_backend()
if configured and _is_backend_available(configured):
@@ -629,94 +512,13 @@ def check_web_api_key() -> bool:
# time, before any dispatch has populated the registry.
try:
_ensure_web_plugins_loaded()
from agent.web_search_registry import (
get_active_search_provider,
get_active_extract_provider,
)
return (
_provider_is_ready(get_active_search_provider())
or _provider_is_ready(get_active_extract_provider())
)
from agent.web_search_registry import get_active_search_provider, get_active_extract_provider
return _provider_is_ready(get_active_search_provider()) or _provider_is_ready(get_active_extract_provider())
except Exception as exc: # noqa: BLE001 — registry optional; never fatal
logger.debug("web provider registry availability check failed: %s", exc)
return False
_DEMO_BACKEND_LINES = {
"exa": " Using Exa API (https://exa.ai)",
"parallel": " Using Parallel API (https://parallel.ai)",
"brave-free": " Using Brave Search free tier (search only)",
"ddgs": " Using DuckDuckGo via ddgs package (search only)",
}
if __name__ == "__main__":
"""
Simple test/demo when run directly
"""
print("🌐 Standalone Web Tools Module")
print("=" * 40)
web_available = check_web_api_key()
from hermes_cli.config import get_env_value as _gev
if web_available:
backend = _get_backend()
print(f"✅ Web backend: {backend}")
if backend in _DEMO_BACKEND_LINES:
print(_DEMO_BACKEND_LINES[backend])
elif backend == "tavily":
if _has_env("TAVILY_API_KEY"):
print(" Using Tavily API (https://tavily.com)")
else:
print(" Using Tavily keyless (https://docs.tavily.com/documentation/keyless)")
elif backend == "searxng":
print(f" Using SearXNG (search only): {_env_value('SEARXNG_URL')}")
elif (_gev("FIRECRAWL_API_URL") or "").strip():
print(f" Using self-hosted Firecrawl: {(_gev('FIRECRAWL_API_URL') or '').strip().rstrip('/')}")
elif (_gev("FIRECRAWL_API_KEY") or "").strip():
print(" Using direct Firecrawl cloud API")
elif _is_tool_gateway_ready():
print(f" Using Firecrawl tool-gateway: {_get_firecrawl_gateway_url()}")
else:
print(" Firecrawl backend selected but not configured")
else:
print("❌ No web search backend configured")
print(
"Set EXA_API_KEY, PARALLEL_API_KEY, TAVILY_API_KEY, KEENABLE_API_KEY, FIRECRAWL_API_KEY, FIRECRAWL_API_URL"
f"{_firecrawl_backend_help_suffix()}"
)
sys.exit(1)
print("🛠️ Web tools ready for use!")
print(f" Extract char limit: {_get_extract_char_limit()} chars "
"(pages over this are truncated; full text stored in cache/web)")
if _debug.active:
print(f"🐛 Debug mode ENABLED - Session ID: {_debug.session_id}")
print(f" Debug logs will be saved to: {_debug.log_dir}/web_tools_debug_{_debug.session_id}.json")
else:
print("🐛 Debug mode disabled (set WEB_TOOLS_DEBUG=true to enable)")
print("\nBasic usage:")
print(" from web_tools import web_search_tool, web_extract_tool")
print(" import asyncio")
print("")
print(" # Search (synchronous)")
print(" results = web_search_tool('Python tutorials')")
print("")
print(" # Extract (asynchronous, no LLM — truncate-and-store)")
print(" async def main():")
print(" content = await web_extract_tool(['https://example.com'])")
print(" # bigger budget for one call:")
print(" content = await web_extract_tool(['https://docs.python.org'], char_limit=40000)")
print(" asyncio.run(main())")
print("\nDebug mode:")
print(" export WEB_TOOLS_DEBUG=true")
print(" # Logs saved to: ./logs/web_tools_debug_UUID.json")
# ---------------------------------------------------------------------------
# Registry
# ---------------------------------------------------------------------------

85
tools/web_tools_extract.py
View File

@@ -3,11 +3,8 @@
Order of controls (each is a gate, never skipped by a cache hit): secret-URL
refusal -> SSRF filter (in web_tools.web_extract_tool) -> provider resolution
(strict selection) -> per-URL website policy -> disk cache -> vendor call with
one-shot keyless rescue.
Extracted from tools/web_tools.py; the names are re-imported there so
``tools.web_tools._validate_extract_urls`` etc. keep working. Logs under the
origin logger name for parity.
one-shot keyless rescue. Names are re-imported by tools/web_tools.py so
``tools.web_tools._validate_extract_urls`` etc. keep working; logs under the origin logger.
"""
import asyncio
@@ -20,6 +17,9 @@ from tools.web_tools_rescue import _rescue_eligible, _rescue_extract
logger = logging.getLogger("tools.web_tools")
_NO_RESULT_ERROR = "Extract backend returned no result for this URL"
_EXTRACT_BACKENDS_HINT = "firecrawl, tavily, keenable, exa, or parallel."
def _web_extract_url(value: Any) -> Optional[str]:
"""URL from a model-supplied extract item (str, or dict with ``url``/``href``); None if unusable.
@@ -31,8 +31,7 @@ def _web_extract_url(value: Any) -> Optional[str]:
value = value.get("url") or value.get("href")
if not isinstance(value, str):
return None
value = value.strip()
return value or None
return value.strip() or None
def _disabled_plugin_error(capability: str, disabled_key: str) -> str:
@@ -57,9 +56,7 @@ def _strict_selection_error(capability: str, backend: str) -> str:
disabled_key = _disabled_web_plugin_for(capability=capability)
if disabled_key:
return _disabled_plugin_error(capability, disabled_key)
return selection_error(
"web", f"'{backend}'", f"no registered web {capability} provider has that name"
)
return selection_error("web", f"'{backend}'", f"no registered web {capability} provider has that name")
def _no_provider_error(capability: str, fallback: str) -> str:
@@ -74,14 +71,21 @@ def _result_entry(url: str, error: Optional[str]) -> Dict[str, Any]:
return {"url": url, "title": "", "content": "", "error": error}
_NO_RESULT_ERROR = "Extract backend returned no result for this URL"
_EXTRACT_BACKENDS_HINT = "firecrawl, tavily, keenable, exa, or parallel."
def _extract_error_json(error: str) -> str:
return json.dumps({"success": False, "error": error}, ensure_ascii=False)
def _merge_in_order(
total: int, fixed: Dict[int, dict], fetch_positions: List[int], fetch_urls: List[str], results: List[dict]
) -> List[dict]:
"""Rebuild a ``total``-long result list: *fixed* entries by position, fetched *results* at
*fetch_positions* (a short provider list yields ``_NO_RESULT_ERROR`` entries for the rest)."""
merged = dict(fixed)
for pos, position in enumerate(fetch_positions):
merged[position] = results[pos] if pos < len(results) else _result_entry(fetch_urls[pos], _NO_RESULT_ERROR)
return [merged[i] for i in range(total)]
def _validate_extract_urls(urls: List[Any]):
"""Normalize model-supplied items and block URLs carrying secrets.
@@ -133,14 +137,11 @@ def _validate_extract_urls(urls: List[Any]):
def _resolve_extract_provider(backend: str):
"""Resolve the extract provider for *backend*; returns ``(provider, error_json)``.
A registered search-only backend is a typed error (never a silent switch).
An unregistered name with a stored web selection is a strict-selection
error; with no selection, fall through to the availability walk.
A registered search-only backend is a typed error (never a silent switch). An unregistered
name with a stored web selection is a strict-selection error; with no selection, fall
through to the availability walk.
"""
from agent.web_search_registry import (
get_active_extract_provider,
get_provider as _wsp_get_provider,
)
from agent.web_search_registry import get_active_extract_provider, get_provider as _wsp_get_provider
provider = _wsp_get_provider(backend) if backend else None
if provider is not None and provider.supports_extract():
@@ -158,9 +159,7 @@ def _resolve_extract_provider(backend: str):
provider = get_active_extract_provider()
if provider is None:
return None, _extract_error_json(_no_provider_error(
"extract",
"No web extract provider configured. Set web.extract_backend to "
+ _EXTRACT_BACKENDS_HINT,
"extract", "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT,
))
return provider, None
@@ -188,32 +187,23 @@ async def _dispatch_extract(provider, fetch_urls: List[str], format: Optional[st
return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, results)
# Cache each successful fetch's full clean text (best-effort; oversized skipped).
for fetched_pos, fetched in enumerate(results):
if fetched_pos >= len(fetch_urls):
break
for url, fetched in zip(fetch_urls, results):
if fetched.get("error"):
continue
_content = fetched.get("raw_content", "") or fetched.get("content", "")
if _content:
extract_cache_put(
fetch_urls[fetched_pos],
_content,
title=fetched.get("title", ""),
format=format,
provider=provider.name,
)
extract_cache_put(url, _content, title=fetched.get("title", ""), format=format, provider=provider.name)
return results
async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[str]) -> List[dict]:
"""Serve cache hits, fetch the rest, and merge back in ``safe_urls`` order.
The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate,
SSRF gate, and provider resolution, and is gated per-URL on the website
policy — a hit skips only the vendor call, never a control. Policy-blocked
URLs are cache misses so dispatch handles them exactly as without a cache.
Keys include provider and format, so switching either within the TTL never
serves the other's content.
The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate, SSRF gate, and
provider resolution, and is gated per-URL on the website policy — a hit skips only the vendor
call, never a control. Policy-blocked URLs are cache misses so dispatch handles them exactly
as without a cache. Keys include provider and format, so switching either within the TTL
never serves the other's content.
"""
from tools.web_result_cache import extract_cache_get
from tools.website_policy import check_website_access as _check_site
@@ -222,13 +212,11 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st
fetch_urls: List[str] = []
fetch_positions: List[int] = []
for position, url in enumerate(safe_urls):
hit = None
try:
_policy_block = _check_site(url)
except Exception: # noqa: BLE001 — policy errors fail open like dispatch
_policy_block = None
if _policy_block is None:
hit = extract_cache_get(url, format=format, provider=provider.name)
hit = extract_cache_get(url, format=format, provider=provider.name) if _policy_block is None else None
if hit is not None:
cached_results[position] = hit
else:
@@ -242,13 +230,4 @@ async def _extract_safe_urls(provider, safe_urls: List[str], format: Optional[st
results = await _dispatch_extract(provider, fetch_urls, format)
if not cached_results:
return results
merged: List[Dict[str, Any]] = [None] * len(safe_urls) # type: ignore[list-item]
for position, hit in cached_results.items():
merged[position] = hit
for fetched_pos, position in enumerate(fetch_positions):
merged[position] = (
results[fetched_pos]
if fetched_pos < len(results)
else _result_entry(safe_urls[position], _NO_RESULT_ERROR)
)
return merged
return _merge_in_order(len(safe_urls), cached_results, fetch_positions, fetch_urls, results)

47
tools/web_tools_rescue.py
View File

@@ -3,11 +3,8 @@
Stateless by design: a rescue routes THIS call through the free-tier ring
(plugins/web/keyless_mcp.py); the next web_search/web_extract call attempts
the chosen backend again. Callers must never cache a rescue-served response,
or the one-shot rescue becomes sticky for a whole TTL.
Extracted from tools/web_tools.py; the names are re-imported there so
``tools.web_tools._rescue_eligible`` etc. keep working. Logs under the
origin logger name for parity.
or the one-shot rescue becomes sticky for a whole TTL. Names are re-imported by
tools/web_tools.py (``tools.web_tools._rescue_eligible``); logs under the origin logger.
"""
import logging
@@ -16,10 +13,7 @@ logger = logging.getLogger("tools.web_tools")
# Ring vendor -> env var holding its paid key (keyed mode ⇒ eligible for rescue).
_RING_KEY_VARS = {
"exa": "EXA_API_KEY",
"parallel": "PARALLEL_API_KEY",
"firecrawl": "FIRECRAWL_API_KEY",
"keenable": "KEENABLE_API_KEY",
"exa": "EXA_API_KEY", "parallel": "PARALLEL_API_KEY", "firecrawl": "FIRECRAWL_API_KEY", "keenable": "KEENABLE_API_KEY",
}
@@ -41,9 +35,8 @@ def _keyless_rescue_enabled() -> bool:
def _rescue_eligible(provider) -> bool:
"""True when a failed call on *provider* should get a one-shot rescue.
Eligible: a keyed/configured path — any non-ring backend, or a ring vendor
in keyed mode. A ring vendor already in keyless mode is NOT eligible: its
failure means the ring was already walked.
Eligible: a keyed/configured path — any non-ring backend, or a ring vendor in keyed mode. A ring
vendor already in keyless mode is NOT eligible: its failure means the ring was already walked.
"""
if not _keyless_rescue_enabled() or provider is None:
return False
@@ -67,10 +60,7 @@ def _rescue_search(provider_name: str, original_error: str, query: str, limit: i
"""Rescue a failed search via the ring; annotate the result with the original failure."""
from plugins.web.keyless_mcp import search_with_failover
logger.warning(
"web_search backend '%s' failed (%s); one-shot keyless rescue",
provider_name, (original_error or "")[:200],
)
logger.warning("web_search backend '%s' failed (%s); one-shot keyless rescue", provider_name, (original_error or "")[:200])
rescued = search_with_failover(provider_name, query, limit)
if rescued.get("success"):
data = rescued.setdefault("data", {})
@@ -85,42 +75,31 @@ def _rescue_search(provider_name: str, original_error: str, query: str, limit: i
# Ring also failed: the ORIGINAL error names the user's setup, so lead with it.
return {
"success": False,
"error": (
f"{original_error or 'search failed'} "
f"(keyless rescue also failed: {rescued.get('error', 'unknown')})"
),
"error": f"{original_error or 'search failed'} (keyless rescue also failed: {rescued.get('error', 'unknown')})",
}
def _policy_blocked_result(result: dict) -> bool:
"""True for a website-policy refusal — intentional, never rescued (it would fetch blocked content)."""
if result.get("blocked_by_policy"):
return True
return "blocked by website policy" in str(result.get("error") or "").lower()
return bool(result.get("blocked_by_policy")) or "blocked by website policy" in str(result.get("error") or "").lower()
def _rescue_extract(provider_name: str, urls: list, results: list) -> list:
"""Rescue a whole-batch extract failure via the ring.
Only genuine failures are re-fetched; policy-blocked entries are preserved
verbatim. If the provider broke url/result order parity, every entry is
treated as rescueable and the ring's list replaces the batch wholesale.
Only genuine failures are re-fetched; policy-blocked entries are preserved verbatim. If the
provider broke url/result order parity, every entry is treated as rescueable and the ring's
list replaces the batch wholesale.
"""
from plugins.web.keyless_mcp import extract_with_failover
parity = len(results) == len(urls)
if parity:
rescue_idx = [i for i, r in enumerate(results) if not _policy_blocked_result(r)]
else:
rescue_idx = list(range(len(results)))
rescue_idx = [i for i, r in enumerate(results) if not parity or not _policy_blocked_result(r)]
if not rescue_idx:
return results # every failure is an intentional policy block
rescue_urls = [urls[i] for i in rescue_idx] if parity else list(urls)
original_error = next(
(results[i].get("error") for i in rescue_idx if results[i].get("error")),
"extract failed",
)
original_error = next((results[i].get("error") for i in rescue_idx if results[i].get("error")), "extract failed")
logger.warning(
"web_extract backend '%s' failed all %d URL(s) (%s); one-shot keyless rescue",
provider_name, len(rescue_urls), (original_error or "")[:200],

63
tools/web_tools_truncate.py
View File

@@ -3,11 +3,9 @@
Pages at or under the char budget are returned whole; larger pages become a
head+tail window plus a footer that says how much is shown, where the full text
is stored (cache/web) and the exact read_file call that pages the omitted middle.
Inline base64 images are replaced with ``[IMAGE: alt]`` placeholders.
Extracted from tools/web_tools.py; the names are re-imported there so
``tools.web_tools.MAX_STORED_TEXT_CHARS`` etc. keep working. Logs under the
origin logger name for parity.
Inline base64 images are replaced with ``[IMAGE: alt]`` placeholders. Names are
re-imported by tools/web_tools.py (``tools.web_tools.MAX_STORED_TEXT_CHARS``); logs
under the origin logger name for parity.
"""
import logging
@@ -30,8 +28,7 @@ _CHAR_LIMIT_FLOOR, _CHAR_LIMIT_CEILING = 2000, 500_000
def _clamp_char_limit(value: Any) -> int:
"""Clamp to [2k, 500k]; raises TypeError/ValueError for non-numeric input.
Floor: below 2k the truncation footer dominates. Ceiling: a config typo
must not blow up context.
Floor: below 2k the truncation footer dominates. Ceiling: a config typo must not blow up context.
"""
return max(_CHAR_LIMIT_FLOOR, min(int(value), _CHAR_LIMIT_CEILING))
@@ -60,10 +57,7 @@ def convert_base64_images_to_links(text: str) -> str:
alt = (m.group("alt") or "").strip()
return f"[IMAGE: {alt}]" if alt else "[IMAGE]"
md_b64 = re.compile(
r"!\[(?P<alt>[^\]]*)\]\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)"
)
out = md_b64.sub(_md_repl, text)
out = re.sub(r"!\[(?P<alt>[^\]]*)\]\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)", _md_repl, text)
out = re.sub(r"\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)", "[IMAGE]", out)
out = re.sub(r"data:image/[^;]+;base64,[A-Za-z0-9+/=]+", "[IMAGE]", out)
return out
@@ -72,9 +66,9 @@ def convert_base64_images_to_links(text: str) -> str:
def _store_full_text(url: str, content: str) -> Optional[str]:
"""Write the full page to cache/web; absolute path or None.
cache/web is mounted read-only into remote backends (credential_files
_CACHE_DIRS) so read_file can page the complete text on any backend.
Best-effort: on failure the truncated content is still returned to the model.
cache/web is mounted read-only into remote backends (credential_files _CACHE_DIRS) so read_file
can page the complete text on any backend. Best-effort: on failure the truncated content is
still returned to the model.
"""
try:
import hashlib
@@ -94,9 +88,9 @@ def _store_full_text(url: str, content: str) -> Optional[str]:
)
from tools.spill_safety import write_text_exclusive
# Deterministic name in a well-known dir: refuse symlinks (lstat-unlink +
# exclusive create); same-URL re-extraction legitimately overwrites. Not
# private: cache/web is bind-mounted into remote backends' container UID.
# Deterministic name in a well-known dir: refuse symlinks (lstat-unlink + exclusive create);
# same-URL re-extraction legitimately overwrites. Not private: cache/web is bind-mounted
# into remote backends' container UID.
write_text_exclusive(path, content, private=False, overwrite=True)
return str(path)
except Exception as exc: # noqa: BLE001
@@ -104,16 +98,12 @@ def _store_full_text(url: str, content: str) -> Optional[str]:
return None
def _truncate_with_footer(
content: str,
url: str,
char_limit: int,
) -> tuple[str, bool]:
def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, bool]:
"""Return (model_text, was_truncated).
Pages over ``char_limit`` become a ~75% head / ~25% tail window cut on line
boundaries, plus a footer saying how much is shown, where the full text is
stored, and the read_file call that pages the omitted middle. Deterministic.
Pages over ``char_limit`` become a ~75% head / ~25% tail window cut on line boundaries, plus a
footer saying how much is shown, where the full text is stored, and the read_file call that
pages the omitted middle. Deterministic.
"""
if len(content) <= char_limit:
return content, False
@@ -131,14 +121,12 @@ def _truncate_with_footer(
if 0 <= nl < tail_budget * 0.5:
tail = tail[nl + 1:]
total = len(content)
stored_path = _store_full_text(url, content)
footer_lines = [
"",
"─" * 8 + " [TRUNCATED] " + "─" * 8,
f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) "
f"of {total:,} total clean characters.",
f"of {len(content):,} total clean characters.",
]
if stored_path:
# read_file is 1-indexed; +2 lands on the first line after the shown head.
@@ -155,10 +143,7 @@ def _truncate_with_footer(
"specific URL or use browser_navigate for the complete page."
)
footer_lines.append("─" * 29)
model_text = head + "\n\n[... middle omitted — see footer ...]\n\n" + tail
model_text += "\n" + "\n".join(footer_lines)
return model_text, True
return head + "\n\n[... middle omitted — see footer ...]\n\n" + tail + "\n" + "\n".join(footer_lines), True
def _effective_char_limit(char_limit: Optional[int]) -> int:
@@ -171,10 +156,8 @@ def _effective_char_limit(char_limit: Optional[int]) -> int:
def _truncate_results(results: List[dict], char_limit: int, debug_call_data: dict) -> None:
"""In place: replace each successful entry's content with its base64-cleaned, budgeted text.
Records per-page truncation metrics into ``debug_call_data``.
"""
"""In place: replace each successful entry's content with its base64-cleaned, budgeted text;
per-page truncation metrics go into ``debug_call_data``."""
for result in results:
if result.get("error"):
continue
@@ -187,11 +170,9 @@ def _truncate_results(results: List[dict], char_limit: int, debug_call_data: dic
result["content"] = model_text
if truncated:
debug_call_data["pages_truncated"] += 1
debug_call_data["truncation_metrics"].append({
"url": url,
"original_size": len(clean),
"sent_size": len(model_text),
})
debug_call_data["truncation_metrics"].append(
{"url": url, "original_size": len(clean), "sent_size": len(model_text)}
)
logger.info("%s (truncated %d -> %d chars)", url, len(clean), len(model_text))
else:
logger.info("%s (%d chars, whole)", url, len(clean))

61
tools/website_policy.py
View File

@@ -1,9 +1,9 @@
"""Website access policy helpers for URL-capable tools.
Loads a user-managed website blocklist (``security.website_blocklist`` in
~/.hermes/config.yaml plus optional shared list files) without pulling in the
heavier CLI config stack. The parsed policy is cached with a short TTL so config
edits take effect quickly without re-parsing YAML on every URL check.
Loads a user-managed website blocklist (``security.website_blocklist`` in ~/.hermes/config.yaml
plus optional shared list files) without pulling in the heavier CLI config stack. The parsed
policy is cached with a short TTL so config edits take effect quickly without re-parsing YAML on
every URL check.
"""
from __future__ import annotations
@@ -21,11 +21,7 @@ from tools.url_safety import _normalize_hostname as _normalize_host
logger = logging.getLogger(__name__)
_DEFAULT_WEBSITE_BLOCKLIST = {
"enabled": False,
"domains": [],
"shared_files": [],
}
_DEFAULT_WEBSITE_BLOCKLIST = {"enabled": False, "domains": [], "shared_files": []}
# Without this cache a 50-URL extract would mean 51 YAML parses of config.yaml.
_CACHE_TTL_SECONDS = 30.0
@@ -60,11 +56,8 @@ def _normalize_rule(rule: Any) -> Optional[str]:
def _iter_blocklist_file_rules(path: Path) -> List[str]:
"""Load rules from a shared blocklist file.
Missing or unreadable files warn and yield nothing rather than raising — a bad
file path must not disable all web tools.
"""
"""Rules from a shared blocklist file; missing/unreadable files warn and yield nothing
rather than raising — a bad file path must not disable all web tools."""
try:
raw = path.read_text(encoding="utf-8")
except FileNotFoundError:
@@ -85,11 +78,9 @@ def _require_mapping(value: Any, label: str) -> Dict[str, Any]:
return value
def _load_policy_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
config_path = config_path or _get_default_config_path()
def _load_policy_config(config_path: Path) -> Dict[str, Any]:
if not config_path.exists():
return dict(_DEFAULT_WEBSITE_BLOCKLIST)
try:
import yaml
except ImportError:
@@ -107,13 +98,8 @@ def _load_policy_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
raise WebsitePolicyError("config root must be a mapping")
security = _require_mapping(config.get("security", {}), "security")
website_blocklist = _require_mapping(
security.get("website_blocklist", {}), "security.website_blocklist"
)
policy = dict(_DEFAULT_WEBSITE_BLOCKLIST)
policy.update(website_blocklist)
return policy
website_blocklist = _require_mapping(security.get("website_blocklist", {}), "security.website_blocklist")
return {**_DEFAULT_WEBSITE_BLOCKLIST, **website_blocklist}
def _require_type(policy: Dict[str, Any], key: str, kind: type, default: Any) -> Any:
@@ -129,13 +115,13 @@ def _require_type(policy: Dict[str, Any], key: str, kind: type, default: Any) ->
def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]:
"""Load and return the parsed website blocklist policy (``{"enabled", "rules"}``).
Cached for ``_CACHE_TTL_SECONDS`` for the default config path only; an
explicit ``config_path`` (tests) always bypasses and never populates the cache.
Cached for ``_CACHE_TTL_SECONDS`` for the default config path only; an explicit
``config_path`` (tests) always bypasses and never populates the cache.
"""
global _cached_policy, _cached_policy_path, _cached_policy_time
default_path = _get_default_config_path()
resolved_path = str(config_path) if config_path else str(default_path)
resolved_path = str(config_path or default_path)
now = time.monotonic()
if config_path is None:
@@ -149,7 +135,6 @@ def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]
config_path = config_path or default_path
policy = _load_policy_config(config_path)
raw_domains = _require_type(policy, "domains", list, [])
raw_shared_files = _require_type(policy, "shared_files", list, [])
enabled = _require_type(policy, "enabled", bool, True)
@@ -164,7 +149,6 @@ def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]
for raw_rule in raw_domains:
_add(_normalize_rule(raw_rule), "config")
for shared_file in raw_shared_files:
if not isinstance(shared_file, str) or not shared_file.strip():
continue
@@ -175,13 +159,9 @@ def load_website_blocklist(config_path: Optional[Path] = None) -> Dict[str, Any]
_add(normalized, str(path))
result = {"enabled": enabled, "rules": rules}
if config_path == default_path: # explicit paths are tests — never cache them
with _cache_lock:
_cached_policy = result
_cached_policy_path = resolved_path
_cached_policy_time = now
_cached_policy, _cached_policy_path, _cached_policy_time = result, resolved_path, now
return result
@@ -207,10 +187,9 @@ def _extract_host_from_urlish(url: str) -> str:
def check_website_access(url: str, config_path: Optional[Path] = None) -> Optional[Dict[str, str]]:
"""Check whether a URL is allowed by the website blocklist policy.
Returns ``None`` if allowed, else block metadata (``host``, ``rule``,
``source``, ``message``). Fails open on policy errors (warn + ``None``) so a
config typo can't break all web tools — except with an explicit ``config_path``
(tests), where errors propagate.
Returns ``None`` if allowed, else block metadata (``host``, ``rule``, ``source``, ``message``).
Fails open on policy errors (warn + ``None``) so a config typo can't break all web tools —
except with an explicit ``config_path`` (tests), where errors propagate.
"""
# Fast path: cached policy disabled/empty → no YAML read, no host extraction.
if config_path is None:
@@ -235,7 +214,6 @@ def check_website_access(url: str, config_path: Optional[Path] = None) -> Option
if not policy.get("enabled"):
return None
for rule in policy.get("rules", []):
pattern = rule.get("pattern", "")
if _match_host_against_rule(host, pattern):
@@ -246,9 +224,6 @@ def check_website_access(url: str, config_path: Optional[Path] = None) -> Option
"host": host,
"rule": pattern,
"source": source,
"message": (
f"Blocked by website policy: '{host}' matched rule '{pattern}'"
f" from {source}"
),
"message": f"Blocked by website policy: '{host}' matched rule '{pattern}' from {source}",
}
return None