From f7cfecd4e579ddb4091e39217f4ec3640a56aaf9 Mon Sep 17 00:00:00 2001 From: Siddharth Balyan <52913345+alt-glitch@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:59:32 +0530 Subject: [PATCH] Web search: keyless search moves to the next free vendor on 402, 5xx and Keenable refusals (#134048) * fix(web): keyless search fails over when a free tier refuses the client Firecrawl's anonymous endpoint answers 403 (an IP reputation gate) for some clients, and a free tier can also answer 402 (credits gone) or a 5xx. Main already moves a keyless search to the next ring vendor on a structured 401/403 (2f78707a79, from #91619), but two cases still stopped the walk: - Keenable's keyless client raised only the response body, without the status code, so its 401/403 never matched the structured status check. - The keyless Firecrawl client raised httpx's default "Server error '503 ...'" text for 5xx, and no status pattern covered 402 or 5xx. The status pattern now covers 401/402/403 and 5xx, and both keyless clients raise "HTTP : ", so the walk sees the status and the error keeps the vendor's reason. Extract keeps main's rule: it fails over only when every URL is rate-limited, because a 403 there can come from the page rather than the vendor. * fix(web): a search error that echoes the query no longer walks the keyless ring (review) The vendor-refusal status match now judges only the status the error starts with, so 'HTTP 400: invalid query "http 503"' stops the walk instead of trying every free vendor. --- plugins/web/firecrawl/provider.py | 6 +++- plugins/web/keyless_mcp.py | 19 +++++++---- tests/tools/test_web_keyless_fallback.py | 33 +++++++++++++++++++ website/docs/user-guide/configuration.md | 2 +- .../docs/user-guide/features/web-search.md | 4 +-- 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/plugins/web/firecrawl/provider.py b/plugins/web/firecrawl/provider.py index 12d6286a83..948c529a12 100644 --- a/plugins/web/firecrawl/provider.py +++ b/plugins/web/firecrawl/provider.py @@ -118,7 +118,11 @@ class _KeylessFirecrawlClient: def _post(self, path: str, payload: Dict[str, Any]) -> Dict[str, Any]: response = httpx.post(f"{self.api_url}{path}", json=payload, headers={"Content-Type": "application/json"}, timeout=60.0) - response.raise_for_status() + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + # Keep the vendor's reason (e.g. "your IP address looks suspicious") and the ring's ``HTTP `` shape. + raise httpx.HTTPStatusError(f"HTTP {response.status_code}: {response.text.strip()[:300]}", request=exc.request, response=response) from exc return response.json() search = lambda self, *, query, limit=5: self._post("/v2/search", {"query": query, "limit": limit}) # noqa: E731 diff --git a/plugins/web/keyless_mcp.py b/plugins/web/keyless_mcp.py index 7c2f721e34..f9c1f351ed 100644 --- a/plugins/web/keyless_mcp.py +++ b/plugins/web/keyless_mcp.py @@ -47,9 +47,14 @@ def _is_rate_limitish(message: str) -> bool: return any(marker in (message or "").lower() for marker in _RATE_LIMIT_MARKERS) -_AUTH_STATUS_RE = re.compile( - r"\b(?:http(?:\s+status)?|status(?:\s+code)?|client\s+error|error(?:\s+code)?)" - r"\s*[:=']*\s*(?:401|403)\b", +# A free tier that refuses this client (401/402/403: anonymous access revoked, IP reputation gate) +# or errors server-side (5xx) fails every query from here, while another vendor may still serve it. +# Anchored to the status the error starts with (after the ``Keyless search failed:`` prefix), +# so a terminal error that echoes the query ("HTTP 400: invalid query 'http 503'") does not match. +_VENDOR_REFUSAL_RE = re.compile( + r"\s*(?:keyless\s+\w+\s+search\s+failed:\s*)?" + r"(?:http(?:\s+status)?|status(?:\s+code)?|client\s+error|error\s+code)" + r"\s*[:=']*\s*(?:40[123]|5\d\d)\b", re.IGNORECASE, ) @@ -57,11 +62,11 @@ _AUTH_STATUS_RE = re.compile( def _is_search_failover_eligible(message: str) -> bool: """Return whether another anonymous vendor may serve the search. - Rate limits and structured HTTP 401/403 provider rejections are local to + Rate limits and structured HTTP 401/402/403/5xx vendor refusals are local to one free search endpoint. Free-text markers are deliberately ignored: vendors may echo the query in an otherwise terminal error. """ - return _is_rate_limitish(message) or bool(_AUTH_STATUS_RE.search(message or "")) + return _is_rate_limitish(message) or bool(_VENDOR_REFUSAL_RE.match(message or "")) def _fail_msg(vendor: str, kind: str, exc: Any, *, other_backends: bool = True) -> str: @@ -308,7 +313,7 @@ def _keenable_request(method: str, path: str, **kwargs: Any) -> Dict[str, Any]: headers["Content-Type"] = "application/json" response = getattr(requests, method)(f"{KEENABLE_API_URL}{path}", headers=headers, timeout=_TIMEOUT_SECONDS, **kwargs) if response.status_code >= 400: - raise KeylessMCPError(_response_text(response).strip() or f"HTTP {response.status_code}") + raise KeylessMCPError(f"HTTP {response.status_code}: {_response_text(response).strip()[:300]}") return response.json() @@ -391,7 +396,7 @@ def _walk_ring(name: str, kind: str, call, throttled) -> tuple: def search_with_failover(name: str, query: str, limit: int = 5) -> Dict[str, Any]: - """Rate limits and anonymous-endpoint auth/policy rejections advance to the next + """Rate limits and vendor refusals (HTTP 401/402/403/5xx) advance to the next vendor, other errors stop the walk (a malformed query fails everywhere). ``data.served_by`` is set when the serving vendor differs from *name*.""" diff --git a/tests/tools/test_web_keyless_fallback.py b/tests/tools/test_web_keyless_fallback.py index f20191ff5e..9e83f398fd 100644 --- a/tests/tools/test_web_keyless_fallback.py +++ b/tests/tools/test_web_keyless_fallback.py @@ -148,6 +148,29 @@ class TestParseMcpBody: else: keyless_mcp._keenable_request("get", "/v1/search/public") + def test_keenable_error_keeps_status_and_reason(self): + import requests + + response = requests.Response() + response.status_code = 403 + response.headers["Content-Type"] = "text/plain; charset=utf-8" + response._content = b"anonymous access disabled" + with patch.object(requests, "post", return_value=response): + with pytest.raises(keyless_mcp.KeylessMCPError, match="^HTTP 403: anonymous access disabled$"): + keyless_mcp._keenable_request("post", "/v1/search/public", json={"query": "q"}) + + def test_keyless_firecrawl_error_keeps_status_and_reason(self, monkeypatch): + import httpx + + from plugins.web.firecrawl import provider as firecrawl_provider + + def _forbidden(url, **_kwargs): + return httpx.Response(403, text="your IP address looks suspicious\n", request=httpx.Request("POST", url)) + + monkeypatch.setattr(firecrawl_provider.httpx, "post", _forbidden) + with pytest.raises(httpx.HTTPStatusError, match="^HTTP 403: your IP address looks suspicious$"): + firecrawl_provider._KeylessFirecrawlClient().search(query="q", limit=1) + class TestExaTextParsing: def test_parses_blocks(self): @@ -521,6 +544,16 @@ class TestKeylessFailover: ("FORBIDDEN", False), ("extract failed for 'forbidden kingdom trailer': connection timeout", False), ("HTTP 400: malformed request", False), + ("HTTP 402: free credits exhausted", True), + ("HTTP 503: upstream unavailable", True), + ("HTTP 404: not found", False), + ("Keyless Firecrawl search failed: HTTP 403: your IP address looks suspicious", True), + # A terminal error that echoes the query is judged by the status it starts with. + ("HTTP 400: invalid query 'http 503'", False), + ("HTTP 422: query contains 'status=500'", False), + ("search failed for 'HTTP 503 error codes': timeout", False), + ("error: 500 results max exceeded", False), + ("Keyless Exa search failed: HTTP 400: invalid query 'http 403'", False), ("", False), ], ) diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 57a0dd2330..7873d8083e 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -2749,7 +2749,7 @@ web: | **Perplexity** | `PERPLEXITY_API_KEY` | ✔ | ✔ (query-relevant snippets) | | **Exa** | `EXA_API_KEY` (optional — keyless free tier) | ✔ | ✔ | -**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PERPLEXITY_API_KEY` is set, Perplexity; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits — see the [Web Search guide](./features/web-search.md) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key. +**Backend selection:** The runtime always uses the stored `web.backend` selection (set via `hermes tools`; `nous` routes through the managed Tool Gateway). Only if no web backend has ever been selected is one auto-detected from available API keys: if only `SEARXNG_URL` is set, SearXNG is used; if only `EXA_API_KEY` is set, Exa; if only `TAVILY_API_KEY` is set, Tavily; if only `PERPLEXITY_API_KEY` is set, Perplexity; if only `PARALLEL_API_KEY` is set, Parallel; if only `KEENABLE_API_KEY` is set, Keenable. With **no selection and no credentials at all**, requests rotate round-robin across the keyless free-tier ring (Exa / Parallel / Firecrawl / Keenable) with automatic next-in-line failover on rate limits (and, for search, on vendor refusals) — see the [Web Search guide](./features/web-search.md) for details. Once a selection exists, adding a key to `.env` does not change the route. Selecting Tavily, Firecrawl, or Keenable in `hermes tools` also works without a key. **SearXNG** is a free, self-hosted, privacy-respecting metasearch engine that queries 70+ search engines. No API key needed — just set `SEARXNG_URL` to your instance (e.g., `http://localhost:8080`). SearXNG is search-only; `web_extract` requires a separate extract provider (set `web.extract_backend`). See the [Web Search setup guide](./features/web-search.md) for Docker setup instructions. diff --git a/website/docs/user-guide/features/web-search.md b/website/docs/user-guide/features/web-search.md index c8125c220b..adfb43dfc7 100644 --- a/website/docs/user-guide/features/web-search.md +++ b/website/docs/user-guide/features/web-search.md @@ -39,7 +39,7 @@ Brave Search, DDGS, xAI, and OpenAI Native are **search-only** — pair any of t **Per-capability split:** you can use different providers for search and extract independently — for example SearXNG (free) for search and Firecrawl for extract. See [Per-capability configuration](#per-capability-configuration) below. :::info Works out of the box — keyless free-tier rotation -A fresh install with **no web credentials at all** gets working `web_search` and `web_extract` out of the box: requests rotate round-robin across the ring vendors' public free tiers — **Exa, Parallel, Firecrawl, and Keenable** — spreading load evenly, and a rate-limited request automatically retries on the next vendor in the ring (multi-hop, until one serves or all are throttled). No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For guaranteed, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. +A fresh install with **no web credentials at all** gets working `web_search` and `web_extract` out of the box: requests rotate round-robin across the ring vendors' public free tiers — **Exa, Parallel, Firecrawl, and Keenable** — spreading load evenly, and a rate-limited request automatically retries on the next vendor in the ring (multi-hop, until one serves or all fail). A search that a vendor refuses (a 401/402/403 access gate or a 5xx) retries the same way. No signup, no key. This tier is strictly last-resort — any configured backend or present API key always wins — and requests carry no user identifiers (only a random per-process session id, rotated on restart). For guaranteed, unthrottled service, set up a keyed provider. Disable the keyless tier entirely with `web.keyless_fallback: false`. ::: **Choosing free vs paid explicitly:** in `hermes tools`, Exa, Parallel, and Keenable each appear as two rows — **Free (keyless)** and **Paid (API key)**. Picking Free pins that vendor's anonymous endpoint (even if you later add a key); picking Paid pins the keyed path (a missing key then errors instead of silently downgrading to the free tier). The selection is stored as `web.provider_tier.: free|paid`; leave it unset for auto (key present → paid, otherwise the keyless ring). @@ -448,7 +448,7 @@ If no shared backend has **ever** been selected (no `web.backend` written by you | Nous identity present (any account, including the free tier and anonymous guests) | managed web search (`web_search` only); extract and the keyless ring below are unchanged | | *(nothing set at all)* | keyless ring: exa / parallel / firecrawl / keenable (round-robin) | -**Keyless free-tier ring:** when *no* credential above is present, requests rotate across the ring vendors' public free tiers (Exa, Parallel, Firecrawl, Keenable) so web tools work on a fresh install with zero setup — and a rate-limited request fails over to the next vendor in the ring automatically. Pin one vendor in `hermes tools` to stop the rotation (the ring is then only used as failover succession on throttles). All free tiers are vendor-rate-limited under burst load; sustained normal usage goes through fine. Set `web.keyless_fallback: false` to turn the tier off — with it off and no credentials, web tools are unavailable until a provider is configured. A Nous identity (guest included) is unaffected by that switch: managed `web_search` still applies, though `web_extract` needs a configured provider. +**Keyless free-tier ring:** when *no* credential above is present, requests rotate across the ring vendors' public free tiers (Exa, Parallel, Firecrawl, Keenable) so web tools work on a fresh install with zero setup — and a rate-limited request, or a search the vendor refuses, fails over to the next vendor in the ring automatically. Pin one vendor in `hermes tools` to stop the rotation (the ring is then only used as failover succession on throttles). All free tiers are vendor-rate-limited under burst load; sustained normal usage goes through fine. Set `web.keyless_fallback: false` to turn the tier off — with it off and no credentials, web tools are unavailable until a provider is configured. A Nous identity (guest included) is unaffected by that switch: managed `web_search` still applies, though `web_extract` needs a configured provider. **One-shot keyless rescue for keyed backends:** when your chosen/keyed backend — including the Nous Tool Gateway route (`web.backend: nous`) — fails a call (bad key, outage, unreachable gateway, upstream 5xx), that single call automatically retries on the keyless free-tier ring instead of erroring — the result notes which vendor served it and why (`rescued_from` / `backend_error`). The failover is never sticky: the very next `web_search`/`web_extract` call attempts your chosen backend again. Disable with `web.keyless_rescue: false` (also off whenever `keyless_fallback` is off).