"""Plugin-side tests for the web search provider migration (PR #25182). Covers: - All bundled plugins (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl, keenable, xai) instantiate and self-report the expected capabilities + ABC-derived defaults. - Each plugin's ``is_available()`` correctly reflects env-var presence. - The web_search_registry resolves an active provider in the documented scenarios (explicit config wins ignoring availability, fallback walks legacy preference filtered by availability, unknown name falls back). - Plugin response shapes match the legacy bit-for-bit contract. Per the dev skill: these tests use *real* imports from the plugin modules — no mocking of provider classes themselves — so the test catches drift in the ABC interface, the registry, and the plugin glue layer simultaneously. """ from __future__ import annotations import asyncio import pytest # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None: """Strip every web-provider env var so is_available() returns False.""" for k in ( "BRAVE_SEARCH_API_KEY", "SEARXNG_URL", "KEENABLE_API_KEY", "TAVILY_API_KEY", "TAVILY_BASE_URL", "EXA_API_KEY", "PARALLEL_API_KEY", "PARALLEL_SEARCH_MODE", "FIRECRAWL_API_KEY", "FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN", "XAI_API_KEY", ): monkeypatch.delenv(k, raising=False) def _ensure_plugins_loaded() -> None: """Idempotently load plugins so the registry is populated.""" from hermes_cli.plugins import _ensure_plugins_discovered _ensure_plugins_discovered() # --------------------------------------------------------------------------- # Per-plugin discovery + capability flags # --------------------------------------------------------------------------- @pytest.fixture(autouse=True) def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: """Each test starts with a clean web-provider env.""" _clear_web_env(monkeypatch) # --------------------------------------------------------------------------- # is_available() behavior # --------------------------------------------------------------------------- class TestIsAvailable: """Each plugin's ``is_available()`` returns False without env config.""" def test_brave_free_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("brave-free") assert p is not None assert p.is_available() is False # no BRAVE_SEARCH_API_KEY monkeypatch.setenv("BRAVE_SEARCH_API_KEY", "real") assert p.is_available() is True def test_searxng_requires_url(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("searxng") assert p is not None assert p.is_available() is False monkeypatch.setenv("SEARXNG_URL", "http://localhost:8080") assert p.is_available() is True def test_keenable_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("keenable") assert p is not None assert p.is_available() is False monkeypatch.setenv("KEENABLE_API_KEY", "real") assert p.is_available() is True def test_tavily_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("tavily") assert p is not None assert p.is_available() is False monkeypatch.setenv("TAVILY_API_KEY", "real") assert p.is_available() is True def test_exa_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("exa") assert p is not None assert p.is_available() is False monkeypatch.setenv("EXA_API_KEY", "real") assert p.is_available() is True def test_parallel_requires_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("parallel") assert p is not None assert p.is_available() is False monkeypatch.setenv("PARALLEL_API_KEY", "real") assert p.is_available() is True def test_firecrawl_requires_either_key_or_url( self, monkeypatch: pytest.MonkeyPatch ) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("firecrawl") assert p is not None assert p.is_available() is False # Either FIRECRAWL_API_KEY or FIRECRAWL_API_URL lights it up. monkeypatch.setenv("FIRECRAWL_API_KEY", "real") assert p.is_available() is True monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False) monkeypatch.setenv("FIRECRAWL_API_URL", "http://localhost:3002") assert p.is_available() is True def test_firecrawl_explicit_config_allows_keyless_cloud( self, monkeypatch: pytest.MonkeyPatch ) -> None: _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("firecrawl") assert p is not None assert p.is_available() is False monkeypatch.setattr( "tools.web_tools._load_web_config", lambda: {"backend": "firecrawl"}, raising=False, ) assert p.is_available() is True def test_xai_requires_api_key_or_oauth(self, monkeypatch: pytest.MonkeyPatch) -> None: """xAI needs XAI_API_KEY or OAuth tokens in auth.json.""" _ensure_plugins_loaded() from agent.web_search_registry import get_provider p = get_provider("xai") assert p is not None assert p.is_available() is False # no XAI_API_KEY, no auth.json monkeypatch.setenv("XAI_API_KEY", "real") assert p.is_available() is True # --------------------------------------------------------------------------- # Registry resolution semantics (Option B — conservative smart fallback) # --------------------------------------------------------------------------- class TestRegistryResolution: """``_resolve()`` follows explicit-config + availability-filtered fallback.""" def test_explicit_configured_provider_returned_even_when_unavailable( self, ) -> None: """Explicit ``web.search_backend`` wins regardless of is_available(). Without availability filtering on the explicit path, the dispatcher would silently switch backends; with this check the dispatcher surfaces a precise "FOO_API_KEY is not set" error instead. """ _ensure_plugins_loaded() from agent.web_search_registry import _resolve # No BRAVE_SEARCH_API_KEY (fixture cleared it). result = _resolve("brave-free", capability="search") assert result is not None assert result.name == "brave-free" # Confirm it's the unavailable one — dispatcher will surface # a typed credential-missing error to the caller. assert result.is_available() is False def test_unknown_configured_name_falls_back_to_available_provider( self, monkeypatch: pytest.MonkeyPatch ) -> None: """Typo / uninstalled plugin → walk legacy preference, pick available.""" _ensure_plugins_loaded() from agent.web_search_registry import _resolve monkeypatch.setenv("EXA_API_KEY", "real") result = _resolve("not-a-real-provider", capability="search") # Either ddgs (no-key fallback) or exa (the only available # premium provider) — both are valid. The point is the unknown # name shouldn't return None when SOMETHING is available. assert result is not None assert result.is_available() is True def test_no_config_no_credentials_returns_none( self, ) -> None: """No backend configured AND no credentials → keyless tier or ddgs. Resolution order with zero credentials: ddgs if its Python package is importable, else the keyless free tier (Parallel/Exa public endpoints — resolves with ``is_available() == False`` but ``is_keyless_available() == True``), else None (keyless tier disabled). All three outcomes are correct; a provider that is neither keyed nor keyless-capable means an env var leaked in. """ _ensure_plugins_loaded() from agent.web_search_registry import _resolve result = _resolve(None, capability="search") if result is not None: assert result.is_available() or result.is_keyless_available() # --------------------------------------------------------------------------- # Firecrawl timeout alignment (#43272) # --------------------------------------------------------------------------- class TestFirecrawlScrapeTimeout: """Verify the server-side scrape timeout matches the asyncio deadline.""" def test_extract_passes_timeout_ms_to_scrape(self, monkeypatch: pytest.MonkeyPatch) -> None: """scrape() must receive timeout=60000 (ms) so the server-side deadline matches the 60 s asyncio.wait_for deadline. Without this the API uses its 30 s default, causing SCRAPE_TIMEOUT.""" from unittest.mock import MagicMock, patch from plugins.web.firecrawl import provider as firecrawl_provider # Direct-credential mode: with no FIRECRAWL_API_KEY extract() routes # to the keyless ring instead of the direct scrape path under test. monkeypatch.setenv("FIRECRAWL_API_KEY", "test-key") # Set up a fake client whose scrape() returns a minimal response fake_client = MagicMock() fake_client.scrape.return_value = { "metadata": {"title": "Test", "sourceURL": "https://example.com"}, "markdown": "content", } # Patch _get_firecrawl_client to return our fake client monkeypatch.setattr( firecrawl_provider, "_get_firecrawl_client", lambda capability=None: fake_client ) # Patch check_website_access to allow the URL monkeypatch.setattr( firecrawl_provider, "check_website_access", lambda url: None ) p = firecrawl_provider.FirecrawlWebSearchProvider() results = asyncio.run(p.extract(["https://example.com"])) # Verify scrape was called with timeout=60000 fake_client.scrape.assert_called_once() call_kwargs = fake_client.scrape.call_args assert call_kwargs.kwargs.get("timeout") == 60_000, ( f"Expected timeout=60000, got {call_kwargs.kwargs.get('timeout')}" ) # Verify the result was processed correctly assert len(results) == 1 assert results[0]["url"] == "https://example.com" assert results[0]["content"] == "content"