From 89e75f4770705f12dabfb61bd713c8fbceb7643e Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:24:46 -0700 Subject: [PATCH] test(tools): pin strict provider-string selection per category New tests/tools/test_strict_provider_selection.py covers read_selection semantics (legacy use_gateway interpretation, seeded stt local, empty strings, browser.backend vs cloud_provider) and the three strict behaviors per category: managed 'nous' selection wins over present direct keys, a vendor selection with missing credentials raises the selection-naming error with NO managed call, and never-configured installs keep today's autodetect. Updated the tests that pinned the old credential-first precedence (TTS resolver gateway override, STT silent managed fallback, web invalid-backend reroute, video_gen picker writes). Sabotage-verified: reverting the image FAL strict switch makes the new managed-selection tests fail. --- tests/hermes_cli/test_nous_subscription.py | 9 +- tests/hermes_cli/test_tools_config.py | 4 +- tests/tools/test_managed_media_gateways.py | 6 +- tests/tools/test_strict_provider_selection.py | 357 ++++++++++++++++++ tests/tools/test_transcription_tools.py | 14 + tests/tools/test_tts_openai_config.py | 38 +- tests/tools/test_web_tools_config.py | 23 +- 7 files changed, 434 insertions(+), 17 deletions(-) create mode 100644 tests/tools/test_strict_provider_selection.py diff --git a/tests/hermes_cli/test_nous_subscription.py b/tests/hermes_cli/test_nous_subscription.py index b71a0fb284..73f680d3d1 100644 --- a/tests/hermes_cli/test_nous_subscription.py +++ b/tests/hermes_cli/test_nous_subscription.py @@ -150,9 +150,8 @@ def test_prompt_enable_tool_gateway_pool_offers_covered_tools_only(monkeypatch): def test_apply_nous_managed_defaults_writes_video_gen_config(monkeypatch): - """apply_nous_managed_defaults must write video_gen.provider and - video_gen.use_gateway when a Nous subscriber selects video_gen - without a direct FAL_KEY.""" + """apply_nous_managed_defaults must store the managed 'nous' selection + when a Nous subscriber selects video_gen without a direct FAL_KEY.""" monkeypatch.setattr(ns, "managed_nous_tools_enabled", lambda **kw: True) monkeypatch.delenv("FAL_KEY", raising=False) monkeypatch.setattr(ns, "fal_key_is_configured", lambda: False) @@ -167,8 +166,8 @@ def test_apply_nous_managed_defaults_writes_video_gen_config(monkeypatch): ) assert "video_gen" in changed - assert config["video_gen"]["provider"] == "fal" - assert config["video_gen"]["use_gateway"] is True + assert config["video_gen"]["provider"] == "nous" + assert "use_gateway" not in config["video_gen"] # --------------------------------------------------------------------------- diff --git a/tests/hermes_cli/test_tools_config.py b/tests/hermes_cli/test_tools_config.py index f978a490c0..3212eb6146 100644 --- a/tests/hermes_cli/test_tools_config.py +++ b/tests/hermes_cli/test_tools_config.py @@ -256,8 +256,8 @@ def test_first_install_nous_auto_configures_video_gen(monkeypatch): tools_command(first_install=True, config=config) - assert config["video_gen"]["provider"] == "fal" - assert config["video_gen"]["use_gateway"] is True + assert config["video_gen"]["provider"] == "nous" + assert "use_gateway" not in config["video_gen"] # video_gen should NOT appear in the manual configure list — it's auto-configured assert "video_gen" not in configured diff --git a/tests/tools/test_managed_media_gateways.py b/tests/tools/test_managed_media_gateways.py index 01343140b6..fa73128b6a 100644 --- a/tests/tools/test_managed_media_gateways.py +++ b/tests/tools/test_managed_media_gateways.py @@ -247,7 +247,9 @@ def test_transcription_uses_model_specific_response_formats(monkeypatch, tmp_pat _install_fake_tools_package() _install_fake_openai_module(whisper_capture, transcription_response="hello from whisper") monkeypatch.setenv("HERMES_HOME", str(tmp_path)) - (tmp_path / "config.yaml").write_text("stt:\n provider: openai\n") + # The managed audio route is the stored "nous" selection (strict model); + # a stored "openai" selection now means direct credentials only. + (tmp_path / "config.yaml").write_text("stt:\n provider: nous\n") monkeypatch.delenv("VOICE_TOOLS_OPENAI_KEY", raising=False) monkeypatch.delenv("OPENAI_API_KEY", raising=False) monkeypatch.setenv("TOOL_GATEWAY_DOMAIN", "nousresearch.com") @@ -257,7 +259,7 @@ def test_transcription_uses_model_specific_response_formats(monkeypatch, tmp_pat "tools.transcription_tools", "transcription_tools.py", ) - transcription_tools._load_stt_config = lambda: {"provider": "openai"} + transcription_tools._load_stt_config = lambda: {"provider": "nous"} audio_path = tmp_path / "audio.wav" audio_path.write_bytes(b"RIFF0000WAVEfmt ") diff --git a/tests/tools/test_strict_provider_selection.py b/tests/tools/test_strict_provider_selection.py new file mode 100644 index 0000000000..cbb158e47a --- /dev/null +++ b/tests/tools/test_strict_provider_selection.py @@ -0,0 +1,357 @@ +"""Strict tool-provider selection: the `hermes tools` choice always wins. + +Policy (owner decision): the provider string stored in config.yaml is what +runs at call time. "nous" → managed Nous Tool Gateway only; a vendor name → +that vendor direct with the user's own credentials; no key ever written → +today's credential autodetect. Credential presence must NEVER select or +reroute; a selected-but-broken provider produces an honest error naming the +selection and pointing at `hermes tools`. + +Per category these tests pin the three strict behaviors: + (a) managed selection + direct key present ⇒ managed route (key ignored) + (b) vendor selection + key missing ⇒ selection-naming error, NO managed call + (c) never-configured ⇒ legacy autodetect unchanged +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from tools import tool_backend_helpers as tbh + + +MANAGED = SimpleNamespace( + nous_user_token="managed-token", + gateway_origin="https://gateway.nousresearch.com", +) + + +# --------------------------------------------------------------------------- +# read_selection — the shared helper +# --------------------------------------------------------------------------- + + +class TestReadSelection: + def _with_raw(self, raw): + return patch( + "hermes_cli.config.read_raw_config_readonly", + return_value=raw, + ) + + def test_never_configured_returns_none(self): + with self._with_raw({}): + assert tbh.read_selection("image_gen") is None + + def test_vendor_provider_returned(self): + with self._with_raw({"image_gen": {"provider": "fal"}}): + assert tbh.read_selection("image_gen") == "fal" + + def test_nous_provider_returned(self): + with self._with_raw({"image_gen": {"provider": "nous"}}): + assert tbh.read_selection("image_gen") == "nous" + + def test_legacy_use_gateway_true_maps_to_nous(self): + """Old configs stored use_gateway: true beside a vendor name — only + the managed picker row ever wrote it, so it means 'nous'.""" + with self._with_raw({"video_gen": {"provider": "fal", "use_gateway": True}}): + assert tbh.read_selection("video_gen") == "nous" + + def test_legacy_use_gateway_false_keeps_vendor(self): + with self._with_raw({"tts": {"provider": "openai", "use_gateway": False}}): + assert tbh.read_selection("tts") == "openai" + + def test_empty_string_backend_is_no_selection(self): + """DEFAULT_CONFIG's seeded empty strings are not selections.""" + with self._with_raw({"web": {"backend": ""}}): + assert tbh.read_selection("web") is None + + def test_seeded_stt_local_is_no_selection(self): + """Legacy DEFAULT_CONFIG seeded stt.provider: local on every + install; that value alone must be treated as never-configured.""" + with self._with_raw({"stt": {"provider": "local"}}): + assert tbh.read_selection("stt") is None + + def test_stt_local_with_use_gateway_key_is_a_selection(self): + """A picker-written stt section (use_gateway key present) means + local was a genuine choice.""" + with self._with_raw({"stt": {"provider": "local", "use_gateway": False}}): + assert tbh.read_selection("stt") == "local" + + def test_browser_backend_key_is_not_the_cloud_selection(self): + """browser.backend is the driver choice (browser-use CLI vs built-in + tools), not the cloud provider selection.""" + with self._with_raw({"browser": {"backend": "browser-use"}}): + assert tbh.read_selection("browser") is None + + def test_web_per_capability_keys_mark_configured(self): + with self._with_raw({"web": {"search_backend": "searxng"}}): + assert tbh.read_selection("web") is None + assert tbh.selection_exists("web") is True + + +# --------------------------------------------------------------------------- +# Image generation (FAL) +# --------------------------------------------------------------------------- + + +class TestImageFalStrictSelection: + def test_nous_selection_routes_managed_even_with_fal_key(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value="nous"), \ + patch.object(it, "fal_key_is_configured", return_value=True), \ + patch.object(it, "resolve_managed_tool_gateway", return_value=MANAGED) as gw: + assert it._resolve_managed_fal_gateway() is MANAGED + gw.assert_called_once_with("fal-queue") + + def test_nous_selection_unentitled_raises_selection_error(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value="nous"), \ + patch.object(it, "fal_key_is_configured", return_value=True), \ + patch.object(it, "resolve_managed_tool_gateway", return_value=None): + with pytest.raises(ValueError) as exc: + it._resolve_managed_fal_gateway() + assert "image_gen is configured to use nous" in str(exc.value) + assert "hermes tools" in str(exc.value) + + def test_fal_selection_missing_key_errors_without_managed_call(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value="fal"), \ + patch.object(it, "fal_key_is_configured", return_value=False), \ + patch.object(it, "resolve_managed_tool_gateway") as gw: + with pytest.raises(ValueError) as exc: + it._resolve_managed_fal_gateway() + gw.assert_not_called() + assert "FAL_KEY" in str(exc.value) + assert "image_gen is configured to use fal" in str(exc.value) + assert "hermes tools" in str(exc.value) + + def test_fal_selection_with_key_routes_direct(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value="fal"), \ + patch.object(it, "fal_key_is_configured", return_value=True), \ + patch.object(it, "resolve_managed_tool_gateway") as gw: + assert it._resolve_managed_fal_gateway() is None + gw.assert_not_called() + + def test_never_configured_autodetect_direct_when_key_present(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value=None), \ + patch.object(it, "fal_key_is_configured", return_value=True): + assert it._resolve_managed_fal_gateway() is None + + def test_never_configured_autodetect_managed_when_no_key(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value=None), \ + patch.object(it, "fal_key_is_configured", return_value=False), \ + patch.object(it, "resolve_managed_tool_gateway", return_value=MANAGED): + assert it._resolve_managed_fal_gateway() is MANAGED + + def test_check_fal_api_key_reflects_selection(self): + from tools import image_generation_tool as it + + with patch.object(it, "read_selection", return_value="fal"), \ + patch.object(it, "fal_key_is_configured", return_value=False), \ + patch.object(it, "resolve_managed_tool_gateway", return_value=MANAGED): + # Broken vendor selection reports unavailable even though the + # managed gateway would resolve. + assert it.check_fal_api_key() is False + + +# --------------------------------------------------------------------------- +# Video generation (FAL plugin) +# --------------------------------------------------------------------------- + + +class TestVideoFalStrictSelection: + def test_nous_selection_routes_managed_even_with_fal_key(self): + from plugins.video_gen import fal as vf + + with patch("tools.tool_backend_helpers.read_selection", return_value="nous"), \ + patch("tools.tool_backend_helpers.fal_key_is_configured", return_value=True), \ + patch("tools.managed_tool_gateway.resolve_managed_tool_gateway", return_value=MANAGED): + assert vf._resolve_managed_fal_video_gateway() is MANAGED + + def test_fal_selection_missing_key_errors_without_managed_call(self): + from plugins.video_gen import fal as vf + + with patch("tools.tool_backend_helpers.read_selection", return_value="fal"), \ + patch("tools.tool_backend_helpers.fal_key_is_configured", return_value=False), \ + patch("tools.managed_tool_gateway.resolve_managed_tool_gateway") as gw: + with pytest.raises(ValueError) as exc: + vf._resolve_managed_fal_video_gateway() + gw.assert_not_called() + assert "video_gen is configured to use fal" in str(exc.value) + assert "FAL_KEY" in str(exc.value) + + def test_never_configured_autodetect_unchanged(self): + from plugins.video_gen import fal as vf + + with patch("tools.tool_backend_helpers.read_selection", return_value=None), \ + patch("tools.tool_backend_helpers.fal_key_is_configured", return_value=True): + assert vf._resolve_managed_fal_video_gateway() is None + + +# --------------------------------------------------------------------------- +# STT (OpenAI audio resolver — previously ignored the stored intent entirely) +# --------------------------------------------------------------------------- + + +class TestSttStrictSelection: + def test_nous_selection_beats_direct_openai_key(self): + from tools import transcription_tools as tt + + with patch.object(tt, "_load_stt_config", return_value={"openai": {"api_key": "sk-direct"}}), \ + patch("tools.tool_backend_helpers.read_selection", return_value="nous"), \ + patch.object(tt, "resolve_managed_tool_gateway", return_value=MANAGED): + api_key, base_url = tt._resolve_openai_audio_client_config() + assert api_key == "managed-token" + assert base_url.startswith("https://gateway.nousresearch.com") + + def test_vendor_selection_missing_key_errors_without_managed_call(self): + from tools import transcription_tools as tt + + with patch.object(tt, "_load_stt_config", return_value={}), \ + patch("tools.tool_backend_helpers.read_selection", return_value="openai"), \ + patch.object(tt, "resolve_openai_audio_api_key", return_value=""), \ + patch.object(tt, "resolve_managed_tool_gateway") as gw: + with pytest.raises(ValueError) as exc: + tt._resolve_openai_audio_client_config() + gw.assert_not_called() + assert "stt is configured to use openai" in str(exc.value) + assert "hermes tools" in str(exc.value) + + def test_never_configured_keeps_legacy_ladder(self): + from tools import transcription_tools as tt + + with patch.object(tt, "_load_stt_config", return_value={}), \ + patch("tools.tool_backend_helpers.read_selection", return_value=None), \ + patch.object(tt, "resolve_openai_audio_api_key", return_value="sk-env"): + api_key, base_url = tt._resolve_openai_audio_client_config() + assert api_key == "sk-env" + + +# --------------------------------------------------------------------------- +# Browser Use provider +# --------------------------------------------------------------------------- + + +class TestBrowserUseStrictSelection: + def _provider(self): + from plugins.browser.browser_use.provider import BrowserUseBrowserProvider + + return BrowserUseBrowserProvider() + + def test_nous_selection_routes_managed_even_with_direct_key(self): + provider = self._provider() + with patch("plugins.browser.browser_use.provider.get_secret", return_value="bu-key"), \ + patch("tools.tool_backend_helpers.read_selection", return_value="nous"), \ + patch("tools.managed_tool_gateway.resolve_managed_tool_gateway", return_value=MANAGED): + config = provider._get_config_or_none() + assert config["managed_mode"] is True + assert config["api_key"] == "managed-token" + + def test_vendor_selection_missing_key_errors_without_managed_call(self): + provider = self._provider() + with patch("plugins.browser.browser_use.provider.get_secret", return_value=""), \ + patch("tools.tool_backend_helpers.read_selection", return_value="browser-use"), \ + patch("tools.managed_tool_gateway.resolve_managed_tool_gateway") as gw: + with pytest.raises(ValueError) as exc: + provider._get_config() + gw.assert_not_called() + assert "browser is configured to use browser-use" in str(exc.value) + assert "BROWSER_USE_API_KEY" in str(exc.value) + + def test_never_configured_key_still_routes_direct(self): + provider = self._provider() + with patch("plugins.browser.browser_use.provider.get_secret", return_value="bu-key"), \ + patch("tools.tool_backend_helpers.read_selection", return_value=None): + config = provider._get_config_or_none() + assert config["managed_mode"] is False + assert config["api_key"] == "bu-key" + + +# --------------------------------------------------------------------------- +# Camofox: selection over env var +# --------------------------------------------------------------------------- + + +class TestCamofoxSelection: + def test_camofox_selection_activates_mode(self, monkeypatch): + from tools import browser_camofox as bc + + monkeypatch.delenv("BROWSER_CDP_URL", raising=False) + with patch.object(bc, "_config_cdp_url", return_value=""), \ + patch("tools.tool_backend_helpers.read_selection", return_value="camofox"): + assert bc.is_camofox_mode() is True + + def test_other_selection_beats_camofox_url_env(self, monkeypatch): + """CAMOFOX_URL is the ADDRESS, not the choice: an explicit different + browser selection wins.""" + from tools import browser_camofox as bc + + monkeypatch.delenv("BROWSER_CDP_URL", raising=False) + with patch.object(bc, "_config_cdp_url", return_value=""), \ + patch.object(bc, "get_camofox_url", return_value="http://localhost:9377"), \ + patch("tools.tool_backend_helpers.read_selection", return_value="local"): + assert bc.is_camofox_mode() is False + + def test_never_configured_env_url_still_activates(self, monkeypatch): + from tools import browser_camofox as bc + + monkeypatch.delenv("BROWSER_CDP_URL", raising=False) + with patch.object(bc, "_config_cdp_url", return_value=""), \ + patch.object(bc, "get_camofox_url", return_value="http://localhost:9377"), \ + patch("tools.tool_backend_helpers.read_selection", return_value=None): + assert bc.is_camofox_mode() is True + + +# --------------------------------------------------------------------------- +# tools_config writers: one provider string per row, no use_gateway writes +# --------------------------------------------------------------------------- + + +class TestWriteProviderConfig: + def test_managed_row_writes_nous_and_clears_legacy_flag(self): + from hermes_cli.tools_config import _write_provider_config + + config = {"tts": {"provider": "edge", "use_gateway": False}} + provider = {"name": "Nous Subscription", "tts_provider": "openai"} + _write_provider_config(provider, config, managed_feature="tts") + assert config["tts"]["provider"] == "nous" + assert "use_gateway" not in config["tts"] + + def test_byok_row_writes_vendor_and_clears_legacy_flag(self): + from hermes_cli.tools_config import _write_provider_config + + config = {"web": {"backend": "nous", "use_gateway": True}} + provider = {"name": "Tavily", "web_backend": "tavily"} + _write_provider_config(provider, config, managed_feature=None) + assert config["web"]["backend"] == "tavily" + assert "use_gateway" not in config["web"] + + def test_managed_image_row_persists_nous_provider(self): + from hermes_cli.tools_config import _write_provider_config + + config = {} + provider = {"name": "Nous Subscription", "imagegen_backend": "fal"} + _write_provider_config(provider, config, managed_feature="image_gen") + assert config["image_gen"]["provider"] == "nous" + assert "use_gateway" not in config["image_gen"] + + def test_plugin_injected_byok_row_clears_stale_use_gateway(self): + """Plugin-injected rows are not in TOOL_CATEGORIES' hardcoded + provider lists; the legacy clear-loop skipped them.""" + from hermes_cli.tools_config import _write_provider_config + + config = {"stt": {"provider": "nous", "use_gateway": True}} + provider = {"name": "Groq Whisper", "stt_provider": "groq"} + _write_provider_config(provider, config, managed_feature=None) + assert config["stt"]["provider"] == "groq" + assert "use_gateway" not in config["stt"] diff --git a/tests/tools/test_transcription_tools.py b/tests/tools/test_transcription_tools.py index a2b586a9cd..97177ff027 100644 --- a/tests/tools/test_transcription_tools.py +++ b/tests/tools/test_transcription_tools.py @@ -131,11 +131,25 @@ class TestExplicitProviderRespected: monkeypatch.delenv("GROQ_API_KEY", raising=False) with patch("tools.transcription_tools._HAS_FASTER_WHISPER", False), \ patch("tools.transcription_tools._has_local_command", return_value=False), \ + patch("tools.tool_backend_helpers.read_selection", return_value="local"), \ patch("tools.transcription_tools._HAS_OPENAI", True): from tools.transcription_tools import _get_provider result = _get_provider({"provider": "local"}) assert result == "none", f"Expected 'none' but got {result!r}" + def test_seeded_local_without_stored_selection_autodetects(self, monkeypatch): + """The DEFAULT_CONFIG-seeded stt.provider: local (no raw-config + selection) is treated as never-configured: autodetect runs instead of + hard-pinning to a missing local backend.""" + monkeypatch.setenv("GROQ_API_KEY", "gsk-test") + with patch("tools.transcription_tools._HAS_FASTER_WHISPER", False), \ + patch("tools.transcription_tools._has_local_command", return_value=False), \ + patch("tools.transcription_tools._try_lazy_install_stt", return_value=False), \ + patch("tools.tool_backend_helpers.read_selection", return_value=None), \ + patch("tools.transcription_tools._HAS_OPENAI", True): + from tools.transcription_tools import _get_provider + assert _get_provider({"provider": "local"}) == "groq" + def test_explicit_local_uses_local_command_fallback(self, monkeypatch): """Local-to-local_command fallback is fine — both are local.""" monkeypatch.setenv( diff --git a/tests/tools/test_tts_openai_config.py b/tests/tools/test_tts_openai_config.py index f489ab8560..8aacce7ac3 100644 --- a/tests/tools/test_tts_openai_config.py +++ b/tests/tools/test_tts_openai_config.py @@ -25,7 +25,7 @@ class TestResolveOpenaiAudioClientConfig: } with patch.object(tts_tool, "_load_tts_config", return_value=config), \ - patch.object(tts_tool, "prefers_gateway", return_value=False), \ + patch.object(tts_tool, "read_selection", return_value="openai"), \ patch.object(tts_tool, "resolve_openai_audio_api_key", return_value="env-key"), \ patch.object(tts_tool, "resolve_managed_tool_gateway", return_value=None): assert tts_tool._resolve_openai_audio_client_config() == ( @@ -38,7 +38,7 @@ class TestResolveOpenaiAudioClientConfig: config = {"openai": {"api_key": "cfg-key"}} with patch.object(tts_tool, "_load_tts_config", return_value=config), \ - patch.object(tts_tool, "prefers_gateway", return_value=False): + patch.object(tts_tool, "read_selection", return_value=None): assert tts_tool._resolve_openai_audio_client_config() == ( "cfg-key", tts_tool.DEFAULT_OPENAI_BASE_URL, @@ -46,7 +46,9 @@ class TestResolveOpenaiAudioClientConfig: ) - def test_use_gateway_overrides_config_credentials(self): + def test_nous_selection_overrides_config_credentials(self): + """A stored 'nous' selection (or legacy use_gateway: true) routes + managed even when direct credentials are present.""" config = {"openai": {"api_key": "cfg-key", "base_url": "http://localhost:4003/v1"}} managed = SimpleNamespace( nous_user_token="managed-token", @@ -54,7 +56,7 @@ class TestResolveOpenaiAudioClientConfig: ) with patch.object(tts_tool, "_load_tts_config", return_value=config), \ - patch.object(tts_tool, "prefers_gateway", return_value=True), \ + patch.object(tts_tool, "read_selection", return_value="nous"), \ patch.object(tts_tool, "resolve_openai_audio_api_key", return_value="env-key"), \ patch.object(tts_tool, "resolve_managed_tool_gateway", return_value=managed): assert tts_tool._resolve_openai_audio_client_config() == ( @@ -63,9 +65,35 @@ class TestResolveOpenaiAudioClientConfig: True, ) + def test_nous_selection_unentitled_raises_selection_error(self): + """Selected managed route + unavailable gateway = honest error naming + the selection, never a silent fall back to direct credentials.""" + config = {"openai": {"api_key": "cfg-key"}} + with patch.object(tts_tool, "_load_tts_config", return_value=config), \ + patch.object(tts_tool, "read_selection", return_value="nous"), \ + patch.object(tts_tool, "resolve_openai_audio_api_key", return_value="env-key"), \ + patch.object(tts_tool, "resolve_managed_tool_gateway", return_value=None): + with pytest.raises(ValueError) as exc: + tts_tool._resolve_openai_audio_client_config() + assert "nous" in str(exc.value) + assert "hermes tools" in str(exc.value) + + def test_vendor_selection_missing_key_raises_selection_error(self): + """A stored vendor selection with no credentials errors by name — + NO managed gateway call is attempted.""" + with patch.object(tts_tool, "_load_tts_config", return_value={"provider": "openai"}), \ + patch.object(tts_tool, "read_selection", return_value="openai"), \ + patch.object(tts_tool, "resolve_openai_audio_api_key", return_value=""), \ + patch.object(tts_tool, "resolve_managed_tool_gateway") as gateway_mock: + with pytest.raises(ValueError) as exc: + tts_tool._resolve_openai_audio_client_config() + gateway_mock.assert_not_called() + assert "openai" in str(exc.value) + assert "hermes tools" in str(exc.value) + def test_missing_config_and_env_raises_updated_error(self): with patch.object(tts_tool, "_load_tts_config", return_value={}), \ - patch.object(tts_tool, "prefers_gateway", return_value=False), \ + patch.object(tts_tool, "read_selection", return_value=None), \ patch.object(tts_tool, "resolve_openai_audio_api_key", return_value=""), \ patch.object(tts_tool, "resolve_managed_tool_gateway", return_value=None), \ patch.object(tts_tool, "managed_nous_tools_enabled", return_value=False): diff --git a/tests/tools/test_web_tools_config.py b/tests/tools/test_web_tools_config.py index 237037a22f..56cbaad0bd 100644 --- a/tests/tools/test_web_tools_config.py +++ b/tests/tools/test_web_tools_config.py @@ -222,12 +222,29 @@ class TestBackendSelection: patch("tools.web_tools._ddgs_package_importable", return_value=False): assert _get_backend() == "firecrawl" - def test_invalid_config_falls_through_to_fallback(self): - """web.backend=invalid → ignored, uses key-based fallback.""" + def test_invalid_config_is_returned_verbatim(self): + """Strict selection: web.backend=nonexistent is returned as-is so the + dispatch path raises the honest selection-naming error — never + silently rerouted through the credential ladder.""" from tools.web_tools import _get_backend with patch("tools.web_tools._load_web_config", return_value={"backend": "nonexistent"}), \ patch.dict(os.environ, {"PARALLEL_API_KEY": "test-key"}): - assert _get_backend() == "parallel" + assert _get_backend() == "nonexistent" + + def test_stored_backend_wins_over_other_credentials(self): + """Strict selection: a stored web.backend beats env keys for other + vendors — no availability probe, no credential override.""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={"backend": "firecrawl"}), \ + patch.dict(os.environ, {"TAVILY_API_KEY": "tvly-test"}): + assert _get_backend() == "firecrawl" + + def test_nous_backend_maps_to_firecrawl(self): + """The managed 'nous' selection is serviced by the firecrawl + provider (whose client resolver routes managed).""" + from tools.web_tools import _get_backend + with patch("tools.web_tools._load_web_config", return_value={"backend": "nous"}): + assert _get_backend() == "firecrawl" def test_managed_gateway_does_not_preempt_explicit_tavily(self): """Regression: a Nous OAuth token (managed gateway "ready") must NOT