fix(web): drop tavily from the removed-backend registry after the restore
#100540 added a REMOVED_BACKENDS startup warning keyed on tavily; with the backend restored, that entry would warn on a working provider. The registry stays (empty) for future removals; migration tests now pin the machinery via a synthetic entry plus a guard asserting no live provider is ever listed as removed.
This commit is contained in:
@@ -1,54 +1,68 @@
|
||||
"""Removed-backend migration warnings (post-#99199 Tavily removal).
|
||||
"""Removed-backend migration warnings (registry added in #100540).
|
||||
|
||||
A config still pointing at a backend that no longer ships in-tree
|
||||
(``web.backend: tavily``) must fail loudly and specifically:
|
||||
A config still pointing at a backend registered in
|
||||
``tools.tool_backend_helpers.REMOVED_BACKENDS`` must fail loudly and
|
||||
specifically:
|
||||
|
||||
1. startup — ``validate_config_structure`` emits a warning naming the
|
||||
removal, instead of staying silent until the first tool call;
|
||||
2. tool call — ``selection_error`` explains the backend was removed and
|
||||
names alternatives, instead of the generic "no registered provider
|
||||
has that name".
|
||||
2. tool call — ``selection_error`` explains the backend was removed,
|
||||
instead of the generic "no registered provider has that name".
|
||||
|
||||
Regression source: keyed Tavily users upgrading to v0.21.0 saw their
|
||||
config silently become invalid with no migration or startup notice
|
||||
(reported on PR #99731).
|
||||
The registry ships empty on main (the Tavily removal that motivated it,
|
||||
#99199, was reverted by the #99731 restore), so these tests inject a
|
||||
synthetic ``legacysearch`` entry — they pin the machinery, not any
|
||||
specific vendor's membership.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.tool_backend_helpers as tbh
|
||||
from hermes_cli.config import validate_config_structure
|
||||
from tools.tool_backend_helpers import (
|
||||
REMOVED_BACKENDS,
|
||||
removed_backend_note,
|
||||
selection_error,
|
||||
)
|
||||
from tools.tool_backend_helpers import removed_backend_note, selection_error
|
||||
|
||||
_NOTE = "the LegacySearch backend was removed in v0.0.0 (alternatives: exa, parallel)"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_removed(monkeypatch):
|
||||
monkeypatch.setitem(tbh.REMOVED_BACKENDS, "web", {"legacysearch": _NOTE})
|
||||
|
||||
|
||||
class TestRemovedBackendNote:
|
||||
def test_tavily_is_registered_as_removed_web_backend(self):
|
||||
assert "tavily" in REMOVED_BACKENDS["web"]
|
||||
def test_note_lookup_normalizes_quotes_and_case(self, legacy_removed):
|
||||
assert removed_backend_note("web", "legacysearch") == _NOTE
|
||||
assert removed_backend_note("web", "'LegacySearch'") == _NOTE
|
||||
assert removed_backend_note("web", ' "LEGACYSEARCH" ') == _NOTE
|
||||
|
||||
def test_note_lookup_normalizes_quotes_and_case(self):
|
||||
plain = removed_backend_note("web", "tavily")
|
||||
assert plain is not None
|
||||
assert removed_backend_note("web", "'Tavily'") == plain
|
||||
assert removed_backend_note("web", ' "TAVILY" ') == plain
|
||||
|
||||
def test_unknown_names_and_sections_return_none(self):
|
||||
def test_unknown_names_and_sections_return_none(self, legacy_removed):
|
||||
assert removed_backend_note("web", "exa") is None
|
||||
assert removed_backend_note("web", "") is None
|
||||
assert removed_backend_note("stt", "tavily") is None
|
||||
assert removed_backend_note("stt", "legacysearch") is None
|
||||
|
||||
def test_registry_ships_without_live_backends(self):
|
||||
# Restored/live backends must never sit in REMOVED_BACKENDS — the
|
||||
# startup warning would fire on a working provider. Guards the
|
||||
# #99731 restore against a stale tavily entry reappearing.
|
||||
from agent.web_search_registry import get_provider
|
||||
|
||||
for name in tbh.REMOVED_BACKENDS.get("web", {}):
|
||||
assert get_provider(name) is None, (
|
||||
f"{name!r} is registered as removed but a live web provider "
|
||||
"with that name exists"
|
||||
)
|
||||
|
||||
|
||||
class TestSelectionErrorRemovedBackend:
|
||||
def test_removed_backend_gets_specific_explanation(self):
|
||||
msg = selection_error("web", "'tavily'", "no registered web search provider has that name")
|
||||
assert "removed" in msg
|
||||
assert "tavily" in msg.lower()
|
||||
def test_removed_backend_gets_specific_explanation(self, legacy_removed):
|
||||
msg = selection_error("web", "'legacysearch'", "no registered web search provider has that name")
|
||||
assert _NOTE in msg
|
||||
# generic failure text replaced, not appended
|
||||
assert "no registered web search provider" not in msg
|
||||
# still ends with the uniform remediation contract
|
||||
assert "Run 'hermes tools' to change it." in msg
|
||||
|
||||
def test_live_backend_keeps_caller_failure_text(self):
|
||||
def test_live_backend_keeps_caller_failure_text(self, legacy_removed):
|
||||
msg = selection_error("web", "'exa'", "no registered web search provider has that name")
|
||||
assert "no registered web search provider has that name" in msg
|
||||
assert "removed" not in msg
|
||||
@@ -59,26 +73,26 @@ class TestStartupWarningForRemovedWebBackend:
|
||||
def _removed_issues(config):
|
||||
return [
|
||||
i for i in validate_config_structure(config)
|
||||
if "removed" in i.message and "tavily" in i.message
|
||||
if "removed" in i.message and "legacysearch" in i.message
|
||||
]
|
||||
|
||||
def test_stale_web_backend_warns_at_startup(self):
|
||||
issues = self._removed_issues({"web": {"backend": "tavily"}})
|
||||
def test_stale_web_backend_warns_at_startup(self, legacy_removed):
|
||||
issues = self._removed_issues({"web": {"backend": "legacysearch"}})
|
||||
assert len(issues) == 1
|
||||
assert issues[0].severity == "warning"
|
||||
assert "hermes tools" in issues[0].hint
|
||||
|
||||
def test_per_capability_keys_are_checked(self):
|
||||
assert len(self._removed_issues({"web": {"search_backend": "tavily"}})) == 1
|
||||
assert len(self._removed_issues({"web": {"extract_backend": "tavily"}})) == 1
|
||||
def test_per_capability_keys_are_checked(self, legacy_removed):
|
||||
assert len(self._removed_issues({"web": {"search_backend": "legacysearch"}})) == 1
|
||||
assert len(self._removed_issues({"web": {"extract_backend": "legacysearch"}})) == 1
|
||||
|
||||
def test_same_stale_value_warns_once(self):
|
||||
def test_same_stale_value_warns_once(self, legacy_removed):
|
||||
issues = self._removed_issues(
|
||||
{"web": {"backend": "tavily", "search_backend": "tavily", "extract_backend": "tavily"}}
|
||||
{"web": {"backend": "legacysearch", "search_backend": "legacysearch", "extract_backend": "legacysearch"}}
|
||||
)
|
||||
assert len(issues) == 1
|
||||
|
||||
def test_healthy_backend_produces_no_removed_warning(self):
|
||||
def test_healthy_backend_produces_no_removed_warning(self, legacy_removed):
|
||||
assert self._removed_issues({"web": {"backend": "exa"}}) == []
|
||||
assert self._removed_issues({"web": {}}) == []
|
||||
assert self._removed_issues({}) == []
|
||||
|
||||
@@ -411,12 +411,10 @@ def selection_exists(section: str) -> bool:
|
||||
# happened and what to do. Declared data, one policy — add future removals
|
||||
# here, never as one-off string checks at call sites.
|
||||
REMOVED_BACKENDS: Dict[str, Dict[str, str]] = {
|
||||
"web": {
|
||||
"tavily": (
|
||||
"the Tavily backend was removed in v0.21.0 "
|
||||
"(keyless alternatives: exa, parallel, firecrawl, keenable)"
|
||||
),
|
||||
},
|
||||
# Currently empty: the Tavily removal (#99199) that introduced this
|
||||
# registry was reverted by the #99731 restore. Future backend removals
|
||||
# add an entry here, e.g.
|
||||
# "web": {"<name>": "the <Name> backend was removed in vX.Y.Z (...)"},
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user