fix(web): drop tavily from the removed-backend registry after the restore

#100540 added a REMOVED_BACKENDS startup warning keyed on tavily; with the
backend restored, that entry would warn on a working provider. The registry
stays (empty) for future removals; migration tests now pin the machinery via
a synthetic entry plus a guard asserting no live provider is ever listed as
removed.
This commit is contained in:
Teknium
2026-09-01 10:47:10 -07:00
parent 89ca5e614b
commit 7cd91114b4
2 changed files with 56 additions and 44 deletions

View File

@@ -1,54 +1,68 @@
"""Removed-backend migration warnings (post-#99199 Tavily removal).
"""Removed-backend migration warnings (registry added in #100540).
A config still pointing at a backend that no longer ships in-tree
(``web.backend: tavily``) must fail loudly and specifically:
A config still pointing at a backend registered in
``tools.tool_backend_helpers.REMOVED_BACKENDS`` must fail loudly and
specifically:
1. startup — ``validate_config_structure`` emits a warning naming the
removal, instead of staying silent until the first tool call;
2. tool call — ``selection_error`` explains the backend was removed and
names alternatives, instead of the generic "no registered provider
has that name".
2. tool call — ``selection_error`` explains the backend was removed,
instead of the generic "no registered provider has that name".
Regression source: keyed Tavily users upgrading to v0.21.0 saw their
config silently become invalid with no migration or startup notice
(reported on PR #99731).
The registry ships empty on main (the Tavily removal that motivated it,
#99199, was reverted by the #99731 restore), so these tests inject a
synthetic ``legacysearch`` entry — they pin the machinery, not any
specific vendor's membership.
"""
import pytest
import tools.tool_backend_helpers as tbh
from hermes_cli.config import validate_config_structure
from tools.tool_backend_helpers import (
REMOVED_BACKENDS,
removed_backend_note,
selection_error,
)
from tools.tool_backend_helpers import removed_backend_note, selection_error
_NOTE = "the LegacySearch backend was removed in v0.0.0 (alternatives: exa, parallel)"
@pytest.fixture
def legacy_removed(monkeypatch):
monkeypatch.setitem(tbh.REMOVED_BACKENDS, "web", {"legacysearch": _NOTE})
class TestRemovedBackendNote:
def test_tavily_is_registered_as_removed_web_backend(self):
assert "tavily" in REMOVED_BACKENDS["web"]
def test_note_lookup_normalizes_quotes_and_case(self, legacy_removed):
assert removed_backend_note("web", "legacysearch") == _NOTE
assert removed_backend_note("web", "'LegacySearch'") == _NOTE
assert removed_backend_note("web", ' "LEGACYSEARCH" ') == _NOTE
def test_note_lookup_normalizes_quotes_and_case(self):
plain = removed_backend_note("web", "tavily")
assert plain is not None
assert removed_backend_note("web", "'Tavily'") == plain
assert removed_backend_note("web", ' "TAVILY" ') == plain
def test_unknown_names_and_sections_return_none(self):
def test_unknown_names_and_sections_return_none(self, legacy_removed):
assert removed_backend_note("web", "exa") is None
assert removed_backend_note("web", "") is None
assert removed_backend_note("stt", "tavily") is None
assert removed_backend_note("stt", "legacysearch") is None
def test_registry_ships_without_live_backends(self):
# Restored/live backends must never sit in REMOVED_BACKENDS — the
# startup warning would fire on a working provider. Guards the
# #99731 restore against a stale tavily entry reappearing.
from agent.web_search_registry import get_provider
for name in tbh.REMOVED_BACKENDS.get("web", {}):
assert get_provider(name) is None, (
f"{name!r} is registered as removed but a live web provider "
"with that name exists"
)
class TestSelectionErrorRemovedBackend:
def test_removed_backend_gets_specific_explanation(self):
msg = selection_error("web", "'tavily'", "no registered web search provider has that name")
assert "removed" in msg
assert "tavily" in msg.lower()
def test_removed_backend_gets_specific_explanation(self, legacy_removed):
msg = selection_error("web", "'legacysearch'", "no registered web search provider has that name")
assert _NOTE in msg
# generic failure text replaced, not appended
assert "no registered web search provider" not in msg
# still ends with the uniform remediation contract
assert "Run 'hermes tools' to change it." in msg
def test_live_backend_keeps_caller_failure_text(self):
def test_live_backend_keeps_caller_failure_text(self, legacy_removed):
msg = selection_error("web", "'exa'", "no registered web search provider has that name")
assert "no registered web search provider has that name" in msg
assert "removed" not in msg
@@ -59,26 +73,26 @@ class TestStartupWarningForRemovedWebBackend:
def _removed_issues(config):
return [
i for i in validate_config_structure(config)
if "removed" in i.message and "tavily" in i.message
if "removed" in i.message and "legacysearch" in i.message
]
def test_stale_web_backend_warns_at_startup(self):
issues = self._removed_issues({"web": {"backend": "tavily"}})
def test_stale_web_backend_warns_at_startup(self, legacy_removed):
issues = self._removed_issues({"web": {"backend": "legacysearch"}})
assert len(issues) == 1
assert issues[0].severity == "warning"
assert "hermes tools" in issues[0].hint
def test_per_capability_keys_are_checked(self):
assert len(self._removed_issues({"web": {"search_backend": "tavily"}})) == 1
assert len(self._removed_issues({"web": {"extract_backend": "tavily"}})) == 1
def test_per_capability_keys_are_checked(self, legacy_removed):
assert len(self._removed_issues({"web": {"search_backend": "legacysearch"}})) == 1
assert len(self._removed_issues({"web": {"extract_backend": "legacysearch"}})) == 1
def test_same_stale_value_warns_once(self):
def test_same_stale_value_warns_once(self, legacy_removed):
issues = self._removed_issues(
{"web": {"backend": "tavily", "search_backend": "tavily", "extract_backend": "tavily"}}
{"web": {"backend": "legacysearch", "search_backend": "legacysearch", "extract_backend": "legacysearch"}}
)
assert len(issues) == 1
def test_healthy_backend_produces_no_removed_warning(self):
def test_healthy_backend_produces_no_removed_warning(self, legacy_removed):
assert self._removed_issues({"web": {"backend": "exa"}}) == []
assert self._removed_issues({"web": {}}) == []
assert self._removed_issues({}) == []

View File

@@ -411,12 +411,10 @@ def selection_exists(section: str) -> bool:
# happened and what to do. Declared data, one policy — add future removals
# here, never as one-off string checks at call sites.
REMOVED_BACKENDS: Dict[str, Dict[str, str]] = {
"web": {
"tavily": (
"the Tavily backend was removed in v0.21.0 "
"(keyless alternatives: exa, parallel, firecrawl, keenable)"
),
},
# Currently empty: the Tavily removal (#99199) that introduced this
# registry was reverted by the #99731 restore. Future backend removals
# add an entry here, e.g.
# "web": {"<name>": "the <Name> backend was removed in vX.Y.Z (...)"},
}