c1762ff11c
The rename sweep in the base commit missed the sibling-test blast radius (18 red files on CI). Three classes, all fixed: 1. Stale old names in tests (todo/cronjob/process/tour/tip) — updated to todo_list/cronjob_manage/process_manage/gui_tour/show_tip at every registry.get_entry/dispatch/coerce/preview/allowlist call site, plus the coding-brief sentence in agent/coding_context.py now names todo_list (and its gating test). 2. Missed rename in production: AGENT_RUNTIME_POST_HOOK_TOOL_NAMES still held 'tour' — post-hook ownership would have double-emitted for gui_tour via the bridge path. 3. Tests pinning pre-deferral assembly (blank-slate surface, modal sandbox resolution, desktop diet, HUD note) now pin their ACTUAL contract under the legacy defer:[] override, or assert on granted tool names instead of visible schemas. Also fixes a pre-existing ordering flake surfaced by the sweep: test_holds_exactly_the_gui_affordances depended on whether an earlier test had imported apply_layout_tool (registry-registered, not in the static desktop_ui list) — now forces discovery and pins the full set. 649 tests green locally across all touched files, both orderings.
359 lines
12 KiB
Python
359 lines
12 KiB
Python
"""Tests for toolsets.py — toolset resolution, validation, and composition."""
|
|
|
|
import toolsets as toolsets_mod
|
|
from tools.registry import ToolRegistry
|
|
from toolsets import (
|
|
TOOLSETS,
|
|
get_toolset,
|
|
resolve_toolset,
|
|
resolve_multiple_toolsets,
|
|
get_all_toolsets,
|
|
validate_toolset,
|
|
create_custom_toolset,
|
|
get_toolset_info,
|
|
)
|
|
|
|
|
|
def _dummy_handler(args, **kwargs):
|
|
return "{}"
|
|
|
|
|
|
def _make_schema(name: str, description: str = "test tool"):
|
|
return {
|
|
"name": name,
|
|
"description": description,
|
|
"parameters": {"type": "object", "properties": {}},
|
|
}
|
|
|
|
|
|
class TestGetToolset:
|
|
def test_known_toolset(self):
|
|
ts = get_toolset("web")
|
|
assert ts is not None
|
|
assert "web_search" in ts["tools"]
|
|
|
|
def test_x_search_toolset_marks_read_only_and_points_to_xurl(self):
|
|
ts = get_toolset("x_search")
|
|
assert ts is not None
|
|
assert ts["tools"] == ["x_search"]
|
|
description = ts["description"].lower()
|
|
assert "read-only" in description
|
|
assert "xurl" in description
|
|
assert "authenticated" in description
|
|
|
|
def test_merges_registry_tools_into_builtin_toolset(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="web_search_plus",
|
|
toolset="web",
|
|
schema=_make_schema("web_search_plus", "Plugin web search"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
ts = get_toolset("web")
|
|
assert ts is not None
|
|
assert set(ts["tools"]) == {"web_search", "web_extract", "web_search_plus"}
|
|
|
|
|
|
|
|
class TestResolveToolset:
|
|
def test_leaf_toolset(self):
|
|
tools = resolve_toolset("web")
|
|
assert set(tools) == {"web_search", "web_extract"}
|
|
|
|
def test_composite_toolset(self):
|
|
tools = resolve_toolset("debugging")
|
|
assert "terminal" in tools
|
|
assert "web_search" in tools
|
|
assert "web_extract" in tools
|
|
|
|
def test_cycle_detection(self):
|
|
# Create a cycle: A includes B, B includes A
|
|
TOOLSETS["_cycle_a"] = {"description": "test", "tools": ["t1"], "includes": ["_cycle_b"]}
|
|
TOOLSETS["_cycle_b"] = {"description": "test", "tools": ["t2"], "includes": ["_cycle_a"]}
|
|
try:
|
|
tools = resolve_toolset("_cycle_a")
|
|
# Should not infinite loop — cycle is detected
|
|
assert "t1" in tools
|
|
assert "t2" in tools
|
|
finally:
|
|
del TOOLSETS["_cycle_a"]
|
|
del TOOLSETS["_cycle_b"]
|
|
|
|
|
|
def test_plugin_toolset_uses_registry_snapshot(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="plugin_b",
|
|
toolset="plugin_example",
|
|
schema=_make_schema("plugin_b", "B"),
|
|
handler=_dummy_handler,
|
|
)
|
|
reg.register(
|
|
name="plugin_a",
|
|
toolset="plugin_example",
|
|
schema=_make_schema("plugin_a", "A"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert resolve_toolset("plugin_example") == ["plugin_a", "plugin_b"]
|
|
|
|
|
|
|
|
|
|
class TestResolveMultipleToolsets:
|
|
def test_combines_and_deduplicates(self):
|
|
tools = resolve_multiple_toolsets(["web", "terminal"])
|
|
assert "web_search" in tools
|
|
assert "web_extract" in tools
|
|
assert "terminal" in tools
|
|
# No duplicates
|
|
assert len(tools) == len(set(tools))
|
|
|
|
|
|
|
|
class TestValidateToolset:
|
|
def test_valid(self):
|
|
assert validate_toolset("web") is True
|
|
assert validate_toolset("terminal") is True
|
|
|
|
|
|
def test_invalid(self):
|
|
assert validate_toolset("nonexistent") is False
|
|
|
|
def test_mcp_alias_uses_live_registry(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="mcp__dynserver__ping",
|
|
toolset="mcp-dynserver",
|
|
schema=_make_schema("mcp__dynserver__ping", "Ping"),
|
|
handler=_dummy_handler,
|
|
)
|
|
reg.register_toolset_alias("dynserver", "mcp-dynserver")
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert validate_toolset("dynserver") is True
|
|
assert validate_toolset("mcp-dynserver") is True
|
|
assert "mcp__dynserver__ping" in resolve_toolset("dynserver")
|
|
|
|
|
|
class TestGetToolsetInfo:
|
|
def test_leaf(self):
|
|
info = get_toolset_info("web")
|
|
assert info["name"] == "web"
|
|
assert info["is_composite"] is False
|
|
assert info["tool_count"] == 2
|
|
|
|
def test_composite(self):
|
|
info = get_toolset_info("debugging")
|
|
assert info["is_composite"] is True
|
|
assert info["tool_count"] > len(info["direct_tools"])
|
|
|
|
|
|
|
|
class TestCreateCustomToolset:
|
|
def test_runtime_creation(self):
|
|
create_custom_toolset(
|
|
name="_test_custom",
|
|
description="Test toolset",
|
|
tools=["web_search"],
|
|
includes=["terminal"],
|
|
)
|
|
try:
|
|
tools = resolve_toolset("_test_custom")
|
|
assert "web_search" in tools
|
|
assert "terminal" in tools
|
|
assert validate_toolset("_test_custom") is True
|
|
finally:
|
|
del TOOLSETS["_test_custom"]
|
|
|
|
|
|
class TestRegistryOwnedToolsets:
|
|
def test_registry_membership_is_live(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="test_live_toolset_tool",
|
|
toolset="test-live-toolset",
|
|
schema=_make_schema("test_live_toolset_tool", "Live"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
assert validate_toolset("test-live-toolset") is True
|
|
assert get_toolset("test-live-toolset")["tools"] == ["test_live_toolset_tool"]
|
|
assert resolve_toolset("test-live-toolset") == ["test_live_toolset_tool"]
|
|
|
|
|
|
class TestToolsetConsistency:
|
|
"""Verify structural integrity of the built-in TOOLSETS dict."""
|
|
|
|
def test_all_toolsets_have_required_keys(self):
|
|
for name, ts in TOOLSETS.items():
|
|
assert "description" in ts, f"{name} missing description"
|
|
assert "tools" in ts, f"{name} missing tools"
|
|
assert "includes" in ts, f"{name} missing includes"
|
|
|
|
|
|
def test_hermes_platforms_share_core_tools(self):
|
|
"""All hermes-* platform toolsets share the same core tools.
|
|
|
|
Platform-specific additions (e.g. ``discord`` / ``discord_admin``
|
|
on hermes-discord, gated on DISCORD_BOT_TOKEN) are allowed on top —
|
|
the invariant is that the core set is identical across platforms.
|
|
"""
|
|
platforms = ["hermes-cli", "hermes-telegram", "hermes-discord", "hermes-whatsapp", "hermes-slack", "hermes-signal", "hermes-homeassistant"]
|
|
tool_sets = [set(TOOLSETS[p]["tools"]) for p in platforms]
|
|
# All platforms must contain the shared core; platform-specific
|
|
# extras are OK (subset check, not equality).
|
|
core = set.intersection(*tool_sets)
|
|
for name, ts in zip(platforms, tool_sets):
|
|
assert core.issubset(ts), f"{name} is missing core tools: {core - ts}"
|
|
# Sanity: the shared core must be non-trivial (i.e. we didn't
|
|
# silently let a platform diverge so far that nothing is shared).
|
|
assert len(core) > 20, f"Suspiciously small shared core: {len(core)} tools"
|
|
|
|
|
|
class TestPluginToolsets:
|
|
def test_get_all_toolsets_includes_plugin_toolset(self, monkeypatch):
|
|
reg = ToolRegistry()
|
|
reg.register(
|
|
name="plugin_tool",
|
|
toolset="plugin_bundle",
|
|
schema=_make_schema("plugin_tool", "Plugin tool"),
|
|
handler=_dummy_handler,
|
|
)
|
|
|
|
monkeypatch.setattr("tools.registry.registry", reg)
|
|
|
|
all_toolsets = get_all_toolsets()
|
|
assert "plugin_bundle" in all_toolsets
|
|
assert all_toolsets["plugin_bundle"]["tools"] == ["plugin_tool"]
|
|
|
|
|
|
class TestDefaultPlatformWebSearchCoverage:
|
|
def test_hermes_whatsapp_toolset_includes_web_search(self):
|
|
assert "web_search" in resolve_toolset("hermes-whatsapp")
|
|
|
|
|
|
|
|
class TestResolveToolsetIncludeRegistry:
|
|
"""include_registry flag exposes the static (pre-registry-merge) view used
|
|
by platform reverse-mapping. Regression harness for issue #49622."""
|
|
|
|
def test_include_registry_false_excludes_registry_tools(self):
|
|
from tools.registry import discover_builtin_tools, registry
|
|
discover_builtin_tools()
|
|
|
|
# Register a tool into `terminal` at runtime, the way plugins and MCP
|
|
# servers do, so the split is exercised on the mechanism rather than on
|
|
# whichever built-in currently happens to live where.
|
|
registry.register(
|
|
name="__probe_registry_only_tool__",
|
|
toolset="terminal",
|
|
schema={"name": "__probe_registry_only_tool__", "parameters": {"type": "object", "properties": {}}},
|
|
handler=lambda args, **kw: "",
|
|
)
|
|
try:
|
|
merged = set(resolve_toolset("terminal"))
|
|
static = set(resolve_toolset("terminal", include_registry=False))
|
|
finally:
|
|
registry.deregister("__probe_registry_only_tool__")
|
|
|
|
assert static == {"terminal", "process_manage"}, static
|
|
# Registered into 'terminal' but not part of the static definition — it
|
|
# must only appear in the merged view.
|
|
assert "__probe_registry_only_tool__" in merged
|
|
assert "__probe_registry_only_tool__" not in static
|
|
|
|
|
|
def test_static_view_threads_through_includes(self):
|
|
# 'debugging' has direct tools [terminal, process] and includes [web, file]
|
|
static = set(resolve_toolset("debugging", include_registry=False))
|
|
assert {"terminal", "process_manage"} <= static
|
|
assert "web_search" in static
|
|
assert "read_file" in static
|
|
|
|
|
|
def test_registry_only_toolset_static_view_is_empty(self):
|
|
assert resolve_toolset("__definitely_not_a_real_toolset__", include_registry=False) == []
|
|
|
|
|
|
class TestResolveToolsetMemo:
|
|
"""Measured-work pins for the generation-keyed resolution memo."""
|
|
|
|
def test_second_resolution_is_cached(self, monkeypatch):
|
|
"""Repeated resolves of the same toolset must not re-walk the registry.
|
|
|
|
resolve_toolset is called dozens of times per _get_platform_tools()
|
|
(every /tools completion keystroke). The memo keyed on the registry
|
|
generation makes repeat calls a dict lookup instead of a full
|
|
includes-walk + registry snapshot.
|
|
"""
|
|
from tools.registry import registry
|
|
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
get_toolset_calls = {"n": 0}
|
|
|
|
orig_get_toolset = toolsets_mod.get_toolset
|
|
|
|
def counting_get_toolset(name, *, include_registry=True):
|
|
get_toolset_calls["n"] += 1
|
|
return orig_get_toolset(name, include_registry=include_registry)
|
|
|
|
monkeypatch.setattr(toolsets_mod, "get_toolset", counting_get_toolset)
|
|
|
|
registry_id = id(registry)
|
|
generation = registry._generation
|
|
|
|
first = resolve_toolset("hermes-cli")
|
|
second = resolve_toolset("hermes-cli")
|
|
|
|
assert first == second
|
|
assert get_toolset_calls["n"] == 1, (
|
|
"second resolution must be a memo hit (no get_toolset re-walk), "
|
|
f"got {get_toolset_calls['n']} calls"
|
|
)
|
|
assert (
|
|
"hermes-cli", True, registry_id, generation
|
|
) in toolsets_mod._resolve_toolset_memo
|
|
|
|
def test_generation_bump_invalidates_memo(self, monkeypatch):
|
|
"""A registry mutation (generation bump) must force a fresh resolve."""
|
|
from tools.registry import registry
|
|
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
get_toolset_calls = {"n": 0}
|
|
|
|
orig_get_toolset = toolsets_mod.get_toolset
|
|
|
|
def counting_get_toolset(name, *, include_registry=True):
|
|
get_toolset_calls["n"] += 1
|
|
return orig_get_toolset(name, include_registry=include_registry)
|
|
|
|
monkeypatch.setattr(toolsets_mod, "get_toolset", counting_get_toolset)
|
|
|
|
resolve_toolset("hermes-cli")
|
|
assert get_toolset_calls["n"] == 1
|
|
|
|
# Simulate a registry mutation bumping the generation.
|
|
registry._generation += 1
|
|
resolve_toolset("hermes-cli")
|
|
assert get_toolset_calls["n"] == 2, (
|
|
"generation bump must invalidate the memo and re-resolve"
|
|
)
|
|
|
|
def test_memo_result_matches_fresh_resolution(self):
|
|
"""The memo must never change the resolved result."""
|
|
toolsets_mod._resolve_toolset_memo.clear()
|
|
first = resolve_toolset("hermes-cli", include_registry=False)
|
|
second = resolve_toolset("hermes-cli", include_registry=False)
|
|
assert first == second
|
|
assert first # non-empty sanity
|
|
|