From 217ab2f8df5712c8dd43017a43a19ebaa62c52e7 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:10:01 -0700 Subject: [PATCH] =?UTF-8?q?refactor(desktop-tools):=20consolidate=20previe?= =?UTF-8?q?w=20+=20project,=20diet=20the=20desktop=5Fui=20suite=20(3,861?= =?UTF-8?q?=20=E2=86=92=202,293=20tok/call,=20=E2=88=9241%)=20(#97659)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(desktop-tools): consolidate preview(open/close/read) + project(create/switch/list), diet the desktop_ui suite — 3,861 -> 2,293 tok/call on desktop sessions (-41%) * rename: preview -> desktop_preview, project -> desktop_project — namespace desktop-app tools against MCP/plugin name collisions * test: sync remaining old-name pins — per-file registration import, GUI_TOOLS set, post-hook case read_preview -> desktop_preview action=read --- agent/agent_runtime_helpers.py | 27 +++-- agent/tool_executor.py | 19 +-- tests/run_agent/test_run_agent.py | 2 +- tests/tools/test_close_preview_tool.py | 10 +- tests/tools/test_desktop_tools_diet.py | 113 ++++++++++++++++++ tests/tools/test_open_preview_tool.py | 10 +- tests/tools/test_read_preview_tool.py | 9 +- .../tui_gateway/test_gui_surface_toolsets.py | 4 +- tools/annotate_preview_tool.py | 36 ++---- tools/apply_layout_tool.py | 11 +- tools/close_preview_tool.py | 9 +- tools/drive_preview_tool.py | 79 +++++------- tools/focus_pane_tool.py | 9 +- tools/open_preview_tool.py | 9 +- tools/preview_tool.py | 85 +++++++++++++ tools/project_tools.py | 70 +++++------ tools/read_preview_tool.py | 13 +- tools/read_terminal_tool.py | 10 +- tools/read_window_tool.py | 17 +-- tools/setup_mcp_tool.py | 35 ++---- tools/tip_tool.py | 26 ++-- tools/tour_tool.py | 50 ++++---- toolsets.py | 4 +- 23 files changed, 383 insertions(+), 274 deletions(-) create mode 100644 tests/tools/test_desktop_tools_diet.py create mode 100644 tools/preview_tool.py diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 4ae7a18315..7f1b67822d 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -108,7 +108,7 @@ def _ra(): AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset( - {"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "drive_preview", "annotate_preview", "read_window_below", "setup_mcp", "tour", "delegate_task"} + {"todo", "session_search", "memory", "clarify", "read_terminal", "desktop_preview", "drive_preview", "annotate_preview", "read_window_below", "setup_mcp", "tour", "delegate_task"} ) @@ -3481,17 +3481,22 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i ), next_args, ) - elif function_name == "read_preview": + elif function_name == "desktop_preview": def _execute(next_args: dict) -> Any: - from tools.read_preview_tool import read_preview_tool as _read_preview_tool - return _finish_agent_tool( - _read_preview_tool( - start=next_args.get("start"), - count=next_args.get("count"), - callback=getattr(agent, "read_preview_callback", None), - ), - next_args, - ) + # action=read needs the GUI callback (agent-level); open/close go + # through the registry handler like any other tool. + if (next_args.get("action") or "").strip() == "read": + from tools.read_preview_tool import read_preview_tool as _read_preview_tool + return _finish_agent_tool( + _read_preview_tool( + start=next_args.get("start"), + count=next_args.get("count"), + callback=getattr(agent, "read_preview_callback", None), + ), + next_args, + ) + from tools.preview_tool import _handle_preview + return _finish_agent_tool(_handle_preview(next_args), next_args) elif function_name == "drive_preview": def _execute(next_args: dict) -> Any: from tools.drive_preview_tool import drive_preview_tool as _drive_preview_tool diff --git a/agent/tool_executor.py b/agent/tool_executor.py index 5ed51b42f8..5ee9da8444 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -2242,14 +2242,17 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") - elif function_name == "read_preview": + elif function_name == "desktop_preview": def _execute(next_args: dict) -> Any: - from tools.read_preview_tool import read_preview_tool as _read_preview_tool - return _read_preview_tool( - start=next_args.get("start"), - count=next_args.get("count"), - callback=getattr(agent, "read_preview_callback", None), - ) + if (next_args.get("action") or "").strip() == "read": + from tools.read_preview_tool import read_preview_tool as _read_preview_tool + return _read_preview_tool( + start=next_args.get("start"), + count=next_args.get("count"), + callback=getattr(agent, "read_preview_callback", None), + ) + from tools.preview_tool import _handle_preview + return _handle_preview(next_args) function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( agent, function_name=function_name, @@ -2262,7 +2265,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe )) tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): - agent._vprint(f" {_get_cute_tool_message_impl('read_preview', function_args, tool_duration, result=function_result)}") + agent._vprint(f" {_get_cute_tool_message_impl('desktop_preview', function_args, tool_duration, result=function_result)}") elif function_name == "drive_preview": def _execute(next_args: dict) -> Any: from tools.drive_preview_tool import drive_preview_tool as _drive_preview_tool diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 1e7b7d0f5a..af6207c683 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2446,7 +2446,7 @@ class TestAgentRuntimePostHookOwnershipSync: ("memory", {"action": "view", "target": "memory"}), ("clarify", {"question": "Continue?"}), ("read_terminal", {}), - ("read_preview", {}), + ("desktop_preview", {"action": "read"}), ("drive_preview", {"action": "elements"}), ("annotate_preview", {"action": "clear"}), ("read_window_below", {}), diff --git a/tests/tools/test_close_preview_tool.py b/tests/tools/test_close_preview_tool.py index 8cd11b5743..daee79158b 100644 --- a/tests/tools/test_close_preview_tool.py +++ b/tests/tools/test_close_preview_tool.py @@ -17,11 +17,13 @@ def _reset_emitter(): def test_lives_in_the_gui_surface_toolset(monkeypatch): - """Reaches a desktop client on ANY backend, including one with no - HERMES_DESKTOP in its environment (URL / cloud gateways).""" + import tools.preview_tool # noqa: F401 — registers desktop_preview + """Consolidated (#95681): this module's tool became an action of the + single `desktop_preview` tool in desktop_ui; the old registration is gone and + `preview` reaches a desktop client on ANY backend (no env gate).""" monkeypatch.delenv("HERMES_DESKTOP", raising=False) - entry = registry.get_entry("close_preview") - + assert registry.get_entry("close_preview") is None + entry = registry.get_entry("desktop_preview") assert entry is not None assert entry.toolset == "desktop_ui" assert entry.check_fn is None diff --git a/tests/tools/test_desktop_tools_diet.py b/tests/tools/test_desktop_tools_diet.py new file mode 100644 index 0000000000..4329662b78 --- /dev/null +++ b/tests/tools/test_desktop_tools_diet.py @@ -0,0 +1,113 @@ +"""Desktop tool consolidation + diet (#95681, maintainer-directed). + +preview = open/close/read as one action tool (576 -> ~235); project = +create/switch/list as one (244 -> ~155). Old names are GONE from the +toolsets (desktop-only tools; no long-transcript compat needed). The +preview read action still routes through the agent-level GUI callback. +""" +import json +import os +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + + +class TestConsolidatedToolsets(unittest.TestCase): + def test_old_names_gone_new_names_present(self): + from toolsets import TOOLSETS + + ui = TOOLSETS["desktop_ui"]["tools"] + self.assertIn("desktop_preview", ui) + for dead in ("open_preview", "close_preview", "read_preview"): + self.assertNotIn(dead, ui) + proj = TOOLSETS["project"]["tools"] + self.assertEqual(proj, ["desktop_project"]) + + def test_registry_serves_only_new_names(self): + from model_tools import get_tool_definitions + + names = { + t["function"]["name"] + for t in get_tool_definitions( + quiet_mode=True, enabled_toolsets=["desktop_ui", "project"] + ) + } + self.assertIn("desktop_preview", names) + self.assertIn("desktop_project", names) + for dead in ( + "open_preview", "close_preview", "read_preview", + "project_create", "project_switch", "project_list", + ): + self.assertNotIn(dead, names) + + +class TestPreviewHandler(unittest.TestCase): + def test_open_and_close_route_through_desktop_ui(self): + from tools import preview_tool + + sent = [] + with patch("tools.desktop_ui.emit", side_effect=lambda ev, p: sent.append((ev, p)) or True): + r = json.loads(preview_tool._handle_preview({"action": "open", "url": "www.cnn.com"})) + self.assertTrue(r["success"]) + self.assertEqual(r["url"], "https://www.cnn.com") # normalizer kept + r = json.loads(preview_tool._handle_preview({"action": "close"})) + self.assertTrue(r["success"]) + self.assertEqual([e for e, _ in sent], ["preview.open", "preview.close"]) + + def test_read_outside_desktop_session_teaches(self): + from tools import preview_tool + + r = json.loads(preview_tool._handle_preview({"action": "read"})) + self.assertFalse(r.get("success", False)) + + def test_unknown_action_teaches(self): + from tools import preview_tool + + r = json.loads(preview_tool._handle_preview({"action": "zap"})) + self.assertIn("open, close, read", r.get("error", "")) + + +class TestProjectHandler(unittest.TestCase): + def test_dispatch_shapes(self): + import tools.project_tools as pt + + with patch.object(pt, "project_list", return_value='{"success": true}') as pl: + pt._handle_project({"action": "list"}) + pl.assert_called_once() + with patch.object(pt, "project_create", return_value='{"success": true}') as pc: + pt._handle_project({"action": "create", "name": "X", "path": "C:/tmp"}) + pc.assert_called_once_with(name="X", path="C:/tmp", task_id=None) + with patch.object(pt, "project_switch", return_value='{"success": true}') as ps: + pt._handle_project({"action": "switch", "name": "aurora"}) + ps.assert_called_once_with(project="aurora", task_id=None) + r = json.loads(pt._handle_project({"action": "bogus"})) + self.assertFalse(r["success"]) + + +class TestDietBudget(unittest.TestCase): + def test_desktop_surface_under_budget(self): + """The 15-tool surface serialized to ~15.5K chars (≈3,861 tok); + consolidation + diet brought the (now 11-tool) surface to ~9.2K + chars (≈2,293 tok). Guard: stay under 10.5K chars (≈2,600 tok) — + a bloat regression guard, not a snapshot pin. Chars, not tokens: + tiktoken is not a repo dependency, and the chars/tokens ratio for + these schemas is stable (~4.0).""" + from model_tools import get_tool_definitions + + targets = { + "drive_preview", "tour", "annotate_preview", "setup_mcp", "tip", + "desktop_preview", "desktop_project", "read_window_below", "apply_layout", + "read_terminal", "focus_pane", + } + total = 0 + for t in get_tool_definitions(quiet_mode=True, enabled_toolsets=["desktop_ui", "project"]): + f = t["function"] + if f["name"] in targets: + total += len(json.dumps(f, separators=(",", ":"), ensure_ascii=False)) + self.assertLess(total, 10_500, f"desktop tool surface regressed to {total} chars") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tools/test_open_preview_tool.py b/tests/tools/test_open_preview_tool.py index dc2064e86b..03dd375ea7 100644 --- a/tests/tools/test_open_preview_tool.py +++ b/tests/tools/test_open_preview_tool.py @@ -17,11 +17,13 @@ def _reset_emitter(): def test_lives_in_the_gui_surface_toolset(monkeypatch): - """Reaches a desktop client on ANY backend, including one with no - HERMES_DESKTOP in its environment (URL / cloud gateways).""" + import tools.preview_tool # noqa: F401 — registers desktop_preview + """Consolidated (#95681): this module's tool became an action of the + single `desktop_preview` tool in desktop_ui; the old registration is gone and + `preview` reaches a desktop client on ANY backend (no env gate).""" monkeypatch.delenv("HERMES_DESKTOP", raising=False) - entry = registry.get_entry("open_preview") - + assert registry.get_entry("open_preview") is None + entry = registry.get_entry("desktop_preview") assert entry is not None assert entry.toolset == "desktop_ui" assert entry.check_fn is None diff --git a/tests/tools/test_read_preview_tool.py b/tests/tools/test_read_preview_tool.py index 04cb8f93ae..e57f508f07 100644 --- a/tests/tools/test_read_preview_tool.py +++ b/tests/tools/test_read_preview_tool.py @@ -7,10 +7,13 @@ from tools.registry import registry def test_lives_in_the_gui_surface_toolset(monkeypatch): - """Mirrors read_terminal: scoped by toolset, not by the backend's env.""" + import tools.preview_tool # noqa: F401 — registers desktop_preview + """Consolidated (#95681): this module's tool became an action of the + single `desktop_preview` tool in desktop_ui; the old registration is gone and + `preview` reaches a desktop client on ANY backend (no env gate).""" monkeypatch.delenv("HERMES_DESKTOP", raising=False) - entry = registry.get_entry("read_preview") - + assert registry.get_entry("read_preview") is None + entry = registry.get_entry("desktop_preview") assert entry is not None assert entry.toolset == "desktop_ui" assert entry.check_fn is None diff --git a/tests/tui_gateway/test_gui_surface_toolsets.py b/tests/tui_gateway/test_gui_surface_toolsets.py index 7a12395d7c..b22f04feb0 100644 --- a/tests/tui_gateway/test_gui_surface_toolsets.py +++ b/tests/tui_gateway/test_gui_surface_toolsets.py @@ -19,12 +19,10 @@ from toolsets import TOOLSETS, resolve_toolset GUI_TOOLS = { "annotate_preview", - "close_preview", + "desktop_preview", "drive_preview", "close_terminal", "focus_pane", - "open_preview", - "read_preview", "read_terminal", "read_window_below", "react_to_message", diff --git a/tools/annotate_preview_tool.py b/tools/annotate_preview_tool.py index 71747798b7..eebe33ed5f 100644 --- a/tools/annotate_preview_tool.py +++ b/tools/annotate_preview_tool.py @@ -89,22 +89,14 @@ def annotate_preview_tool( ANNOTATE_PREVIEW_SCHEMA = { "name": "annotate_preview", "description": ( - "Draw a lasting mark on the page open in the in-app browser / preview " - "pane of the Hermes desktop GUI. Everything drive_preview draws as it " - "works fades on its own; an annotation STAYS until you remove it, so " - "this is how you point at something. Use it to show the user what you " - "found ('here are the three cheapest'), flag what you are about to " - "change before you change it, or keep your place while you work " - "elsewhere on the page. Address elements by the same refs " - "drive_preview action='elements' hands back. action='add' outlines an " - "element and gives it an optional short label; 'hold' freezes the WHOLE " - "visible field at once — every element the page offers, outlined and " - "named — which is the " - "picture drive_preview flashes as it works, made to stay; 'remove' " - "takes one down; 'clear' takes them all down. Annotations follow their element " - "as the page scrolls and disappear if it does, so a navigation clears " - "them for you. Keep labels to a word or two — they are drawn on the " - "page, not read aloud." + "Leave a LASTING mark on the preview-pane page (drive_preview's own " + "marks fade; annotations stay until removed) — point at findings, " + "flag what you're about to change, keep your place. Use the refs " + "from drive_preview action='elements'. add: outline one element " + "(optional short label — a word or two, drawn on the page). hold: " + "freeze the whole visible field, every element outlined and named. " + "remove/clear: take one/all down. Marks follow their element on " + "scroll; navigation clears them." ), "parameters": { "type": "object", @@ -112,23 +104,19 @@ ANNOTATE_PREVIEW_SCHEMA = { "action": { "type": "string", "enum": list(ACTIONS), - "description": ( - "'add' marks one element, 'hold' freezes the whole visible " - "field, 'remove' takes one down, 'clear' takes them all " - "down. Defaults to 'add'." - ), + "description": "Defaults to 'add'.", }, "ref": { "type": "string", - "description": "Element reference from drive_preview action='elements' (e.g. 'btn-sign-in').", + "description": "Ref from drive_preview elements.", }, "selector": { "type": "string", - "description": "CSS selector, as a fallback when no ref fits. Prefer ref.", + "description": "CSS selector fallback. Prefer ref.", }, "label": { "type": "string", - "description": "Short caption drawn on the mark, e.g. 'cheapest'. Optional.", + "description": "Optional caption, e.g. 'cheapest'.", }, }, "required": [], diff --git a/tools/apply_layout_tool.py b/tools/apply_layout_tool.py index b1eb4ab32b..a6f8c89edc 100644 --- a/tools/apply_layout_tool.py +++ b/tools/apply_layout_tool.py @@ -44,13 +44,10 @@ def apply_layout_tool(preset: str) -> str: APPLY_LAYOUT_SCHEMA = { "name": "apply_layout", "description": ( - "Apply a saved layout preset to the Hermes desktop app when the user asks to " - "rearrange the workspace — e.g. \"set up my layout for coding\", \"give me a " - "focused view\", \"put the terminal front and center\". Built-in presets: " - "default (chat + sidebars), focus (chat only), terminal-deck (terminal " - "forward), quad (four zones). Plugin and user-saved presets are addressed by " - "their id. To reveal a single pane without rearranging everything, use " - "focus_pane instead." + "Apply a saved layout preset to the Hermes desktop app when the user " + "asks to rearrange the workspace. Built-ins: default (chat + " + "sidebars), focus (chat only), terminal-deck, quad; plugin/user " + "presets by id. To reveal ONE pane, use focus_pane instead." ), "parameters": { "type": "object", diff --git a/tools/close_preview_tool.py b/tools/close_preview_tool.py index 3fa15b298f..d9727155dd 100644 --- a/tools/close_preview_tool.py +++ b/tools/close_preview_tool.py @@ -55,10 +55,5 @@ CLOSE_PREVIEW_SCHEMA = { } -registry.register( - name="close_preview", - toolset="desktop_ui", - schema=CLOSE_PREVIEW_SCHEMA, - handler=lambda args, **kw: close_preview_tool(url=args.get("url") or ""), - emoji="🖼️", -) +# Registration removed: consolidated into the `preview` tool (#95681); +# this module keeps its functions for the preview_tool. diff --git a/tools/drive_preview_tool.py b/tools/drive_preview_tool.py index a90b0e8475..653819db34 100644 --- a/tools/drive_preview_tool.py +++ b/tools/drive_preview_tool.py @@ -126,44 +126,29 @@ def drive_preview_tool( ACT_PREVIEW_SCHEMA = { "name": "drive_preview", + # Dieted (#95681): world-building compressed; response-shape teaching + # kept only where skipping it causes wasted calls (delta semantics, + # rebound refs, strobe's burst) — those are pre-effect: a model that + # doesn't know them re-reads pages or loops strobe. "description": ( - "Interact with the page open in the in-app browser / preview pane of " - "the Hermes desktop GUI — the pane open_preview opens beside this " - "chat. This is how you USE a web app the user is looking at: log in, " - "fill a form, click through a flow, page a long document. ALWAYS call " - "action='elements' first to get the current inventory of clickable and " - "typable things — each carries a ref like 'btn-sign-in' or 'inp-email' " - "plus its role, label, and value — then act with that ref instead of " - "guessing a selector. A ref keeps working for as long as the page is " - "open, INCLUDING across a re-render that rebuilds the element, so hold " - "onto the ones you were given. " - "Every action answers with the live url/title plus what moved: the " - "first look at a page returns the full 'elements' inventory, and after " - "that you get a 'delta' instead — 'added' entries in full, 'changed' " - "entries carrying only the ref and whichever of label/value/disabled " - "actually moved, 'removed' and 'rebound' as bare ref lists, and 'same' " - "counting the refs that held. A 'rebound' ref needs NO action from you; it " - "means the page rebuilt that element and your ref already follows it. " - "Anything not mentioned in a delta is unchanged, so do not re-read the " - "page to check. Only a navigation invalidates refs; when told they are " - "stale, call elements again. The mouse " - "and keyboard are real: the pointer travels to its target and the page " - "sees genuine input, so hover menus open and hover-only controls work. " - "Actions: 'elements' (inventory), 'click', 'hover' (move the pointer " - "onto something and leave it there — use it to open a dropdown or " - "reveal a tooltip before clicking inside it), 'type' (set a field's " - "text; submit=true also presses Enter and submits the form), 'scroll' " - "(the page, or a ref'd scrollable), 'press' (a named key), 'strobe' " - "(touch nothing — just rattle the highlight through the page again; " - "'elements' already does this once, so reach for it only when asked to " - "flick, flash, or bounce around the page some more, and note one call " - "runs a whole multi-second burst, so never loop it per element), and " - "'back'/'forward'/'reload' for history. The pane draws every move as " - "it happens so the user can follow along; those marks fade on their " - "own, and annotate_preview is how you leave one up on purpose. Use " - "read_preview when you only " - "need the page's text, and the browser_* tools when the work belongs " - "in a separate automated browser rather than the user's own pane." + "Use the web page open in the desktop preview pane (the one " + "`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS " + "start with action='elements' — it inventories clickable/typable " + "things as refs ('btn-sign-in') with role/label/value; act by ref, " + "not guessed selectors. Refs survive re-renders and only die on " + "navigation (you'll be told they're stale — call elements again). " + "After the first full inventory, actions answer with a DELTA: " + "'added' in full, 'changed' as ref + moved fields, 'removed'/" + "'rebound' as ref lists ('rebound' needs nothing from you — the ref " + "already follows the rebuilt element). Anything unmentioned is " + "unchanged; do not re-read to check. Input is real (pointer travels, " + "hover menus open). Actions: elements, click, hover (park the " + "pointer — opens dropdowns before clicking in), type (submit=true " + "also presses Enter), scroll, press, strobe (visual flourish only — " + "one call runs a multi-second burst; never loop it), back/forward/" + "reload. Moves draw live and fade; annotate_preview leaves a lasting " + "mark. Page text only: desktop_preview action=read. Separate automated " + "browser: browser_* tools." ), "parameters": { "type": "object", @@ -171,41 +156,41 @@ ACT_PREVIEW_SCHEMA = { "action": { "type": "string", "enum": list(ACTIONS), - "description": "What to do. Start with 'elements'.", + "description": "Start with 'elements'.", }, "ref": { "type": "string", - "description": "Element reference from any earlier elements call (e.g. 'btn-sign-in'). Good until the page navigates.", + "description": "Element ref from an earlier elements call.", }, "selector": { "type": "string", - "description": "CSS selector, as a fallback when no ref fits. Prefer ref.", + "description": "CSS selector fallback. Prefer ref.", }, - "text": {"type": "string", "description": "For 'type': the text to enter."}, + "text": {"type": "string", "description": "type: the text."}, "submit": { "type": "boolean", - "description": "For 'type': press Enter and submit the owning form afterwards.", + "description": "type: press Enter + submit the form after.", }, "key": { "type": "string", - "description": "For 'press': the key name, e.g. 'Enter', 'Escape', 'ArrowDown'.", + "description": "press: key name ('Enter', 'Escape', 'ArrowDown').", }, "amount": { "type": "integer", - "description": "For 'scroll': pixels to scroll (negative scrolls up). Defaults to about one screen.", + "description": "scroll: pixels (negative = up; default ~one screen).", }, "to": { "type": "string", "enum": list(SCROLL_TO), - "description": "For 'scroll': jump to the top or bottom instead of a distance.", + "description": "scroll: jump to top/bottom instead.", }, "max": { "type": "integer", - "description": "For 'elements': cap the inventory. Defaults to the per-call maximum.", + "description": "elements: cap the inventory.", }, "full": { "type": "boolean", - "description": "For 'elements': re-read the whole page instead of a delta. Rarely needed.", + "description": "elements: full re-read instead of a delta. Rarely needed.", }, }, "required": ["action"], diff --git a/tools/focus_pane_tool.py b/tools/focus_pane_tool.py index e15431281b..b729e93b5e 100644 --- a/tools/focus_pane_tool.py +++ b/tools/focus_pane_tool.py @@ -36,12 +36,9 @@ def focus_pane_tool(pane: str) -> str: FOCUS_PANE_SCHEMA = { "name": "focus_pane", "description": ( - "Reveal and focus a pane in the Hermes desktop app when the user asks to " - "see it — e.g. \"show me the terminal\", \"open the file browser\", \"show " - "the diff\". Panes: chat (the conversation), files (project file browser), " - "terminal (embedded shell), review (git diff), sessions (the session list). " - "To show a URL or file in the preview pane, use open_preview; to close it, " - "use close_preview." + "Reveal and focus a Hermes desktop pane when the user asks to see it: " + "chat, files, terminal, review (git diff), or sessions. For URLs/" + "files use the desktop_preview tool instead." ), "parameters": { "type": "object", diff --git a/tools/open_preview_tool.py b/tools/open_preview_tool.py index 367a02764e..a2c44b14eb 100644 --- a/tools/open_preview_tool.py +++ b/tools/open_preview_tool.py @@ -84,10 +84,5 @@ OPEN_PREVIEW_SCHEMA = { } -registry.register( - name="open_preview", - toolset="desktop_ui", - schema=OPEN_PREVIEW_SCHEMA, - handler=lambda args, **kw: open_preview_tool(url=args.get("url", ""), label=args.get("label", "")), - emoji="🖼️", -) +# Registration removed: consolidated into the `preview` tool (#95681); +# this module keeps its functions for the preview_tool. diff --git a/tools/preview_tool.py b/tools/preview_tool.py new file mode 100644 index 0000000000..2c2181a39e --- /dev/null +++ b/tools/preview_tool.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""The `desktop_preview` tool — the preview pane beside the chat, as ONE tool. + +Consolidation (#95681, maintainer-directed): open_preview, close_preview, +and read_preview each re-taught "the preview pane beside the chat" world; +one action enum states it once (576 -> ~210 tok). The read action keeps +its agent-callback dispatch (agent_runtime_helpers routes action=read +through agent.read_preview_callback, same as read_preview did). + +Lives in the ``desktop_ui`` toolset — desktop-app sessions only. +""" + +import json + +from tools import desktop_ui +from tools.open_preview_tool import _normalize_target, open_preview_tool +from tools.registry import registry, tool_error + + +def preview_open(url: str, label: str = "") -> str: + return open_preview_tool(url=url, label=label) + + +def preview_close(url: str = "") -> str: + target = _normalize_target(url or "") + try: + ok = desktop_ui.emit("preview.close", {"url": target}) + except Exception as exc: # noqa: BLE001 + return tool_error(f"Failed to close the preview: {exc}") + if not ok: + return tool_error("The preview pane is only available in the Hermes desktop app.") + return json.dumps({"success": True, "closed": target or "all"}, ensure_ascii=False) + + +def _handle_preview(args, **kw): + """Non-read actions only: action=read is dispatched at the agent level + (needs the GUI callback), mirroring the old read_preview special path.""" + action = (args.get("action") or "").strip() + if action == "open": + return preview_open(url=args.get("url", ""), label=args.get("label", "")) + if action == "close": + return preview_close(url=args.get("url", "")) + if action == "read": + return tool_error( + "preview read must run inside a desktop session (no GUI callback here)." + ) + return tool_error("action must be one of: open, close, read.") + + +PREVIEW_SCHEMA = { + "name": "desktop_preview", + "description": ( + "The preview pane beside the chat in the Hermes desktop app. open: show " + "a web URL (bare domains fine), a localhost dev server, or a file path " + "(HTML renders live) — opens for the current window only. close: dismiss " + "the whole pane, or one tab via url. read: what the pane currently shows " + "— returns {kind, url, title, text, start, end, total_chars}; a Browser " + "tab's text is the rendered page's visible text, paged with start/count " + "(char offsets); a file tab answers identity only (read the file with " + "read_file)." + ), + "parameters": { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["open", "close", "read"]}, + "url": { + "type": "string", + "description": "open: the target. close: one tab (omit for the whole pane).", + }, + "label": {"type": "string", "description": "open: optional tab label."}, + "start": {"type": "integer", "description": "read: 0-indexed char offset."}, + "count": {"type": "integer", "description": "read: chars to return (capped per read)."}, + }, + "required": ["action"], + }, +} + + +registry.register( + name="desktop_preview", + toolset="desktop_ui", + schema=PREVIEW_SCHEMA, + handler=_handle_preview, + emoji="🖼️", +) diff --git a/tools/project_tools.py b/tools/project_tools.py index 3b4bc70a0f..dc4642a0ba 100644 --- a/tools/project_tools.py +++ b/tools/project_tools.py @@ -139,59 +139,43 @@ def project_switch(project: str, task_id: Optional[str] = None) -> str: return json.dumps({"success": True, "id": proj.id, "slug": proj.slug, "name": proj.name, "primary_path": primary}) -registry.register( - name="project_list", - toolset="project", - schema={ - "name": "project_list", - "description": "List the desktop Projects (named workspaces) and which one is active.", - "parameters": {"type": "object", "properties": {}}, - }, - handler=lambda args, **kw: project_list(task_id=kw.get("task_id")), -) +def _handle_project(args, **kw): + action = (args.get("action") or "").strip() + tid = kw.get("task_id") + if action == "list": + return project_list(task_id=tid) + if action == "create": + return project_create(name=args.get("name", ""), path=args.get("path"), task_id=tid) + if action == "switch": + return project_switch(project=args.get("name", ""), task_id=tid) + return json.dumps({"success": False, "error": "action must be one of: create, switch, list."}) + +# Consolidated (#95681, maintainer-directed): project_list/create/switch each +# re-taught "desktop Projects (named workspaces)"; one action enum says it +# once (244 -> ~145 tok). registry.register( - name="project_create", + name="desktop_project", toolset="project", schema={ - "name": "project_create", + "name": "desktop_project", "description": ( - "Create a desktop Project (a named workspace) and switch this chat into it. " - "Pass `path` to anchor it to a repo/folder — this chat's workspace moves there " - "and the sidebar follows. Use when starting work in a new repo/folder; this is " - "the intentional way to move the session, not `cd`." + "Desktop Projects (named workspaces). create: make one and switch " + "this chat into it — pass path to anchor it to a repo/folder (the " + "chat's workspace moves there, the sidebar follows). switch: move " + "this chat into an existing project by name/slug/id — the " + "intentional way to move the session, not `cd`. list: all " + "projects + which is active." ), "parameters": { "type": "object", "properties": { - "name": {"type": "string", "description": "Human name, e.g. 'Aurora Demo'"}, - "path": {"type": "string", "description": "Primary repo/folder to anchor the project to"}, + "action": {"type": "string", "enum": ["create", "switch", "list"]}, + "name": {"type": "string", "description": "create: human name. switch: name, slug, or id."}, + "path": {"type": "string", "description": "create: repo/folder to anchor to."}, }, - "required": ["name"], + "required": ["action"], }, }, - handler=lambda args, **kw: project_create( - name=args.get("name", ""), path=args.get("path"), task_id=kw.get("task_id") - ), -) - -registry.register( - name="project_switch", - toolset="project", - schema={ - "name": "project_switch", - "description": ( - "Switch this chat into an existing desktop Project (by name, slug, or id). " - "Moves the session's workspace to the project's primary folder and the sidebar " - "follows. The intentional way to move between projects, not `cd`." - ), - "parameters": { - "type": "object", - "properties": { - "project": {"type": "string", "description": "Project name, slug, or id"}, - }, - "required": ["project"], - }, - }, - handler=lambda args, **kw: project_switch(project=args.get("project", ""), task_id=kw.get("task_id")), + handler=_handle_project, ) diff --git a/tools/read_preview_tool.py b/tools/read_preview_tool.py index b4cf19693f..53111ab148 100644 --- a/tools/read_preview_tool.py +++ b/tools/read_preview_tool.py @@ -81,14 +81,5 @@ READ_PREVIEW_SCHEMA = { } -registry.register( - name="read_preview", - toolset="desktop_ui", - schema=READ_PREVIEW_SCHEMA, - handler=lambda args, **kw: read_preview_tool( - start=args.get("start"), - count=args.get("count"), - callback=kw.get("callback"), - ), - emoji="🔍", -) +# Registration removed: consolidated into the `preview` tool (#95681); +# this module keeps its functions for the agent-level preview action=read dispatch. diff --git a/tools/read_terminal_tool.py b/tools/read_terminal_tool.py index 5cd977c0ea..5cd47a572a 100644 --- a/tools/read_terminal_tool.py +++ b/tools/read_terminal_tool.py @@ -53,12 +53,10 @@ def read_terminal_tool( READ_TERMINAL_SCHEMA = { "name": "read_terminal", "description": ( - "Read what's currently shown in the in-app terminal pane of the Hermes " - "desktop GUI (the embedded shell beside this chat). Call with no arguments " - "to get the visible screen plus the total line count (`total_lines`). To " - "page through scrollback, pass `start_line` (0 = oldest line) and `count`; " - "valid lines are [0, total_lines). Returns JSON: " - "{total_lines, start, end, viewport_rows, cursor_row, text}." + "Read the in-app terminal pane beside this chat. No args = visible " + "screen + total_lines; page scrollback with start_line (0 = oldest) " + "+ count. JSON: {total_lines, start, end, viewport_rows, cursor_row, " + "text}." ), "parameters": { "type": "object", diff --git a/tools/read_window_tool.py b/tools/read_window_tool.py index c5d3156db3..171476edf9 100644 --- a/tools/read_window_tool.py +++ b/tools/read_window_tool.py @@ -43,17 +43,12 @@ def read_window_below_tool(callback: Optional[Callable] = None) -> str: READ_WINDOW_BELOW_SCHEMA = { "name": "read_window_below", "description": ( - "Identify the application window directly underneath (behind) the " - "Hermes desktop window — what the user is working in behind this app. " - "Returns JSON: {window: {app, title, bounds{x,y,width,height}, id}, " - "frontmost: {app, title}, platform}. `title` may be empty when the OS " - "withholds window titles (e.g. macOS without the Screen Recording " - "permission — never prompted for, noted in `note`). Other Hermes " - "windows are skipped: the nearest non-Hermes window is reported. " - "Returns {error, platform} instead where the OS cannot enumerate " - "windows at all (e.g. a Wayland session); `error` says what would fix " - "it, so relay it rather than retrying. " - "Metadata only; this never captures pixels or content of other windows." + "Identify the app window directly behind the Hermes desktop window " + "(what the user is working in). JSON: {window: {app, title, bounds, " + "id}, frontmost, platform}. title may be empty when the OS withholds " + "it (noted in `note`); where windows cannot be enumerated at all, " + "{error, platform} says what would fix it — relay that instead of " + "retrying. Metadata only; never captures pixels." ), "parameters": { "type": "object", diff --git a/tools/setup_mcp_tool.py b/tools/setup_mcp_tool.py index 765c5028cd..41f081735c 100644 --- a/tools/setup_mcp_tool.py +++ b/tools/setup_mcp_tool.py @@ -74,45 +74,28 @@ def setup_mcp_tool( SETUP_MCP_SCHEMA = { "name": "setup_mcp", "description": ( - "Propose an MCP server to the user as an inline consent card in the " - "Hermes desktop chat. The card lets them install a catalog entry, " - "re-enable a disabled server, or run an OAuth login — right there, " - "without opening the Capabilities tab — and blocks until they act or " - "decline. Use when the user asks to add/set up an MCP (e.g. \"add the " - "linear mcp\"), or when a task clearly needs one that is missing or " - "unauthorized. Never call it twice for the same server after a " - "decline. Returns JSON {status: installed|enabled|authorized|declined|" - "unanswered|error, server, detail?, tools?}. On declined/unanswered, " - "continue without the server. Catalog names: run `hermes mcp catalog` " - "in the terminal to list them." + "Propose an MCP server as an inline consent card (install a catalog " + "entry, re-enable a disabled server, or run OAuth); blocks until the " + "user acts. Use when they ask to add an MCP or a task clearly needs " + "a missing one. Never re-ask after a decline — on declined/" + "unanswered, continue without it. Catalog names: `hermes mcp " + "catalog` in the terminal." ), "parameters": { "type": "object", "properties": { "server": { "type": "string", - "description": ( - "The server's catalog name (for install) or its name in " - "mcp_servers config (for enable/authorize)." - ), + "description": "Catalog name (install) or mcp_servers config name (enable/authorize).", }, "action": { "type": "string", "enum": ["install", "enable", "authorize"], - "description": ( - "install: add a catalog entry (prompts for any required " - "keys). enable: re-enable a disabled configured server. " - "authorize: run the OAuth browser flow for a configured " - "server. Defaults to install." - ), + "description": "Defaults to install.", }, "reason": { "type": "string", - "description": ( - "One short sentence shown on the card: why this server " - "helps right now (e.g. \"To read the JIRA ticket you " - "linked\")." - ), + "description": "One sentence on the card: why this helps right now.", }, }, "required": ["server"], diff --git a/tools/tip_tool.py b/tools/tip_tool.py index 2b2e6807ed..ab80092f17 100644 --- a/tools/tip_tool.py +++ b/tools/tip_tool.py @@ -62,38 +62,32 @@ def tip_tool(text: str, selector: str, title: str = "", side: str = "") -> str: TIP_SCHEMA = { "name": "tip", "description": ( - "Point at one thing in the Hermes desktop UI with a small accent-lit " - "bubble and an arrow — no dimming, no spotlight, no Next/Prev. Reach " - "for it when a sentence would be clearer with a finger on the thing " - "it's about: 'the model name is a button', 'your files are in here'. " - "Call tour(action='targets') first to see what's on screen and prefer " - "a target reporting `stable: true`; never guess a selector. One tip at " - "a time — a new one replaces the last. Say the same thing in chat as " - "well; the bubble is a pointer, not the message. Use it sparingly: a " - "bubble on every turn is what makes people stop reading them." + "Point at one thing in the desktop UI with a small arrow bubble (no " + "dimming, no tour chrome) — for when a sentence is clearer with a " + "finger on its subject. Get selectors from tour(action='targets'), " + "prefer stable:true, never guess. One tip at a time (new replaces " + "last); say the same thing in chat too — the bubble is a pointer, " + "not the message. Sparingly: a bubble every turn stops being read." ), "parameters": { "type": "object", "properties": { "text": { "type": "string", - "description": "The one line the bubble says. Keep it to a sentence.", + "description": "The one-sentence bubble text.", }, "selector": { "type": "string", - "description": ( - "CSS selector of the element the arrow points at, from " - "tour(action='targets')." - ), + "description": "Selector from tour targets.", }, "title": { "type": "string", - "description": "Optional short heading above the text.", + "description": "Optional heading.", }, "side": { "type": "string", "enum": list(SIDES), - "description": "Preferred side of the element. Omit for 'top'; it flips at a screen edge either way.", + "description": "Omit for 'top'; flips at screen edges.", }, }, "required": ["text", "selector"], diff --git a/tools/tour_tool.py b/tools/tour_tool.py index 9893020ba5..2ccff61002 100644 --- a/tools/tour_tool.py +++ b/tools/tour_tool.py @@ -112,37 +112,33 @@ _STEP_SCHEMA = { "properties": { "selector": { "type": "string", - "description": "CSS selector of the element this step highlights. Omit for a centered narration-only step.", + "description": "Element to highlight; omit = centered narration.", }, "title": {"type": "string", "description": "Popover title."}, - "text": {"type": "string", "description": "Popover body text."}, + "text": {"type": "string", "description": "Popover body."}, "side": { "type": "string", "enum": list(SIDES), - "description": "Preferred popover side. Omit to auto-place.", + "description": "Popover side; omit to auto-place.", }, }, } TOUR_SCHEMA = { "name": "tour", + # Dieted (#95681): targets-first flow + stable-selector preference kept + # (pre-effect: skipping them means guessed selectors on re-rendering UI). "description": ( - "Give a live guided tour in the Hermes desktop GUI: dim the screen, " - "highlight an element, and attach a popover with your own title/text. " - "Works on two surfaces — 'app' (the Hermes app itself) and 'preview' " - "(whatever page is open in the in-app browser, so any web app can be " - "toured). ALWAYS call action='targets' first to discover what is on " - "screen instead of guessing selectors; each target reports " - "`stable: true` when its selector keys off identity (data-tour, id, " - "data-testid, aria-label) and survives a re-render — prefer those, and " - "re-scan if a selector stops matching. Then either narrate at your own " - "pace with action='show' (one highlight per call — replaces the " - "previous one; pair each with a chat message describing it), or hand " - "control to the user with action='start' + a steps array (driver.js " - "renders Next/Prev buttons; 'next'/'prev' also page it " - "programmatically). action='stop' clears the tour. Use when the user " - "asks how something works, where something is, or for a walkthrough of " - "an app or workflow." + "Guided tour in the desktop GUI: dim the screen, highlight an " + "element, attach a titled popover. Surfaces: 'app' (Hermes itself) " + "or 'preview' (the page in the preview pane). ALWAYS call " + "action='targets' first — prefer targets marked stable:true (their " + "selectors survive re-renders); re-scan if one stops matching. Then " + "narrate with action='show' (one highlight per call, replaces the " + "last — pair each with a chat message) or hand over with " + "action='start' + steps (user gets Next/Prev; 'next'/'prev' also " + "page it). 'stop' clears. Use for how-does-X-work / where-is-Y " + "walkthroughs." ), "parameters": { "type": "object", @@ -150,32 +146,32 @@ TOUR_SCHEMA = { "action": { "type": "string", "enum": list(ACTIONS), - "description": "targets: list tourable elements. show: highlight one element. start: begin a multi-step user-paced tour. next/prev: page a started tour. stop: end the tour.", + "description": "targets first; show narrates; start hands over.", }, "surface": { "type": "string", "enum": list(SURFACES), - "description": "Where the tour runs: 'app' (Hermes desktop UI, default) or 'preview' (the page in the in-app browser pane).", + "description": "'app' (default) or 'preview'.", }, "selector": { "type": "string", - "description": "For show: CSS selector of the element to highlight (from action='targets', preferring a stable one). Omit for a centered narration popover.", + "description": "show: selector from targets (prefer stable). Omit = centered narration.", }, - "title": {"type": "string", "description": "For show: popover title."}, - "text": {"type": "string", "description": "For show: popover body text."}, + "title": {"type": "string", "description": "show: popover title."}, + "text": {"type": "string", "description": "show: popover body."}, "side": { "type": "string", "enum": list(SIDES), - "description": "For show: preferred popover side. Omit to auto-place.", + "description": "show: popover side; omit to auto-place.", }, "steps": { "type": "array", "items": _STEP_SCHEMA, - "description": "For start: the ordered tour steps.", + "description": "start: ordered steps.", }, "step_index": { "type": "integer", - "description": "For start: 0-indexed step to begin at (default 0).", + "description": "start: 0-indexed first step.", }, }, "required": ["action"], diff --git a/toolsets.py b/toolsets.py index 55e59d7b22..2ddd6461c1 100644 --- a/toolsets.py +++ b/toolsets.py @@ -236,7 +236,7 @@ TOOLSETS = { "project": { "description": "Desktop Projects — create/switch named workspaces (GUI sessions only)", - "tools": ["project_list", "project_create", "project_switch"], + "tools": ["desktop_project"], "includes": [] }, @@ -253,7 +253,7 @@ TOOLSETS = { "description": "Desktop GUI affordances — in-app terminal/browser panes, pane focus, reactions (GUI sessions only)", "tools": [ "read_terminal", "close_terminal", - "open_preview", "close_preview", "read_preview", "drive_preview", "annotate_preview", + "desktop_preview", "drive_preview", "annotate_preview", "read_window_below", "focus_pane", "react_to_message", "setup_mcp", "tour", "tip",