feat(tools): let Hermes point at one thing with the tip tool
The quiet sibling of `tour`, in the same `desktop_ui` toolset and reading the same `tour(action='targets')` discovery call: one bubble with an arrow, for a sentence that would be clearer with a finger on the thing it's about. Dimming the whole app to say "the model name is a button" is the wrong weight. Fire-and-forget rather than a round-trip, because a tip is not a question and blocking the turn on one would stall the reply it belongs to. The renderer enforces the user's opt-out itself, so a stale config read can never put a bubble on a screen that asked for none.
This commit is contained in:
committed by
brooklyn!
parent
baaf304992
commit
911c6c50d3
@@ -8,6 +8,7 @@ import { $gateway } from '@/store/gateway'
|
||||
import { applyDesktopLayoutPreset, revealDesktopPane } from '@/store/pane-focus'
|
||||
import { recordAgentReaction } from '@/store/reactions-local'
|
||||
import { setMessages } from '@/store/session'
|
||||
import { type ActiveTip, showTip } from '@/store/tips'
|
||||
|
||||
import type { GatewayEventContext } from './types'
|
||||
|
||||
@@ -213,6 +214,33 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
if (event.type === 'tip.show') {
|
||||
// tip tool: point the accent bubble at something and say one line about
|
||||
// it. Fire-and-forget — a tip is not a question, and blocking the turn on
|
||||
// one would stall the sentence the agent is in the middle of. The opt-out
|
||||
// is enforced inside showTip, so the renderer is the last word on it even
|
||||
// if a stale config let the tool through. Active session only: a
|
||||
// background turn must never paint on the user's screen (desktop
|
||||
// AGENTS.md: offer, don't hijack).
|
||||
const selector = typeof payload?.selector === 'string' ? payload.selector : ''
|
||||
const text = typeof payload?.text === 'string' ? payload.text : ''
|
||||
|
||||
// A tip with nothing to point at is just a notification, and the app
|
||||
// already has those. Dropping it here also stops a malformed event from
|
||||
// replacing a rotation tip with a bubble that dismisses itself a frame
|
||||
// later.
|
||||
if (isActiveEvent && selector && text) {
|
||||
showTip({
|
||||
side: (payload?.side as ActiveTip['side']) ?? 'top',
|
||||
targets: [selector],
|
||||
text,
|
||||
title: typeof payload?.title === 'string' ? payload.title : undefined
|
||||
})
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
if (event.type === 'pane.reveal') {
|
||||
// Agent revealed a pane via the desktop-gated focus_pane tool, in
|
||||
// response to an explicit user request. Active session only — a
|
||||
|
||||
@@ -124,7 +124,10 @@ export type GatewayEventPayload = {
|
||||
preset?: string
|
||||
// tour.request (tour tool — agent-guided driver.js walkthrough). `action`
|
||||
// and `steps` name the tour verb and step list; `surface` picks the app's
|
||||
// own DOM vs the preview pane's guest page.
|
||||
// own DOM vs the preview pane's guest page. tip.show (tip tool — one accent
|
||||
// bubble with an arrow, no overlay) adds no fields of its own: it reuses
|
||||
// `selector`/`side` here plus `text`/`title`, and carries no request_id
|
||||
// because a tip is fire-and-forget.
|
||||
surface?: string
|
||||
selector?: string
|
||||
side?: string
|
||||
|
||||
114
tests/tools/test_tip_tool.py
Normal file
114
tests/tools/test_tip_tool.py
Normal file
@@ -0,0 +1,114 @@
|
||||
"""Tests for the GUI-surface ``tip`` tool."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tools import tip_tool as tt
|
||||
from tools.registry import registry
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def emitted(monkeypatch):
|
||||
"""Capture what the desktop bridge is asked to send, and say it landed."""
|
||||
sent = []
|
||||
|
||||
def _emit(event, payload):
|
||||
sent.append((event, payload))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(tt.desktop_ui, "emit", _emit)
|
||||
return sent
|
||||
|
||||
|
||||
def test_lives_in_the_gui_surface_toolset(monkeypatch):
|
||||
"""Scoped by toolset, not by the backend's env — see AGENTS.md."""
|
||||
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
|
||||
entry = registry.get_entry("tip")
|
||||
|
||||
assert entry is not None
|
||||
assert entry.toolset == "desktop_ui"
|
||||
|
||||
|
||||
def test_requires_the_desktop_bridge(monkeypatch):
|
||||
"""Outside the desktop GUI there is no emitter — a clear error, no crash."""
|
||||
monkeypatch.setattr(tt.desktop_ui, "emit", lambda _event, _payload: False)
|
||||
|
||||
result = json.loads(tt.tip_tool(text="Over here", selector="#composer"))
|
||||
|
||||
assert "desktop" in result["error"]
|
||||
|
||||
|
||||
def test_needs_both_something_to_say_and_somewhere_to_point(emitted):
|
||||
assert "tip needs text" in json.loads(tt.tip_tool(text="", selector="#a"))["error"]
|
||||
assert "tip needs a selector" in json.loads(tt.tip_tool(text="Hi", selector=" "))["error"]
|
||||
assert not emitted
|
||||
|
||||
|
||||
def test_rejects_an_unknown_side(emitted):
|
||||
result = json.loads(tt.tip_tool(text="Hi", selector="#a", side="diagonal"))
|
||||
|
||||
assert "side must be one of" in result["error"]
|
||||
assert not emitted
|
||||
|
||||
|
||||
def test_emits_the_renderer_event_omitting_unset_fields(emitted):
|
||||
result = json.loads(tt.tip_tool(text=" The model name is a button ", selector=" #model "))
|
||||
|
||||
assert result == {"success": True, "selector": "#model"}
|
||||
assert emitted == [("tip.show", {"selector": "#model", "text": "The model name is a button"})]
|
||||
|
||||
|
||||
def test_carries_title_and_side_when_given(emitted):
|
||||
tt.tip_tool(text="Type here", selector="#composer", title="Composer", side="top")
|
||||
|
||||
assert emitted[0][1] == {
|
||||
"selector": "#composer",
|
||||
"text": "Type here",
|
||||
"title": "Composer",
|
||||
"side": "top",
|
||||
}
|
||||
|
||||
|
||||
def test_bridge_failure_is_reported(monkeypatch):
|
||||
def _boom(_event, _payload):
|
||||
raise RuntimeError("renderer went away")
|
||||
|
||||
monkeypatch.setattr(tt.desktop_ui, "emit", _boom)
|
||||
|
||||
assert "renderer went away" in json.loads(tt.tip_tool(text="Hi", selector="#a"))["error"]
|
||||
|
||||
|
||||
class TestOptOut:
|
||||
"""The user's switch gates the tool, and an absent key reads as ON."""
|
||||
|
||||
def _with_display(self, monkeypatch, display):
|
||||
import hermes_cli.config as config_mod
|
||||
|
||||
monkeypatch.setattr(config_mod, "load_config_readonly", lambda: {"display": display})
|
||||
|
||||
def test_available_by_default(self, monkeypatch):
|
||||
self._with_display(monkeypatch, {})
|
||||
|
||||
assert tt.check_tips_enabled() is True
|
||||
|
||||
def test_withdrawn_when_the_user_opts_out(self, monkeypatch):
|
||||
self._with_display(monkeypatch, {"in_app_tips": False})
|
||||
|
||||
assert tt.check_tips_enabled() is False
|
||||
|
||||
def test_unreadable_config_leaves_the_tool_available(self, monkeypatch):
|
||||
import hermes_cli.config as config_mod
|
||||
|
||||
def _boom():
|
||||
raise OSError("no config here")
|
||||
|
||||
monkeypatch.setattr(config_mod, "load_config_readonly", _boom)
|
||||
|
||||
assert tt.check_tips_enabled() is True
|
||||
|
||||
def test_the_switch_is_what_gates_registration(self):
|
||||
entry = registry.get_entry("tip")
|
||||
|
||||
assert entry is not None
|
||||
assert entry.check_fn is tt.check_tips_enabled
|
||||
@@ -29,6 +29,7 @@ GUI_TOOLS = {
|
||||
"read_window_below",
|
||||
"react_to_message",
|
||||
"setup_mcp",
|
||||
"tip",
|
||||
"tour",
|
||||
}
|
||||
|
||||
|
||||
135
tools/tip_tool.py
Normal file
135
tools/tip_tool.py
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Point at something in the Hermes desktop GUI and say one line about it.
|
||||
|
||||
The quiet sibling of ``tour``. Same durable ``data-tour`` handles, same
|
||||
discovery call (``tour(action="targets")``) — but no scrim, no spotlight, and no
|
||||
Next/Prev. Just an accent-lit bubble with an arrow into whatever the tip is
|
||||
about, which is the right weight for "that button, there" in the middle of a
|
||||
sentence.
|
||||
|
||||
Fire-and-forget, unlike ``tour``: a tip is not a question, so blocking the turn
|
||||
on a round-trip would stall the reply it belongs to. The renderer enforces the
|
||||
user's Settings → Appearance → In-App Tips switch as well, so a stale config
|
||||
read here can never put a bubble on a screen that asked for none.
|
||||
|
||||
Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for
|
||||
desktop-sourced sessions.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from tools import desktop_ui
|
||||
from tools.registry import registry, tool_error
|
||||
|
||||
SIDES = ("top", "right", "bottom", "left")
|
||||
|
||||
|
||||
def tip_tool(text: str, selector: str, title: str = "", side: str = "") -> str:
|
||||
"""Show one tip bubble anchored to ``selector``."""
|
||||
text = (text or "").strip()
|
||||
selector = (selector or "").strip()
|
||||
|
||||
if not text:
|
||||
return tool_error("tip needs text — the one line the bubble says.")
|
||||
|
||||
if not selector:
|
||||
return tool_error(
|
||||
"tip needs a selector to point at. Call tour(action='targets') to see "
|
||||
"what's on screen and prefer a target reporting stable: true."
|
||||
)
|
||||
|
||||
if side and side not in SIDES:
|
||||
return tool_error(f"side must be one of: {', '.join(SIDES)}.")
|
||||
|
||||
payload = {"selector": selector, "text": text}
|
||||
if title:
|
||||
payload["title"] = title
|
||||
if side:
|
||||
payload["side"] = side
|
||||
|
||||
try:
|
||||
ok = desktop_ui.emit("tip.show", payload)
|
||||
except Exception as exc:
|
||||
return tool_error(f"Failed to show the tip: {exc}")
|
||||
if not ok:
|
||||
return tool_error("tip is only available in the Hermes desktop app.")
|
||||
|
||||
return json.dumps({"success": True, "selector": selector}, ensure_ascii=False)
|
||||
|
||||
|
||||
def check_tips_enabled() -> bool:
|
||||
"""The user's own switch (Settings → Appearance → In-App Tips).
|
||||
|
||||
Opt-OUT, so an absent key reads as ON — that matches the desktop's own
|
||||
default, and the renderer only writes the key when the user changes it. The
|
||||
desktop mirrors the toggle into ``display.in_app_tips`` on the CONNECTED
|
||||
gateway's config, so this reads the right value whether that gateway is
|
||||
local, SSH, URL, or cloud.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
display = load_config_readonly().get("display")
|
||||
except Exception:
|
||||
return True
|
||||
if not isinstance(display, dict):
|
||||
return True
|
||||
return bool(display.get("in_app_tips", True))
|
||||
|
||||
|
||||
TIP_SCHEMA = {
|
||||
"name": "tip",
|
||||
"description": (
|
||||
"Point at one thing in the Hermes desktop UI with a small accent-lit "
|
||||
"bubble and an arrow — no dimming, no spotlight, no Next/Prev. Reach "
|
||||
"for it when a sentence would be clearer with a finger on the thing "
|
||||
"it's about: 'the model name is a button', 'your files are in here'. "
|
||||
"Call tour(action='targets') first to see what's on screen and prefer "
|
||||
"a target reporting `stable: true`; never guess a selector. One tip at "
|
||||
"a time — a new one replaces the last. Say the same thing in chat as "
|
||||
"well; the bubble is a pointer, not the message. Use it sparingly: the "
|
||||
"app shows its own tips on a slow rotation, and a bubble on every turn "
|
||||
"is what makes people turn the feature off."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string",
|
||||
"description": "The one line the bubble says. Keep it to a sentence.",
|
||||
},
|
||||
"selector": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"CSS selector of the element the arrow points at, from "
|
||||
"tour(action='targets')."
|
||||
),
|
||||
},
|
||||
"title": {
|
||||
"type": "string",
|
||||
"description": "Optional short heading above the text.",
|
||||
},
|
||||
"side": {
|
||||
"type": "string",
|
||||
"enum": list(SIDES),
|
||||
"description": "Preferred side of the element. Omit for 'top'; it flips at a screen edge either way.",
|
||||
},
|
||||
},
|
||||
"required": ["text", "selector"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
registry.register(
|
||||
name="tip",
|
||||
toolset="desktop_ui",
|
||||
schema=TIP_SCHEMA,
|
||||
handler=lambda args, **kw: tip_tool(
|
||||
text=args.get("text", ""),
|
||||
selector=args.get("selector", ""),
|
||||
title=args.get("title", ""),
|
||||
side=args.get("side", ""),
|
||||
),
|
||||
check_fn=check_tips_enabled,
|
||||
emoji="💡",
|
||||
)
|
||||
@@ -256,7 +256,7 @@ TOOLSETS = {
|
||||
"open_preview", "close_preview", "read_preview", "drive_preview", "annotate_preview",
|
||||
"read_window_below",
|
||||
"focus_pane", "react_to_message",
|
||||
"setup_mcp", "tour",
|
||||
"setup_mcp", "tour", "tip",
|
||||
],
|
||||
"includes": []
|
||||
},
|
||||
|
||||
@@ -8,7 +8,7 @@ description: "Authoritative reference for Hermes built-in tools, grouped by tool
|
||||
|
||||
This page documents Hermes' built-in tools, grouped by toolset. Availability varies by platform, credentials, and enabled toolsets.
|
||||
|
||||
**Quick counts (current registry):** ~86 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 2 terminal tools (`terminal`, `process`), 11 desktop-GUI tools (`read_terminal`, `close_terminal`, `open_preview`, `close_preview`, `read_preview`, `drive_preview`, `annotate_preview`, `read_window_below`, `focus_pane`, `react_to_message`, `tour` — desktop-app sessions only), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`).
|
||||
**Quick counts (current registry):** ~86 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 2 terminal tools (`terminal`, `process`), 12 desktop-GUI tools (`read_terminal`, `close_terminal`, `open_preview`, `close_preview`, `read_preview`, `drive_preview`, `annotate_preview`, `read_window_below`, `focus_pane`, `react_to_message`, `tour`, `tip` — desktop-app sessions only), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`).
|
||||
|
||||
:::tip MCP Tools
|
||||
In addition to built-in tools, Hermes can load tools dynamically from MCP servers. MCP tools appear with the prefix `mcp__<server>__` (e.g., `mcp__github__create_issue` for the `github` MCP server). See [MCP Integration](/user-guide/features/mcp) for configuration.
|
||||
@@ -205,6 +205,7 @@ messaging, and cron sessions.
|
||||
| `focus_pane` | Reveal and focus a pane in the Hermes desktop app (chat, files, terminal, review, sessions). | — |
|
||||
| `react_to_message` | React to a message with a single emoji, iMessage-tapback style. Opt-in via Settings → Appearance (`display.message_reactions`). | — |
|
||||
| `tour` | Give a live guided tour: dim the screen, highlight an element, and attach a narrated popover (driver.js). Works on the Hermes app's own UI and on any page open in the preview pane; `targets` discovers what's on screen, `show` narrates step-by-step, `start` hands the user Next/Prev controls. | — |
|
||||
| `tip` | Point at one element with a small accent bubble and an arrow — the quiet sibling of `tour`, with no dimming, no spotlight, and no Next/Prev. Same `data-tour` handles and the same `tour(action='targets')` discovery call. Opt-out via Settings → Appearance (`display.in_app_tips`). | — |
|
||||
|
||||
### Tours
|
||||
|
||||
@@ -252,6 +253,22 @@ startTour([
|
||||
|
||||
Pass `'preview'` as the second argument to run against the page in the preview pane instead of the app.
|
||||
|
||||
### Tips
|
||||
|
||||
A tip is a tour step without the production: one bubble, one arrow, no scrim and
|
||||
nothing to page through. It's the right weight for a sentence that would be
|
||||
clearer with a finger on the thing it's about — "the model name is a button" —
|
||||
where dimming the whole app would not be.
|
||||
|
||||
The `tip` tool takes the same selectors `tour(action='targets')` reports, so
|
||||
discovery is one call for both, and the durable `data-tour` handles above name
|
||||
targets for either. One tip is on screen at a time; a new one replaces the last.
|
||||
|
||||
The app also shows its own tips, walking a built-in catalog of app features on a
|
||||
slow rotation whenever things are quiet. Closing one with its ✕ retires that tip
|
||||
for good (Settings → Appearance → Reset brings them back), and the same
|
||||
Appearance switch turns the whole feature — rotation and tool alike — off.
|
||||
|
||||
## `todo` toolset
|
||||
|
||||
| Tool | Description | Requires environment |
|
||||
|
||||
Reference in New Issue
Block a user