feat(tools): let Hermes point at one thing with the tip tool

The quiet sibling of `tour`, in the same `desktop_ui` toolset and reading the
same `tour(action='targets')` discovery call: one bubble with an arrow, for a
sentence that would be clearer with a finger on the thing it's about. Dimming
the whole app to say "the model name is a button" is the wrong weight.

Fire-and-forget rather than a round-trip, because a tip is not a question and
blocking the turn on one would stall the reply it belongs to. The renderer
enforces the user's opt-out itself, so a stale config read can never put a
bubble on a screen that asked for none.
This commit is contained in:
Brooklyn Nicholson
2026-08-27 21:27:17 -05:00
committed by brooklyn!
parent baaf304992
commit 911c6c50d3
7 changed files with 301 additions and 3 deletions

View File

@@ -8,6 +8,7 @@ import { $gateway } from '@/store/gateway'
import { applyDesktopLayoutPreset, revealDesktopPane } from '@/store/pane-focus'
import { recordAgentReaction } from '@/store/reactions-local'
import { setMessages } from '@/store/session'
import { type ActiveTip, showTip } from '@/store/tips'
import type { GatewayEventContext } from './types'
@@ -213,6 +214,33 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean {
return true
}
if (event.type === 'tip.show') {
// tip tool: point the accent bubble at something and say one line about
// it. Fire-and-forget — a tip is not a question, and blocking the turn on
// one would stall the sentence the agent is in the middle of. The opt-out
// is enforced inside showTip, so the renderer is the last word on it even
// if a stale config let the tool through. Active session only: a
// background turn must never paint on the user's screen (desktop
// AGENTS.md: offer, don't hijack).
const selector = typeof payload?.selector === 'string' ? payload.selector : ''
const text = typeof payload?.text === 'string' ? payload.text : ''
// A tip with nothing to point at is just a notification, and the app
// already has those. Dropping it here also stops a malformed event from
// replacing a rotation tip with a bubble that dismisses itself a frame
// later.
if (isActiveEvent && selector && text) {
showTip({
side: (payload?.side as ActiveTip['side']) ?? 'top',
targets: [selector],
text,
title: typeof payload?.title === 'string' ? payload.title : undefined
})
}
return true
}
if (event.type === 'pane.reveal') {
// Agent revealed a pane via the desktop-gated focus_pane tool, in
// response to an explicit user request. Active session only — a

View File

@@ -124,7 +124,10 @@ export type GatewayEventPayload = {
preset?: string
// tour.request (tour tool — agent-guided driver.js walkthrough). `action`
// and `steps` name the tour verb and step list; `surface` picks the app's
// own DOM vs the preview pane's guest page.
// own DOM vs the preview pane's guest page. tip.show (tip tool — one accent
// bubble with an arrow, no overlay) adds no fields of its own: it reuses
// `selector`/`side` here plus `text`/`title`, and carries no request_id
// because a tip is fire-and-forget.
surface?: string
selector?: string
side?: string

View File

@@ -0,0 +1,114 @@
"""Tests for the GUI-surface ``tip`` tool."""
import json
import pytest
from tools import tip_tool as tt
from tools.registry import registry
@pytest.fixture
def emitted(monkeypatch):
"""Capture what the desktop bridge is asked to send, and say it landed."""
sent = []
def _emit(event, payload):
sent.append((event, payload))
return True
monkeypatch.setattr(tt.desktop_ui, "emit", _emit)
return sent
def test_lives_in_the_gui_surface_toolset(monkeypatch):
"""Scoped by toolset, not by the backend's env — see AGENTS.md."""
monkeypatch.delenv("HERMES_DESKTOP", raising=False)
entry = registry.get_entry("tip")
assert entry is not None
assert entry.toolset == "desktop_ui"
def test_requires_the_desktop_bridge(monkeypatch):
"""Outside the desktop GUI there is no emitter — a clear error, no crash."""
monkeypatch.setattr(tt.desktop_ui, "emit", lambda _event, _payload: False)
result = json.loads(tt.tip_tool(text="Over here", selector="#composer"))
assert "desktop" in result["error"]
def test_needs_both_something_to_say_and_somewhere_to_point(emitted):
assert "tip needs text" in json.loads(tt.tip_tool(text="", selector="#a"))["error"]
assert "tip needs a selector" in json.loads(tt.tip_tool(text="Hi", selector=" "))["error"]
assert not emitted
def test_rejects_an_unknown_side(emitted):
result = json.loads(tt.tip_tool(text="Hi", selector="#a", side="diagonal"))
assert "side must be one of" in result["error"]
assert not emitted
def test_emits_the_renderer_event_omitting_unset_fields(emitted):
result = json.loads(tt.tip_tool(text=" The model name is a button ", selector=" #model "))
assert result == {"success": True, "selector": "#model"}
assert emitted == [("tip.show", {"selector": "#model", "text": "The model name is a button"})]
def test_carries_title_and_side_when_given(emitted):
tt.tip_tool(text="Type here", selector="#composer", title="Composer", side="top")
assert emitted[0][1] == {
"selector": "#composer",
"text": "Type here",
"title": "Composer",
"side": "top",
}
def test_bridge_failure_is_reported(monkeypatch):
def _boom(_event, _payload):
raise RuntimeError("renderer went away")
monkeypatch.setattr(tt.desktop_ui, "emit", _boom)
assert "renderer went away" in json.loads(tt.tip_tool(text="Hi", selector="#a"))["error"]
class TestOptOut:
"""The user's switch gates the tool, and an absent key reads as ON."""
def _with_display(self, monkeypatch, display):
import hermes_cli.config as config_mod
monkeypatch.setattr(config_mod, "load_config_readonly", lambda: {"display": display})
def test_available_by_default(self, monkeypatch):
self._with_display(monkeypatch, {})
assert tt.check_tips_enabled() is True
def test_withdrawn_when_the_user_opts_out(self, monkeypatch):
self._with_display(monkeypatch, {"in_app_tips": False})
assert tt.check_tips_enabled() is False
def test_unreadable_config_leaves_the_tool_available(self, monkeypatch):
import hermes_cli.config as config_mod
def _boom():
raise OSError("no config here")
monkeypatch.setattr(config_mod, "load_config_readonly", _boom)
assert tt.check_tips_enabled() is True
def test_the_switch_is_what_gates_registration(self):
entry = registry.get_entry("tip")
assert entry is not None
assert entry.check_fn is tt.check_tips_enabled

View File

@@ -29,6 +29,7 @@ GUI_TOOLS = {
"read_window_below",
"react_to_message",
"setup_mcp",
"tip",
"tour",
}

135
tools/tip_tool.py Normal file
View File

@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""Point at something in the Hermes desktop GUI and say one line about it.
The quiet sibling of ``tour``. Same durable ``data-tour`` handles, same
discovery call (``tour(action="targets")``) — but no scrim, no spotlight, and no
Next/Prev. Just an accent-lit bubble with an arrow into whatever the tip is
about, which is the right weight for "that button, there" in the middle of a
sentence.
Fire-and-forget, unlike ``tour``: a tip is not a question, so blocking the turn
on a round-trip would stall the reply it belongs to. The renderer enforces the
user's Settings → Appearance → In-App Tips switch as well, so a stale config
read here can never put a bubble on a screen that asked for none.
Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for
desktop-sourced sessions.
"""
import json
from tools import desktop_ui
from tools.registry import registry, tool_error
SIDES = ("top", "right", "bottom", "left")
def tip_tool(text: str, selector: str, title: str = "", side: str = "") -> str:
"""Show one tip bubble anchored to ``selector``."""
text = (text or "").strip()
selector = (selector or "").strip()
if not text:
return tool_error("tip needs text — the one line the bubble says.")
if not selector:
return tool_error(
"tip needs a selector to point at. Call tour(action='targets') to see "
"what's on screen and prefer a target reporting stable: true."
)
if side and side not in SIDES:
return tool_error(f"side must be one of: {', '.join(SIDES)}.")
payload = {"selector": selector, "text": text}
if title:
payload["title"] = title
if side:
payload["side"] = side
try:
ok = desktop_ui.emit("tip.show", payload)
except Exception as exc:
return tool_error(f"Failed to show the tip: {exc}")
if not ok:
return tool_error("tip is only available in the Hermes desktop app.")
return json.dumps({"success": True, "selector": selector}, ensure_ascii=False)
def check_tips_enabled() -> bool:
"""The user's own switch (Settings → Appearance → In-App Tips).
Opt-OUT, so an absent key reads as ON — that matches the desktop's own
default, and the renderer only writes the key when the user changes it. The
desktop mirrors the toggle into ``display.in_app_tips`` on the CONNECTED
gateway's config, so this reads the right value whether that gateway is
local, SSH, URL, or cloud.
"""
try:
from hermes_cli.config import load_config_readonly
display = load_config_readonly().get("display")
except Exception:
return True
if not isinstance(display, dict):
return True
return bool(display.get("in_app_tips", True))
TIP_SCHEMA = {
"name": "tip",
"description": (
"Point at one thing in the Hermes desktop UI with a small accent-lit "
"bubble and an arrow — no dimming, no spotlight, no Next/Prev. Reach "
"for it when a sentence would be clearer with a finger on the thing "
"it's about: 'the model name is a button', 'your files are in here'. "
"Call tour(action='targets') first to see what's on screen and prefer "
"a target reporting `stable: true`; never guess a selector. One tip at "
"a time — a new one replaces the last. Say the same thing in chat as "
"well; the bubble is a pointer, not the message. Use it sparingly: the "
"app shows its own tips on a slow rotation, and a bubble on every turn "
"is what makes people turn the feature off."
),
"parameters": {
"type": "object",
"properties": {
"text": {
"type": "string",
"description": "The one line the bubble says. Keep it to a sentence.",
},
"selector": {
"type": "string",
"description": (
"CSS selector of the element the arrow points at, from "
"tour(action='targets')."
),
},
"title": {
"type": "string",
"description": "Optional short heading above the text.",
},
"side": {
"type": "string",
"enum": list(SIDES),
"description": "Preferred side of the element. Omit for 'top'; it flips at a screen edge either way.",
},
},
"required": ["text", "selector"],
},
}
registry.register(
name="tip",
toolset="desktop_ui",
schema=TIP_SCHEMA,
handler=lambda args, **kw: tip_tool(
text=args.get("text", ""),
selector=args.get("selector", ""),
title=args.get("title", ""),
side=args.get("side", ""),
),
check_fn=check_tips_enabled,
emoji="💡",
)

View File

@@ -256,7 +256,7 @@ TOOLSETS = {
"open_preview", "close_preview", "read_preview", "drive_preview", "annotate_preview",
"read_window_below",
"focus_pane", "react_to_message",
"setup_mcp", "tour",
"setup_mcp", "tour", "tip",
],
"includes": []
},

View File

@@ -8,7 +8,7 @@ description: "Authoritative reference for Hermes built-in tools, grouped by tool
This page documents Hermes' built-in tools, grouped by toolset. Availability varies by platform, credentials, and enabled toolsets.
**Quick counts (current registry):** ~86 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 2 terminal tools (`terminal`, `process`), 11 desktop-GUI tools (`read_terminal`, `close_terminal`, `open_preview`, `close_preview`, `read_preview`, `drive_preview`, `annotate_preview`, `read_window_below`, `focus_pane`, `react_to_message`, `tour` — desktop-app sessions only), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`).
**Quick counts (current registry):** ~86 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 2 terminal tools (`terminal`, `process`), 12 desktop-GUI tools (`read_terminal`, `close_terminal`, `open_preview`, `close_preview`, `read_preview`, `drive_preview`, `annotate_preview`, `read_window_below`, `focus_pane`, `react_to_message`, `tour`, `tip` — desktop-app sessions only), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`).
:::tip MCP Tools
In addition to built-in tools, Hermes can load tools dynamically from MCP servers. MCP tools appear with the prefix `mcp__<server>__` (e.g., `mcp__github__create_issue` for the `github` MCP server). See [MCP Integration](/user-guide/features/mcp) for configuration.
@@ -205,6 +205,7 @@ messaging, and cron sessions.
| `focus_pane` | Reveal and focus a pane in the Hermes desktop app (chat, files, terminal, review, sessions). | — |
| `react_to_message` | React to a message with a single emoji, iMessage-tapback style. Opt-in via Settings → Appearance (`display.message_reactions`). | — |
| `tour` | Give a live guided tour: dim the screen, highlight an element, and attach a narrated popover (driver.js). Works on the Hermes app's own UI and on any page open in the preview pane; `targets` discovers what's on screen, `show` narrates step-by-step, `start` hands the user Next/Prev controls. | — |
| `tip` | Point at one element with a small accent bubble and an arrow — the quiet sibling of `tour`, with no dimming, no spotlight, and no Next/Prev. Same `data-tour` handles and the same `tour(action='targets')` discovery call. Opt-out via Settings → Appearance (`display.in_app_tips`). | — |
### Tours
@@ -252,6 +253,22 @@ startTour([
Pass `'preview'` as the second argument to run against the page in the preview pane instead of the app.
### Tips
A tip is a tour step without the production: one bubble, one arrow, no scrim and
nothing to page through. It's the right weight for a sentence that would be
clearer with a finger on the thing it's about — "the model name is a button" —
where dimming the whole app would not be.
The `tip` tool takes the same selectors `tour(action='targets')` reports, so
discovery is one call for both, and the durable `data-tour` handles above name
targets for either. One tip is on screen at a time; a new one replaces the last.
The app also shows its own tips, walking a built-in catalog of app features on a
slow rotation whenever things are quiet. Closing one with its ✕ retires that tip
for good (Settings → Appearance → Reset brings them back), and the same
Appearance switch turns the whole feature — rotation and tool alike — off.
## `todo` toolset
| Tool | Description | Requires environment |