hermes-agent/tools/annotate_preview_tool.py

#!/usr/bin/env python3
"""Leave a mark on the page in the Hermes desktop GUI's in-app browser.

``drive_preview`` already draws every move it makes — the field it can reach,
a box round its target, the cursor going there — but those are transients:
each one stands for a single action and retires itself. That is right for
narrating a click and no use at all for holding a finding on screen.

This is the deliberate one. An annotation outlines an element — or, with
``hold``, the entire visible field at once — and stays until the agent takes it
down, so it can show the user what it found, flag the fields
it is about to fill, or keep its place while it works elsewhere on the page.
Named for TouchDesigner's Annotate — the labelled box you drop around part of a
network to call it out.

Annotations are bound to elements, not coordinates: they ride scrolls and
reflows, and they go when their element does, so a navigation clears them
without the agent having to.

Rides the same ``preview.act`` bridge as ``drive_preview`` rather than opening
a second channel — the renderer already resolves ``@e`` refs and owns the
overlay, so this is one more verb on a wire that exists.

Lives in the ``desktop_ui`` toolset, which the GUI gateway enables only for
desktop-sourced sessions.
"""

import json
from typing import Callable, Optional

from tools.registry import registry, tool_error

ACTIONS = ("add", "hold", "remove", "clear")

# Verbs the renderer knows, keyed by ours. `clear` is `unpin` with nothing to
# aim at, which the overlay reads as "all of them".
WIRE = {"add": "pin", "hold": "hold", "remove": "unpin", "clear": "unpin"}


def annotate_preview_tool(
    action: str = "add",
    ref: Optional[str] = None,
    selector: Optional[str] = None,
    label: Optional[str] = None,
    callback: Optional[Callable] = None,
) -> str:
    """Put one annotation up, take one down, or clear them all."""
    if callback is None:
        return tool_error("annotate_preview is only available in the Hermes desktop app.")

    verb = (action or "add").strip().lower()
    if verb not in ACTIONS:
        return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")

    if verb in ("add", "remove") and not (ref or selector):
        return tool_error(
            f"{verb} needs a ref from drive_preview action='elements' "
            "(e.g. 'btn-sign-in') or a CSS selector."
        )

    payload = {
        name: val
        for name, val in (
            ("action", WIRE[verb]),
            ("ref", None if verb in ("clear", "hold") else ref),
            ("selector", None if verb in ("clear", "hold") else selector),
            ("text", label),
        )
        if val is not None
    }

    try:
        raw = callback(payload)
    except Exception as exc:
        return tool_error(f"Failed to annotate the in-app browser: {exc}")

    if not raw:
        return tool_error(
            "The annotation timed out, or no GUI window answered. "
            "Open a page with open_preview first."
        )

    try:
        return json.dumps(json.loads(raw), ensure_ascii=False)
    except (TypeError, ValueError):
        return json.dumps({"text": str(raw)}, ensure_ascii=False)


ANNOTATE_PREVIEW_SCHEMA = {
    "name": "annotate_preview",
    "description": (
        "Draw a lasting mark on the page open in the in-app browser / preview "
        "pane of the Hermes desktop GUI. Everything drive_preview draws as it "
        "works fades on its own; an annotation STAYS until you remove it, so "
        "this is how you point at something. Use it to show the user what you "
        "found ('here are the three cheapest'), flag what you are about to "
        "change before you change it, or keep your place while you work "
        "elsewhere on the page. Address elements by the same refs "
        "drive_preview action='elements' hands back. action='add' outlines an "
        "element and gives it an optional short label; 'hold' freezes the WHOLE "
        "visible field at once — every element the page offers, outlined and "
        "named — which is the "
        "picture drive_preview flashes as it works, made to stay; 'remove' "
        "takes one down; 'clear' takes them all down. Annotations follow their element "
        "as the page scrolls and disappear if it does, so a navigation clears "
        "them for you. Keep labels to a word or two — they are drawn on the "
        "page, not read aloud."
    ),
    "parameters": {
        "type": "object",
        "properties": {
            "action": {
                "type": "string",
                "enum": list(ACTIONS),
                "description": (
                    "'add' marks one element, 'hold' freezes the whole visible "
                    "field, 'remove' takes one down, 'clear' takes them all "
                    "down. Defaults to 'add'."
                ),
            },
            "ref": {
                "type": "string",
                "description": "Element reference from drive_preview action='elements' (e.g. 'btn-sign-in').",
            },
            "selector": {
                "type": "string",
                "description": "CSS selector, as a fallback when no ref fits. Prefer ref.",
            },
            "label": {
                "type": "string",
                "description": "Short caption drawn on the mark, e.g. 'cheapest'. Optional.",
            },
        },
        "required": [],
    },
}


registry.register(
    name="annotate_preview",
    toolset="desktop_ui",
    schema=ANNOTATE_PREVIEW_SCHEMA,
    handler=lambda args, **kw: annotate_preview_tool(
        action=args.get("action", "add"),
        ref=args.get("ref"),
        selector=args.get("selector"),
        label=args.get("label"),
        callback=kw.get("callback"),
    ),
    emoji="🔖",
)