125 lines
4.2 KiB
Python
125 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Persistent element annotations in the Hermes desktop GUI's in-app browser.
|
|
|
|
``drive_preview`` draws transient marks (one per action, self-retiring). An
|
|
annotation outlines an element — or, with ``hold``, the whole visible field —
|
|
and stays until the agent removes it. Annotations bind to elements, not
|
|
coordinates: they ride scrolls/reflows and vanish with their element, so a
|
|
navigation clears them. Rides the same ``preview.act`` bridge as
|
|
``drive_preview`` (the renderer resolves ``@e`` refs and owns the overlay).
|
|
Lives in the ``desktop_ui`` toolset, enabled only for desktop-sourced sessions.
|
|
"""
|
|
|
|
import json
|
|
from typing import Callable, Optional
|
|
|
|
from tools.registry import registry, tool_error
|
|
|
|
ACTIONS = ("add", "hold", "remove", "clear")
|
|
|
|
# Verbs the renderer knows, keyed by ours. `clear` is `unpin` with nothing to
|
|
# aim at, which the overlay reads as "all of them".
|
|
WIRE = {"add": "pin", "hold": "hold", "remove": "unpin", "clear": "unpin"}
|
|
|
|
|
|
def annotate_preview_tool(
|
|
action: str = "add",
|
|
ref: Optional[str] = None,
|
|
selector: Optional[str] = None,
|
|
label: Optional[str] = None,
|
|
callback: Optional[Callable] = None,
|
|
) -> str:
|
|
"""Put one annotation up, take one down, or clear them all."""
|
|
if callback is None:
|
|
return tool_error("annotate_preview is only available in the Hermes desktop app.")
|
|
|
|
verb = (action or "add").strip().lower()
|
|
if verb not in ACTIONS:
|
|
return tool_error(f"action must be one of: {', '.join(ACTIONS)}.")
|
|
|
|
if verb in ("add", "remove") and not (ref or selector):
|
|
return tool_error(
|
|
f"{verb} needs a ref from drive_preview action='elements' "
|
|
"(e.g. 'btn-sign-in') or a CSS selector."
|
|
)
|
|
|
|
payload = {
|
|
name: val
|
|
for name, val in (
|
|
("action", WIRE[verb]),
|
|
("ref", None if verb in ("clear", "hold") else ref),
|
|
("selector", None if verb in ("clear", "hold") else selector),
|
|
("text", label),
|
|
)
|
|
if val is not None
|
|
}
|
|
|
|
try:
|
|
raw = callback(payload)
|
|
except Exception as exc:
|
|
return tool_error(f"Failed to annotate the in-app browser: {exc}")
|
|
|
|
if not raw:
|
|
return tool_error(
|
|
"The annotation timed out, or no GUI window answered. "
|
|
"Open a page with open_preview first."
|
|
)
|
|
|
|
try:
|
|
return json.dumps(json.loads(raw), ensure_ascii=False)
|
|
except (TypeError, ValueError):
|
|
return json.dumps({"text": str(raw)}, ensure_ascii=False)
|
|
|
|
|
|
ANNOTATE_PREVIEW_SCHEMA = {
|
|
"name": "annotate_preview",
|
|
"description": (
|
|
"Highlight elements on the preview-pane page, lastingly (drive_preview's own "
|
|
"marks fade; annotations stay until removed) — point at findings, "
|
|
"flag what you're about to change, keep your place. Use the refs "
|
|
"from drive_preview action='elements'. add: outline one element "
|
|
"(optional short label — a word or two, drawn on the page). hold: "
|
|
"freeze the whole visible field, every element outlined and named. "
|
|
"remove/clear: take one/all down. Marks follow their element on "
|
|
"scroll; navigation clears them."
|
|
),
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"action": {
|
|
"type": "string",
|
|
"enum": list(ACTIONS),
|
|
"description": "Defaults to 'add'.",
|
|
},
|
|
"ref": {
|
|
"type": "string",
|
|
"description": "Ref from drive_preview elements.",
|
|
},
|
|
"selector": {
|
|
"type": "string",
|
|
"description": "CSS selector fallback. Prefer ref.",
|
|
},
|
|
"label": {
|
|
"type": "string",
|
|
"description": "Optional caption, e.g. 'cheapest'.",
|
|
},
|
|
},
|
|
"required": [],
|
|
},
|
|
}
|
|
|
|
|
|
registry.register(
|
|
name="annotate_preview",
|
|
toolset="desktop_ui",
|
|
schema=ANNOTATE_PREVIEW_SCHEMA,
|
|
handler=lambda args, **kw: annotate_preview_tool(
|
|
action=args.get("action", "add"),
|
|
ref=args.get("ref"),
|
|
selector=args.get("selector"),
|
|
label=args.get("label"),
|
|
callback=kw.get("callback"),
|
|
),
|
|
emoji="🔖",
|
|
)
|