e83816a4d1
For each issue anchor present in BASE 63279301bc non-test .py and absent on HEAD, the BASE comment/docstring block was re-attached at the HEAD location of the code it explained (matched by the distinctive code line / enclosing def). Sentences already covered by an existing HEAD comment were deduped; the issue number always survives. Insert-only: no code lines changed.
172 lines
11 KiB
Python
172 lines
11 KiB
Python
"""Input side of the cua-driver backend: delivery-mode handling and the pointer / keyboard /
|
|
value-setter methods (mixed into ``CuaDriverBackend``)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
|
|
|
|
from tools.computer_use.backend import ActionResult
|
|
from tools.computer_use.cua_backend_parse import _parse_key_combo
|
|
|
|
_NO_TARGET_MSG = "No active window — call capture() first."
|
|
_BTF_UNSUPPORTED_MSG = "The connected cua-driver does not advertise the standalone bring_to_front tool."
|
|
_FOREGROUND_UNSUPPORTED_MSG = ("The connected cua-driver action schema does not accept delivery_mode, so foreground "
|
|
"delivery is unavailable. Use another verified rung without assuming the reported "
|
|
"package version describes the live schema.")
|
|
# (what, extra args) pointer addressing form; ``extra`` is None when the caller did not supply that form and
|
|
# may be a callable when computing it has side effects (capability probes) that must follow the refusal checks.
|
|
_Variant = Tuple[str, Union[None, Dict[str, Any], Callable[[], Dict[str, Any]]]]
|
|
|
|
def _refuse(action: str, message: str, **fields: Any) -> ActionResult:
|
|
return ActionResult(ok=False, action=action, message=message, **fields)
|
|
|
|
|
|
class _InputMixin:
|
|
"""Pointer / keyboard / value-setter actions against the sticky target."""
|
|
|
|
def _target_args(self, action: str, *, need_window: bool = False) -> Tuple[Optional[ActionResult], Dict[str, Any]]:
|
|
"""``(refusal, base args)`` for an input action against the sticky target."""
|
|
if self._active_pid is None or (need_window and self._active_window_id is None):
|
|
return _refuse(action, _NO_TARGET_MSG), {}
|
|
return None, {"pid": self._active_pid, **({"window_id": self._active_window_id} if need_window else {})}
|
|
|
|
def _pointer_args(self, tool: str, args: Dict[str, Any], variants: Sequence[_Variant],
|
|
missing_msg: Optional[str]) -> Optional[ActionResult]:
|
|
"""Fill *args* from the first supplied addressing variant (element or coordinates) plus ``window_id``; refuse
|
|
when the target has a pid but no window_id yet. No variant -> refuse with *missing_msg* (None = bare window)."""
|
|
for what, extra in variants:
|
|
if extra is not None:
|
|
if self._active_window_id is None:
|
|
return _refuse(tool, f"No active window_id for {what}.")
|
|
args.update(extra() if callable(extra) else extra, window_id=self._active_window_id)
|
|
return None
|
|
return _refuse(tool, missing_msg) if missing_msg else None
|
|
|
|
def _apply_delivery(self, action: str, args: Dict[str, Any], delivery_mode: Optional[str]) -> Optional[ActionResult]:
|
|
"""Attach delivery_mode to an input-action args dict. Background is the default and needs no flag.
|
|
Foreground is only sent when the live action schema accepts it; on an older driver we refuse with
|
|
``foreground_unsupported`` instead of silently downgrading to background (which would land input
|
|
where the model didn't expect).
|
|
|
|
Returns an ActionResult to short-circuit on refusal, or None to proceed. See
|
|
NousResearch/hermes-agent#67052 phase B.
|
|
"""
|
|
if not delivery_mode or delivery_mode == "background":
|
|
return None
|
|
if delivery_mode != "foreground":
|
|
return _refuse(action, f"unknown delivery_mode {delivery_mode!r} — use background|foreground.",
|
|
code="bad_delivery_mode")
|
|
if not self._session.supports_input_property(action, "delivery_mode"):
|
|
return _refuse(action, _FOREGROUND_UNSUPPORTED_MSG, code="foreground_unsupported", delivery_mode="foreground")
|
|
args["delivery_mode"] = "foreground"
|
|
return None
|
|
|
|
def _run_input_action(self, action: str, args: Dict[str, Any], delivery_mode: Optional[str],
|
|
bring_to_front: bool) -> ActionResult:
|
|
"""Apply one delivery rung, optionally focusing via its own tool. ``bring_to_front`` is never an
|
|
input-action property: when requested, the separately approved standalone focus action runs first,
|
|
then the original foreground input runs unchanged."""
|
|
refusal = self._apply_delivery(action, args, delivery_mode)
|
|
if refusal is not None:
|
|
return refusal
|
|
if bring_to_front:
|
|
if delivery_mode != "foreground":
|
|
return _refuse(action, "bring_to_front requires delivery_mode='foreground'.",
|
|
code="bring_to_front_requires_foreground")
|
|
if not self._session._has_tool("bring_to_front"):
|
|
return _refuse(action, _BTF_UNSUPPORTED_MSG, code="bring_to_front_unsupported", delivery_mode="foreground")
|
|
if self._active_pid is None or self._active_window_id is None:
|
|
return _refuse(action, "Capture an exact target before requesting persistent foreground focus.",
|
|
code="bring_to_front_target_required", delivery_mode="foreground")
|
|
focused = self.bring_to_front(pid=self._active_pid, window_id=self._active_window_id)
|
|
if not focused.ok:
|
|
return focused
|
|
result = self._action(action, args)
|
|
if bring_to_front:
|
|
result.meta["foreground_focus"] = {"invoked": True, "tool": "bring_to_front"}
|
|
return result
|
|
|
|
def click(self, *, element: Optional[int] = None, x: Optional[int] = None, y: Optional[int] = None,
|
|
button: str = "left", click_count: int = 1, modifiers: Optional[List[str]] = None,
|
|
delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult:
|
|
refusal, args = self._target_args("click")
|
|
if refusal is not None:
|
|
return refusal
|
|
# Tool is chosen by click_count only; `button` goes through click's enum (the driver rejects unknown
|
|
# buttons). `right_click` / `middle_click` MCP tools are deprecated aliases and never invoked here.
|
|
# Choose tool by click_count only — single-vs-double — and pass the button through to `click`'s
|
|
# `button` enum (Surface 5 of NousResearch/hermes-agent#47072). cua-driver-rs gained an explicit
|
|
# `button: "left"|"right"|"middle"` arg on `click` in trycua/cua#1961 which rejects unknown buttons;
|
|
# before that, `middle` was silently mapped to a left-click via name-routing through `right_click`.
|
|
button_norm = (button or "left").lower()
|
|
if button_norm not in {"left", "right", "middle"}:
|
|
return _refuse("click", f"unknown button {button!r} — expected left, right, middle.")
|
|
tool, args["button"] = ("double_click" if click_count == 2 else "click"), button_norm
|
|
refusal = self._pointer_args(tool, args, (
|
|
("element_index click", {"element_index": element} if element is not None else None),
|
|
("coordinate click", {"x": x, "y": y} if x is not None and y is not None else None),
|
|
), "click requires element= or x/y.")
|
|
if modifiers:
|
|
args["modifier"] = modifiers
|
|
return refusal if refusal is not None else self._run_input_action(tool, args, delivery_mode, bring_to_front)
|
|
|
|
def drag(self, *, from_element: Optional[int] = None, to_element: Optional[int] = None,
|
|
from_xy: Optional[Tuple[int, int]] = None, to_xy: Optional[Tuple[int, int]] = None,
|
|
button: str = "left", modifiers: Optional[List[str]] = None,
|
|
delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult:
|
|
refusal, args = self._target_args("drag")
|
|
if refusal is None:
|
|
refusal = self._pointer_args("drag", args, (
|
|
("element-based drag", {"from_element": from_element, "to_element": to_element}
|
|
if from_element is not None and to_element is not None else None),
|
|
("coordinate drag", {"from_x": int(from_xy[0]), "from_y": int(from_xy[1]),
|
|
"to_x": int(to_xy[0]), "to_y": int(to_xy[1])}
|
|
if from_xy is not None and to_xy is not None else None),
|
|
), "drag requires from_element/to_element or from_coordinate/to_coordinate.")
|
|
return refusal if refusal is not None else self._run_input_action("drag", args, delivery_mode, bring_to_front)
|
|
|
|
def scroll(self, *, direction: str, amount: int = 3, element: Optional[int] = None,
|
|
x: Optional[int] = None, y: Optional[int] = None, modifiers: Optional[List[str]] = None,
|
|
delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult:
|
|
refusal, args = self._target_args("scroll")
|
|
if refusal is not None:
|
|
return refusal
|
|
args.update(direction=direction, amount=max(1, min(50, amount)))
|
|
# An element without a known window_id is not an addressing form here; scrolling then falls through
|
|
# to the coordinate form or the bare window. Some driver schemas reject x/y on scroll: only send
|
|
# coordinates when the driver advertises support; otherwise it scrolls the targeted window
|
|
# (window_id is still sent for routing).
|
|
xy = lambda: ({"x": x, "y": y} # noqa: E731
|
|
if self._session.supports_capability("input.scroll.coordinates", tool="scroll") else {})
|
|
refusal = self._pointer_args("scroll", args, (
|
|
("element scroll", {"element_index": element}
|
|
if element is not None and self._active_window_id is not None else None),
|
|
("coordinate scroll", xy if x is not None and y is not None else None),
|
|
), None)
|
|
return refusal if refusal is not None else self._run_input_action("scroll", args, delivery_mode, bring_to_front)
|
|
|
|
def type_text(self, text: str, *, delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult:
|
|
refusal, args = self._target_args("type_text", need_window=True)
|
|
return refusal if refusal is not None else self._run_input_action("type_text", {**args, "text": text},
|
|
delivery_mode, bring_to_front)
|
|
|
|
def key(self, keys: str, *, delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult:
|
|
refusal, args = self._target_args("key", need_window=True)
|
|
if refusal is not None:
|
|
return refusal
|
|
key_name, modifiers = _parse_key_combo(keys)
|
|
if not key_name:
|
|
return _refuse("key", f"Could not parse key from '{keys}'.")
|
|
if modifiers: # hotkey requires at least one modifier + one key
|
|
return self._run_input_action("hotkey", {**args, "keys": modifiers + [key_name]}, delivery_mode, bring_to_front)
|
|
return self._run_input_action("press_key", {**args, "key": key_name}, delivery_mode, bring_to_front)
|
|
|
|
def set_value(self, value: str, element: Optional[int] = None) -> ActionResult:
|
|
"""Set a value on an element. Handles AXPopUpButton selects natively."""
|
|
refusal, args = self._target_args("set_value", need_window=True)
|
|
if refusal is not None:
|
|
return refusal
|
|
if element is None:
|
|
return _refuse("set_value", "set_value requires element= (element index).")
|
|
return self._action("set_value", {**args, "element_index": element, "value": value})
|