1fcb15f206
The frozen computer_use schema shipped `request_handoff` / `wait_for_human` and the 'take over this screen from the Hermes Desktop app' copy to every host, including macOS and Windows where there is no Bot Desktop: the model learned two actions that can never succeed and a hand-off story the user cannot follow. `tools.computer_use.schema.schema_for_host(supported=...)` is a pure function returning the frozen schema when supported and the same schema minus the handoff actions, their only parameters (`reason`, `grace`) and their copy otherwise; one `_DYNAMIC_SCHEMA_REWRITERS` row applies it when `tools.bot_desktop.runtime.is_supported_host()` is False (browser_navigate's web-hint row is the precedent). Linux keeps the identical object, so prompt-cache parity is untouched there. Test: tests/tools/test_computer_use_schema_host.py — supported=False has no handoff vocabulary and keeps every other action; supported=True is the frozen schema (import error on bc36ddb5f969; the host is never faked).
241 lines
11 KiB
Python
241 lines
11 KiB
Python
"""Schema for the generic `computer_use` tool (model-facing; value is byte-frozen —
|
|
the schema goes to the model every turn, so prompt-cache parity depends on it).
|
|
|
|
Model-agnostic: any tool-calling model can drive this. Vision-capable models
|
|
should prefer `capture(mode='som')` then `click(element=N)` — much more reliable
|
|
than pixel coordinates, which remain supported for models trained on them.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Dict
|
|
|
|
# Bot Screen handoff: the human takes the bot's headless Linux screen over from the Hermes Desktop app.
|
|
# Only meaningful where a Bot Desktop can exist (Linux gateway hosts); `schema_for_host` strips it
|
|
# elsewhere so macOS/Windows/seated-Linux models never learn actions that cannot succeed.
|
|
_HANDOFF_ACTIONS = ("request_handoff", "wait_for_human")
|
|
_HANDOFF_ONLY_PROPERTIES = ("reason", "grace")
|
|
_HANDOFF_ACTION_HINT = (
|
|
" When a login, 2FA, CAPTCHA or payment step needs the human, call "
|
|
"`request_handoff` (with `reason`) so they can take over this screen from the Hermes "
|
|
"Desktop app, then `wait_for_human`; while they hold control every other action is refused."
|
|
)
|
|
_HANDOFF_SECONDS_HINT = " wait_for_human: how long to block for the hand-back (default 600, max 1800)."
|
|
|
|
# One consolidated tool with an `action` discriminator keeps the schema compact
|
|
# and the per-turn token cost low. Property groups: capture (mode, app, pid,
|
|
# window_id) / targeting (element, coordinate, button, modifiers) / drag / scroll /
|
|
# set_value / type-key-wait / focus_app / delivery ladder / return shape.
|
|
_PROPERTIES: Dict[str, Any] = {
|
|
"action": {
|
|
"type": "string",
|
|
"enum": [
|
|
"capture",
|
|
"click",
|
|
"double_click",
|
|
"right_click",
|
|
"middle_click",
|
|
"drag",
|
|
"scroll",
|
|
"type",
|
|
"key",
|
|
"set_value",
|
|
"wait",
|
|
"list_apps",
|
|
"list_windows",
|
|
"focus_app",
|
|
"request_handoff",
|
|
"wait_for_human",
|
|
],
|
|
"description": (
|
|
"Which action to perform. `capture` is free (no side effects). All other actions "
|
|
"require approval unless auto-approved. Use `set_value` for select/popup elements and "
|
|
"sliders — it selects the matching option directly without opening the native menu (no "
|
|
"focus steal)." + _HANDOFF_ACTION_HINT
|
|
),
|
|
},
|
|
"mode": {
|
|
"type": "string",
|
|
"enum": ["som", "vision", "ax"],
|
|
"description": (
|
|
"Capture mode. `som` (default) is a screenshot with numbered overlays on every "
|
|
"interactable element plus the AX tree — best for vision models, lets you click by "
|
|
"element index. `vision` is a plain screenshot. `ax` is the accessibility tree only "
|
|
"(no image; useful for text-only models)."
|
|
),
|
|
},
|
|
"app": {
|
|
"type": "string",
|
|
"description": (
|
|
"Optional. Limit capture/action to one app (name e.g. 'Safari', or bundle ID). Omitted "
|
|
"= frontmost window. app='screen' = composited full-screen grab (image only, no "
|
|
"clickable elements); app='desktop' = the OS desktop/shell surface (wallpaper, icons, "
|
|
"taskbar) with its elements."
|
|
),
|
|
},
|
|
"pid": {
|
|
"type": "integer",
|
|
"description": (
|
|
"Optional exact process target for action='capture'. Pair with window_id when "
|
|
"discovery cannot resolve an X11 app."
|
|
),
|
|
},
|
|
"window_id": {
|
|
"type": "integer",
|
|
"description": (
|
|
"Optional exact native window target for action='capture'. Pair with pid when an "
|
|
"external cua-driver list_windows lookup has already identified the window."
|
|
),
|
|
},
|
|
"element": {
|
|
"type": "integer",
|
|
"description": (
|
|
"The 1-based SOM index returned by the last `capture(mode='som')` call. Strongly "
|
|
"preferred over raw coordinates."
|
|
),
|
|
},
|
|
"coordinate": {
|
|
"type": "array",
|
|
"items": {"type": "integer"},
|
|
"minItems": 2,
|
|
"maxItems": 2,
|
|
"description": (
|
|
"Pixel coordinates [x, y] relative to the captured window screenshot (top-left "
|
|
"origin). Only use this if no element index is available."
|
|
),
|
|
},
|
|
"button": {
|
|
"type": "string",
|
|
"enum": ["left", "right", "middle"],
|
|
"description": "Mouse button. Defaults to left.",
|
|
},
|
|
"modifiers": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "string",
|
|
"enum": [
|
|
"cmd",
|
|
"shift",
|
|
"option",
|
|
"alt",
|
|
"ctrl",
|
|
"fn",
|
|
"win",
|
|
"windows",
|
|
"super",
|
|
"meta",
|
|
],
|
|
},
|
|
"description": "Modifier keys held during the action.",
|
|
},
|
|
"from_element": {"type": "integer", "description": "Source element index (drag)."},
|
|
"to_element": {"type": "integer", "description": "Target element index (drag)."},
|
|
"from_coordinate": {
|
|
"type": "array",
|
|
"items": {"type": "integer"},
|
|
"minItems": 2,
|
|
"maxItems": 2,
|
|
"description": "Source [x,y] (drag; use when no element available).",
|
|
},
|
|
"to_coordinate": {
|
|
"type": "array",
|
|
"items": {"type": "integer"},
|
|
"minItems": 2,
|
|
"maxItems": 2,
|
|
"description": "Target [x,y] (drag; use when no element available).",
|
|
},
|
|
"direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "Scroll direction."},
|
|
"amount": {"type": "integer", "description": "Scroll wheel ticks. Default 3."},
|
|
"value": {
|
|
"type": "string",
|
|
"description": (
|
|
"For action='set_value': the value to set on the element. For AXPopUpButton / select "
|
|
"dropdowns, pass the option's display label (e.g. 'Blue'). For sliders and other "
|
|
"AXValue-settable elements, pass the numeric or string value."
|
|
),
|
|
},
|
|
"text": {"type": "string", "description": "Text to type (respects the current layout)."},
|
|
"reason": {"type": "string", "description": "request_handoff: one sentence telling the human what to do on the screen (e.g. 'Sign in to LinkedIn and complete 2FA')."},
|
|
"keys": {
|
|
"type": "string",
|
|
"description": (
|
|
"Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine."
|
|
),
|
|
},
|
|
"seconds": {"type": "number", "description": "wait: seconds to pause (max 30)." + _HANDOFF_SECONDS_HINT},
|
|
"grace": {"type": "number", "description": "wait_for_human: seconds to wait for someone to take over before returning no_takeover (default 60); once a human holds control the full `seconds` applies."},
|
|
"raise_window": {
|
|
"type": "boolean",
|
|
"description": (
|
|
"Only for action='focus_app'. If true, brings the window to front (DISRUPTS the user). "
|
|
"Default false — input is routed to the app without raising, matching the background "
|
|
"co-work model."
|
|
),
|
|
},
|
|
"delivery_mode": {
|
|
"type": "string",
|
|
"enum": ["background", "foreground"],
|
|
"description": (
|
|
"For input actions (click, type, key, drag, scroll). `background` (DEFAULT) delivers "
|
|
"without raising the window or stealing focus. `foreground` briefly fronts the window "
|
|
"then restores focus — a visible change needing its own approval; use it only when a "
|
|
"result's verdict tells you to escalate there. Each result's `verdict` carries the "
|
|
"next step; follow it rather than guessing."
|
|
),
|
|
},
|
|
"bring_to_front": {
|
|
"type": "boolean",
|
|
"description": (
|
|
"Optional and only valid with delivery_mode='foreground'. Explicitly invokes "
|
|
"cua-driver's standalone bring_to_front tool before the input; it is never passed as "
|
|
"an input property. This persistent focus change has a separate approval scope. "
|
|
"Default false."
|
|
),
|
|
},
|
|
"capture_after": {
|
|
"type": "boolean",
|
|
"description": (
|
|
"If true, take a follow-up capture after the action and include it in the response. "
|
|
"Saves a round-trip when you need to verify an action's effect."
|
|
),
|
|
},
|
|
}
|
|
|
|
COMPUTER_USE_SCHEMA: Dict[str, Any] = {
|
|
"name": "computer_use",
|
|
"description": (
|
|
"Drive the desktop via cua-driver — screenshots, mouse, keyboard, scroll, drag — on macOS, "
|
|
"Windows, and Linux. Input is background-FIRST, not background-only: the default delivery "
|
|
"routes to the target window without stealing the user's cursor or focus (works even on "
|
|
"hidden/minimized windows), and when a result's `verdict` says to escalate you climb — "
|
|
"pixel coordinates, or delivery_mode='foreground' (briefly fronts the window; separate "
|
|
"approval). Each result carries a `verdict` with the next step; follow it — never repeat "
|
|
"confirmed input, and re-capture to verify an unverifiable one before retrying. Workflow: "
|
|
"action='capture' (mode='som' gives numbered element overlays), then click by `element` "
|
|
"index; re-capture after state-changing actions (or pass capture_after=true). Image "
|
|
"captures include a shareable `screenshot_path`; deliver it via the platform's MEDIA "
|
|
"syntax when the user asks to see it — not for captures used only for control."
|
|
),
|
|
"parameters": {"type": "object", "properties": _PROPERTIES, "required": ["action"]},
|
|
}
|
|
|
|
def get_computer_use_schema() -> Dict[str, Any]:
|
|
"""Return the generic OpenAI function-calling schema."""
|
|
return COMPUTER_USE_SCHEMA
|
|
|
|
|
|
def schema_for_host(*, supported: bool) -> Dict[str, Any]:
|
|
"""The frozen schema on hosts that can run a Bot Desktop; elsewhere the same schema without the
|
|
handoff actions, their parameters and their copy. Pure: the host decision is the caller's."""
|
|
if supported:
|
|
return COMPUTER_USE_SCHEMA
|
|
props = dict(_PROPERTIES)
|
|
for name in _HANDOFF_ONLY_PROPERTIES:
|
|
props.pop(name)
|
|
action = dict(props["action"])
|
|
action["enum"] = [a for a in action["enum"] if a not in _HANDOFF_ACTIONS]
|
|
action["description"] = action["description"].replace(_HANDOFF_ACTION_HINT, "")
|
|
props["action"] = action
|
|
props["seconds"] = {**props["seconds"], "description": props["seconds"]["description"].replace(_HANDOFF_SECONDS_HINT, "")}
|
|
return {**COMPUTER_USE_SCHEMA, "parameters": {**COMPUTER_USE_SCHEMA["parameters"], "properties": props}}
|