diff --git a/model_tools.py b/model_tools.py index 4193fb2210..1034bbd180 100644 --- a/model_tools.py +++ b/model_tools.py @@ -374,6 +374,16 @@ def _rewrite_browser_navigate(td: Dict[str, Any], available: set) -> Optional[Di return _fn_def({**td["function"], "description": desc}) +def _rewrite_computer_use(td: Dict[str, Any], available: set) -> Optional[Dict[str, Any]]: + """Strip the Bot Screen handoff (`request_handoff` / `wait_for_human`) where no Bot Desktop can exist + (macOS, Windows): the model would otherwise learn actions that cannot succeed on this host.""" + from tools.bot_desktop.runtime import is_supported_host + from tools.computer_use.schema import schema_for_host + if is_supported_host(): + return td + return _fn_def({**td["function"], **schema_for_host(supported=False)}) + + def _rewrite_browser_exec(td: Dict[str, Any], available: set) -> Optional[Dict[str, Any]]: """browser_exec runs arbitrary host Python: a session without the terminal surface must not regain host execution via the browser toolset. Session-level gate rather @@ -461,6 +471,7 @@ _DYNAMIC_SCHEMA_REWRITERS = { "browser_vault_list": _rewrite_browser_vault, "browser_vault_fill": _rewrite_browser_vault, "delegate_task": _rewrite_delegate_task, + "computer_use": _rewrite_computer_use, } diff --git a/tests/tools/test_computer_use_schema_host.py b/tests/tools/test_computer_use_schema_host.py new file mode 100644 index 0000000000..abc89de75b --- /dev/null +++ b/tests/tools/test_computer_use_schema_host.py @@ -0,0 +1,33 @@ +"""The computer_use schema only teaches the Bot Screen handoff (`request_handoff` / `wait_for_human`, +'take over this screen from the Hermes Desktop app') on hosts that can have a Bot Desktop; elsewhere +the model would learn actions that can never succeed.""" + +from __future__ import annotations + +import json + +from tools.computer_use.schema import COMPUTER_USE_SCHEMA, schema_for_host + +_HANDOFF = ("request_handoff", "wait_for_human") + + +def _actions(schema) -> list[str]: + return schema["parameters"]["properties"]["action"]["enum"] + + +def test_unsupported_host_schema_has_no_handoff_vocabulary(): + schema = schema_for_host(supported=False) + text = json.dumps(schema) + assert not any(action in text for action in _HANDOFF), text + assert "take over this screen" not in text + assert "reason" not in schema["parameters"]["properties"] # request_handoff's only parameter + assert "grace" not in schema["parameters"]["properties"] # wait_for_human's only parameter + # Everything else survives untouched. + assert set(_actions(schema)) == set(_actions(COMPUTER_USE_SCHEMA)) - set(_HANDOFF) + assert schema["parameters"]["required"] == ["action"] + + +def test_supported_host_schema_keeps_the_handoff_and_is_the_frozen_schema(): + schema = schema_for_host(supported=True) + assert set(_HANDOFF) <= set(_actions(schema)) + assert schema is COMPUTER_USE_SCHEMA # byte-frozen value: no per-call copy on the common path diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index 880d9bdc58..9eaf7fbe59 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -10,6 +10,18 @@ from __future__ import annotations from typing import Any, Dict +# Bot Screen handoff: the human takes the bot's headless Linux screen over from the Hermes Desktop app. +# Only meaningful where a Bot Desktop can exist (Linux gateway hosts); `schema_for_host` strips it +# elsewhere so macOS/Windows/seated-Linux models never learn actions that cannot succeed. +_HANDOFF_ACTIONS = ("request_handoff", "wait_for_human") +_HANDOFF_ONLY_PROPERTIES = ("reason", "grace") +_HANDOFF_ACTION_HINT = ( + " When a login, 2FA, CAPTCHA or payment step needs the human, call " + "`request_handoff` (with `reason`) so they can take over this screen from the Hermes " + "Desktop app, then `wait_for_human`; while they hold control every other action is refused." +) +_HANDOFF_SECONDS_HINT = " wait_for_human: how long to block for the hand-back (default 600, max 1800)." + # One consolidated tool with an `action` discriminator keeps the schema compact # and the per-turn token cost low. Property groups: capture (mode, app, pid, # window_id) / targeting (element, coordinate, button, modifiers) / drag / scroll / @@ -39,9 +51,7 @@ _PROPERTIES: Dict[str, Any] = { "Which action to perform. `capture` is free (no side effects). All other actions " "require approval unless auto-approved. Use `set_value` for select/popup elements and " "sliders — it selects the matching option directly without opening the native menu (no " - "focus steal). When a login, 2FA, CAPTCHA or payment step needs the human, call " - "`request_handoff` (with `reason`) so they can take over this screen from the Hermes " - "Desktop app, then `wait_for_human`; while they hold control every other action is refused." + "focus steal)." + _HANDOFF_ACTION_HINT ), }, "mode": { @@ -152,7 +162,7 @@ _PROPERTIES: Dict[str, Any] = { "Key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return', 'escape', 'tab'. Use '+' to combine." ), }, - "seconds": {"type": "number", "description": "wait: seconds to pause (max 30). wait_for_human: how long to block for the hand-back (default 600, max 1800)."}, + "seconds": {"type": "number", "description": "wait: seconds to pause (max 30)." + _HANDOFF_SECONDS_HINT}, "grace": {"type": "number", "description": "wait_for_human: seconds to wait for someone to take over before returning no_takeover (default 60); once a human holds control the full `seconds` applies."}, "raise_window": { "type": "boolean", @@ -212,3 +222,19 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { def get_computer_use_schema() -> Dict[str, Any]: """Return the generic OpenAI function-calling schema.""" return COMPUTER_USE_SCHEMA + + +def schema_for_host(*, supported: bool) -> Dict[str, Any]: + """The frozen schema on hosts that can run a Bot Desktop; elsewhere the same schema without the + handoff actions, their parameters and their copy. Pure: the host decision is the caller's.""" + if supported: + return COMPUTER_USE_SCHEMA + props = dict(_PROPERTIES) + for name in _HANDOFF_ONLY_PROPERTIES: + props.pop(name) + action = dict(props["action"]) + action["enum"] = [a for a in action["enum"] if a not in _HANDOFF_ACTIONS] + action["description"] = action["description"].replace(_HANDOFF_ACTION_HINT, "") + props["action"] = action + props["seconds"] = {**props["seconds"], "description": props["seconds"]["description"].replace(_HANDOFF_SECONDS_HINT, "")} + return {**COMPUTER_USE_SCHEMA, "parameters": {**COMPUTER_USE_SCHEMA["parameters"], "properties": props}}