From 5f049b517bf01d93e9447c8782ccb6cefcffde06 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Sun, 16 Aug 2026 15:23:18 +0000 Subject: [PATCH] fix(computer-use): make the typed-browser bind/snapshot split discoverable `cua_browser_state` has two branches, chosen implicitly: any call carrying pid or window_id is a *binding* (browser_route.py:252), anything else is a *snapshot*. A binding clears session state, mints fresh tab_ids, returns binding metadata with no page content, and sets verification_required. Nothing in the response says that. A caller that keeps passing pid/window_id - the natural reading of "bind to this window, then read it" - re-binds forever: the tab_id it just received is unbound by the next bind, so every cua_browser_navigate comes back browser_verification_required, and the refusal ("take a fresh snapshot") points at the same call that just re-bound. Observed live as 11 consecutive refused navigates before the model gave up and fell back to foreground SendInput on the address bar. The same confusion silently swallowed include_screenshot: both calls that requested one were bindings, which carry no page content, so the flag had nothing to attach to and was dropped without comment. A binding response now reports snapshot_required, next_step (fresh_browser_state, matching the existing token convention) and a hint naming the exact next call; requesting a screenshot on a binding reports screenshot_deferred instead of dropping it. The verification refusal now says to call cua_browser_state WITHOUT pid/window_id and why re-sending them does not help. The schema documents that include_screenshot applies to snapshots. Behavior of the bind and snapshot branches themselves is unchanged - this is purely about making the split legible to the caller. Unit-tested. Not verified end to end on the reporting host: the driver refuses the bind upstream there (`browser_requires_setup: no owned DevTools endpoint`, and it does not accept a user-launched --remote-debugging-port), so the typed route never reaches this branch. That attach failure is a separate cua-driver issue. Co-Authored-By: Claude Opus 5 (1M context) --- tests/tools/test_computer_use_cua_0_9.py | 59 ++++++++++++++++++++++++ tools/computer_use/browser_route.py | 25 +++++++++- tools/computer_use/schema.py | 5 +- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/tests/tools/test_computer_use_cua_0_9.py b/tests/tools/test_computer_use_cua_0_9.py index 7e8191480b..6f73128da0 100644 --- a/tests/tools/test_computer_use_cua_0_9.py +++ b/tests/tools/test_computer_use_cua_0_9.py @@ -730,6 +730,7 @@ def test_missing_typed_browser_tool_returns_native_fallback_refusal(): def test_existing_profile_prepare_delegates_to_driver_permission_mode(): + """Past the host-side grant floor, the driver still owns the decision.""" driver = _BrowserDriver() driver.responses["browser_prepare"] = { "status": "refused", @@ -742,6 +743,7 @@ def test_existing_profile_prepare_delegates_to_driver_permission_mode(): window_id=202, profile_mode="existing_profile", allow_launch=True, + grant_existing_profile=True, ) assert result["code"] == "browser_consent_required" @@ -897,3 +899,60 @@ def test_call_tool_restarts_a_dead_session(): session._call_tool_async = call session.call_tool("click", {"pid": 1}) assert starts == [True] + + +def test_bind_response_directs_the_caller_to_drop_pid_and_window_id(): + """A bind looks like a successful read, but carries no page content. + + Any call passing pid/window_id lands in the binding branch, which clears + state and mints new tab_ids. A caller that keeps re-sending them re-binds + forever: the tab_id it just received is already unbound on the next call, + and every mutation stays refused. The response has to say so. + """ + driver = _BrowserDriver() + route = _browser_route(driver) + + payload = route.observe(pid=101, window_id=202) + + assert payload["snapshot_required"] is True + assert payload["next_step"] == "fresh_browser_state" + assert "WITHOUT pid or window_id" in payload["hint"] + assert "screenshot_deferred" not in payload + + +def test_bind_reports_include_screenshot_as_deferred(): + """The flag had no page content to attach to; don't drop it silently.""" + driver = _BrowserDriver() + route = _browser_route(driver) + + payload = route.observe(pid=101, window_id=202, include_screenshot=True) + + assert payload["screenshot_deferred"] is True + assert payload["snapshot_required"] is True + + +def test_snapshot_after_bind_clears_the_requirement(): + driver = _BrowserDriver() + route = _browser_route(driver) + route.observe(pid=101, window_id=202) + + snapshot = route.observe(tab_id="opaque-tab") + + assert snapshot["fresh_state"] is True + assert "snapshot_required" not in snapshot + assert "hint" not in snapshot + + +def test_verification_refusal_names_the_exact_next_call(): + driver = _BrowserDriver() + route = _browser_route(driver) + route.observe(pid=101, window_id=202) + + result = route.mutate( + "browser_navigate", + tab_id="opaque-tab", + args={"url": "about:blank"}, + ) + + assert result["code"] == "browser_verification_required" + assert "WITHOUT pid or window_id" in result["message"] diff --git a/tools/computer_use/browser_route.py b/tools/computer_use/browser_route.py index 55ff095021..3e54ef6e8a 100644 --- a/tools/computer_use/browser_route.py +++ b/tools/computer_use/browser_route.py @@ -294,6 +294,25 @@ class CuaTypedBrowserRoute: # Binding mints the target/tab capabilities but is not a page # snapshot. Require one fresh tab read before any mutation. self.state.verification_required = True + # ...and say so in the payload. Any call carrying pid/window_id + # lands here, so a caller that keeps re-sending them re-binds + # forever: every bind clears state and mints new tab_ids, so the + # tab_id it just received is already unbound on the next call and + # every mutation stays refused. The way out is to drop + # pid/window_id, which is not otherwise discoverable from a bind + # response that looks like a successful read. + payload["snapshot_required"] = True + payload["next_step"] = "fresh_browser_state" + payload["hint"] = ( + "Binding only, no page content. Call cua_browser_state again " + "WITHOUT pid or window_id (optionally with tab_id, query, " + "snapshot_format, include_screenshot) to take the snapshot " + "this binding requires before any mutation." + ) + if include_screenshot: + # A bind carries no page content, so the flag had nothing to + # attach to. Surface that instead of dropping it silently. + payload["screenshot_deferred"] = True payload["exact_binding"] = quality == "exact" if quality != "exact" or not mutation_allowed: payload["native_fallback_required"] = True @@ -483,7 +502,11 @@ class CuaTypedBrowserRoute: if self.state.verification_required and not allow_without_snapshot: return None, _refusal( "browser_verification_required", - "Take a fresh cua_browser_state snapshot before another browser mutation.", + "Take a fresh cua_browser_state snapshot before another " + "browser mutation: call cua_browser_state WITHOUT pid or " + "window_id. Re-sending pid/window_id re-binds instead of " + "snapshotting, which mints new tab_ids and leaves this " + "mutation refused.", ) return selected_tab, None diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index 72651f6bc3..388a4778a9 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -287,7 +287,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = { "type": "boolean", "description": ( "For cua_browser_state, include the current browser screenshot " - "as image content in the tool result. Defaults to false." + "as image content in the tool result. Defaults to false. " + "Applies to snapshot calls only: passing pid/window_id makes " + "the call a binding, which carries no page content and " + "reports screenshot_deferred instead." ), }, "query": {"type": "string", "description": "Optional browser-state query."},