fix(computer-use): make the typed-browser bind/snapshot split discoverable
`cua_browser_state` has two branches, chosen implicitly: any call carrying
pid or window_id is a *binding* (browser_route.py:252), anything else is a
*snapshot*. A binding clears session state, mints fresh tab_ids, returns
binding metadata with no page content, and sets verification_required.
Nothing in the response says that. A caller that keeps passing pid/window_id
- the natural reading of "bind to this window, then read it" - re-binds
forever: the tab_id it just received is unbound by the next bind, so every
cua_browser_navigate comes back browser_verification_required, and the
refusal ("take a fresh snapshot") points at the same call that just re-bound.
Observed live as 11 consecutive refused navigates before the model gave up
and fell back to foreground SendInput on the address bar.
The same confusion silently swallowed include_screenshot: both calls that
requested one were bindings, which carry no page content, so the flag had
nothing to attach to and was dropped without comment.
A binding response now reports snapshot_required, next_step
(fresh_browser_state, matching the existing token convention) and a hint
naming the exact next call; requesting a screenshot on a binding reports
screenshot_deferred instead of dropping it. The verification refusal now
says to call cua_browser_state WITHOUT pid/window_id and why re-sending them
does not help. The schema documents that include_screenshot applies to
snapshots.
Behavior of the bind and snapshot branches themselves is unchanged - this is
purely about making the split legible to the caller.
Unit-tested. Not verified end to end on the reporting host: the driver
refuses the bind upstream there (`browser_requires_setup: no owned DevTools
endpoint`, and it does not accept a user-launched --remote-debugging-port),
so the typed route never reaches this branch. That attach failure is a
separate cua-driver issue.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
committed by
Teknium
parent
9a96fdc5b8
commit
5f049b517b
@@ -730,6 +730,7 @@ def test_missing_typed_browser_tool_returns_native_fallback_refusal():
|
||||
|
||||
|
||||
def test_existing_profile_prepare_delegates_to_driver_permission_mode():
|
||||
"""Past the host-side grant floor, the driver still owns the decision."""
|
||||
driver = _BrowserDriver()
|
||||
driver.responses["browser_prepare"] = {
|
||||
"status": "refused",
|
||||
@@ -742,6 +743,7 @@ def test_existing_profile_prepare_delegates_to_driver_permission_mode():
|
||||
window_id=202,
|
||||
profile_mode="existing_profile",
|
||||
allow_launch=True,
|
||||
grant_existing_profile=True,
|
||||
)
|
||||
|
||||
assert result["code"] == "browser_consent_required"
|
||||
@@ -897,3 +899,60 @@ def test_call_tool_restarts_a_dead_session():
|
||||
session._call_tool_async = call
|
||||
session.call_tool("click", {"pid": 1})
|
||||
assert starts == [True]
|
||||
|
||||
|
||||
def test_bind_response_directs_the_caller_to_drop_pid_and_window_id():
|
||||
"""A bind looks like a successful read, but carries no page content.
|
||||
|
||||
Any call passing pid/window_id lands in the binding branch, which clears
|
||||
state and mints new tab_ids. A caller that keeps re-sending them re-binds
|
||||
forever: the tab_id it just received is already unbound on the next call,
|
||||
and every mutation stays refused. The response has to say so.
|
||||
"""
|
||||
driver = _BrowserDriver()
|
||||
route = _browser_route(driver)
|
||||
|
||||
payload = route.observe(pid=101, window_id=202)
|
||||
|
||||
assert payload["snapshot_required"] is True
|
||||
assert payload["next_step"] == "fresh_browser_state"
|
||||
assert "WITHOUT pid or window_id" in payload["hint"]
|
||||
assert "screenshot_deferred" not in payload
|
||||
|
||||
|
||||
def test_bind_reports_include_screenshot_as_deferred():
|
||||
"""The flag had no page content to attach to; don't drop it silently."""
|
||||
driver = _BrowserDriver()
|
||||
route = _browser_route(driver)
|
||||
|
||||
payload = route.observe(pid=101, window_id=202, include_screenshot=True)
|
||||
|
||||
assert payload["screenshot_deferred"] is True
|
||||
assert payload["snapshot_required"] is True
|
||||
|
||||
|
||||
def test_snapshot_after_bind_clears_the_requirement():
|
||||
driver = _BrowserDriver()
|
||||
route = _browser_route(driver)
|
||||
route.observe(pid=101, window_id=202)
|
||||
|
||||
snapshot = route.observe(tab_id="opaque-tab")
|
||||
|
||||
assert snapshot["fresh_state"] is True
|
||||
assert "snapshot_required" not in snapshot
|
||||
assert "hint" not in snapshot
|
||||
|
||||
|
||||
def test_verification_refusal_names_the_exact_next_call():
|
||||
driver = _BrowserDriver()
|
||||
route = _browser_route(driver)
|
||||
route.observe(pid=101, window_id=202)
|
||||
|
||||
result = route.mutate(
|
||||
"browser_navigate",
|
||||
tab_id="opaque-tab",
|
||||
args={"url": "about:blank"},
|
||||
)
|
||||
|
||||
assert result["code"] == "browser_verification_required"
|
||||
assert "WITHOUT pid or window_id" in result["message"]
|
||||
|
||||
@@ -294,6 +294,25 @@ class CuaTypedBrowserRoute:
|
||||
# Binding mints the target/tab capabilities but is not a page
|
||||
# snapshot. Require one fresh tab read before any mutation.
|
||||
self.state.verification_required = True
|
||||
# ...and say so in the payload. Any call carrying pid/window_id
|
||||
# lands here, so a caller that keeps re-sending them re-binds
|
||||
# forever: every bind clears state and mints new tab_ids, so the
|
||||
# tab_id it just received is already unbound on the next call and
|
||||
# every mutation stays refused. The way out is to drop
|
||||
# pid/window_id, which is not otherwise discoverable from a bind
|
||||
# response that looks like a successful read.
|
||||
payload["snapshot_required"] = True
|
||||
payload["next_step"] = "fresh_browser_state"
|
||||
payload["hint"] = (
|
||||
"Binding only, no page content. Call cua_browser_state again "
|
||||
"WITHOUT pid or window_id (optionally with tab_id, query, "
|
||||
"snapshot_format, include_screenshot) to take the snapshot "
|
||||
"this binding requires before any mutation."
|
||||
)
|
||||
if include_screenshot:
|
||||
# A bind carries no page content, so the flag had nothing to
|
||||
# attach to. Surface that instead of dropping it silently.
|
||||
payload["screenshot_deferred"] = True
|
||||
payload["exact_binding"] = quality == "exact"
|
||||
if quality != "exact" or not mutation_allowed:
|
||||
payload["native_fallback_required"] = True
|
||||
@@ -483,7 +502,11 @@ class CuaTypedBrowserRoute:
|
||||
if self.state.verification_required and not allow_without_snapshot:
|
||||
return None, _refusal(
|
||||
"browser_verification_required",
|
||||
"Take a fresh cua_browser_state snapshot before another browser mutation.",
|
||||
"Take a fresh cua_browser_state snapshot before another "
|
||||
"browser mutation: call cua_browser_state WITHOUT pid or "
|
||||
"window_id. Re-sending pid/window_id re-binds instead of "
|
||||
"snapshotting, which mints new tab_ids and leaves this "
|
||||
"mutation refused.",
|
||||
)
|
||||
return selected_tab, None
|
||||
|
||||
|
||||
@@ -287,7 +287,10 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
|
||||
"type": "boolean",
|
||||
"description": (
|
||||
"For cua_browser_state, include the current browser screenshot "
|
||||
"as image content in the tool result. Defaults to false."
|
||||
"as image content in the tool result. Defaults to false. "
|
||||
"Applies to snapshot calls only: passing pid/window_id makes "
|
||||
"the call a binding, which carries no page content and "
|
||||
"reports screenshot_deferred instead."
|
||||
),
|
||||
},
|
||||
"query": {"type": "string", "description": "Optional browser-state query."},
|
||||
|
||||
Reference in New Issue
Block a user