diff --git a/tests/tools/test_computer_use_ax_walk_bound.py b/tests/tools/test_computer_use_ax_walk_bound.py index dd87edd0e8..016923a67a 100644 --- a/tests/tools/test_computer_use_ax_walk_bound.py +++ b/tests/tools/test_computer_use_ax_walk_bound.py @@ -75,3 +75,31 @@ class TestAxWalkBound: args = _StubCapture()._gws_args() assert args["pid"] == 607 and args["window_id"] == 382 assert args["max_elements"] == 250 + + +class TestCappedWalkHint: + """When the driver's walk stopped at the bound, the spill file is not the full tree — say so.""" + + @staticmethod + def _summary(monkeypatch, n_elements: int, bound: int) -> str: + from tools.computer_use import tool + from tools.computer_use.backend import CaptureResult, UIElement + monkeypatch.setattr(cua_backend, "_cua_configured_ax_max_elements", lambda: bound) + monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: "/tmp/elements.json") + cap = CaptureResult(mode="ax", width=800, height=600, + elements=[UIElement(index=i + 1, role="AXButton", label=f"b{i}") for i in range(n_elements)]) + return "\n".join(tool._capture_summary_lines(tool._capture_view(cap, max_elements=5))) + + def test_capped_walk_is_named_and_full_is_dropped(self, monkeypatch): + text = self._summary(monkeypatch, n_elements=8, bound=8) + assert "accessibility walk capped at 8 elements; pass app= to narrow" in text + assert "full element tree" not in text + assert "element tree with untruncated labels saved to" in text + + def test_uncapped_walk_still_promises_the_full_tree(self, monkeypatch): + text = self._summary(monkeypatch, n_elements=8, bound=50) + assert "accessibility walk capped" not in text + assert "full element tree with untruncated labels saved to" in text + + def test_zero_bound_never_claims_a_cap(self, monkeypatch): + assert "accessibility walk capped" not in self._summary(monkeypatch, n_elements=8, bound=0) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index f21239d3c3..b9f9660a3a 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -551,12 +551,24 @@ def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace: lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible) too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small + # The driver's own AX walk may have stopped at ``computer_use.ax_max_elements``: then the spill file is NOT + # the full tree, and the hint must not promise one. + ax_bound = _ax_walk_bound() + ax_capped = ax_bound > 0 and len(cap.elements) >= ax_bound return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height, truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note, + ax_capped=ax_bound if ax_capped else 0, elements_file=_spill_elements_to_file(cap) if lost_detail else None, screenshot_path=_persist_capture_image(cap) if has_image else None, dims_omitted=dims if too_small else None, has_image=has_image) +def _ax_walk_bound() -> int: + """``computer_use.ax_max_elements`` as the driver saw it (0 = unbounded / unknown).""" + with contextlib.suppress(Exception): + from tools.computer_use.cua_backend import _cua_configured_ax_max_elements + return int(_cua_configured_ax_max_elements()) + return 0 + def _capture_summary_lines(v: SimpleNamespace) -> List[str]: """Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the summary names indices the model can't find.""" @@ -565,8 +577,10 @@ def _capture_summary_lines(v: SimpleNamespace) -> List[str]: f"{v.bounds_scale} ≈ native coordinate)" if v.bounds_scale else ""), v.screenshot_path and f"shareable screenshot saved to {v.screenshot_path}", v.cap.note, - v.elements_file and (f"full element tree with untruncated labels saved to {v.elements_file} — " - "read_file/search_files it if you need dropped label text or elements beyond the cap"), + v.elements_file and (f"{'' if getattr(v, 'ax_capped', 0) else 'full '}element tree with untruncated labels " + f"saved to {v.elements_file} — read_file/search_files it if you need dropped label " + "text or elements beyond the cap"), + getattr(v, "ax_capped", 0) and (f"accessibility walk capped at {v.ax_capped} elements; pass app= to narrow"), ) return [ f"capture mode={v.cap.mode} {v.width}x{v.height}"