fix(computer_use): a capped accessibility walk says so in the capture hint

`_capture_view`'s hint promised a "full element tree saved to elements_file",
but once `computer_use.ax_max_elements` bounds the driver's walk the spill
holds only the first N nodes. When the capture returned at least the bound
(bound > 0), drop "full" and add "accessibility walk capped at N elements;
pass app= to narrow" so the model knows how to reach the rest.
This commit is contained in:
kshitijk4poor
2026-09-22 14:26:51 +05:30
committed by kshitij
parent 74069eb428
commit 87276637ed
2 changed files with 44 additions and 2 deletions

View File

@@ -75,3 +75,31 @@ class TestAxWalkBound:
args = _StubCapture()._gws_args()
assert args["pid"] == 607 and args["window_id"] == 382
assert args["max_elements"] == 250
class TestCappedWalkHint:
"""When the driver's walk stopped at the bound, the spill file is not the full tree — say so."""
@staticmethod
def _summary(monkeypatch, n_elements: int, bound: int) -> str:
from tools.computer_use import tool
from tools.computer_use.backend import CaptureResult, UIElement
monkeypatch.setattr(cua_backend, "_cua_configured_ax_max_elements", lambda: bound)
monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: "/tmp/elements.json")
cap = CaptureResult(mode="ax", width=800, height=600,
elements=[UIElement(index=i + 1, role="AXButton", label=f"b{i}") for i in range(n_elements)])
return "\n".join(tool._capture_summary_lines(tool._capture_view(cap, max_elements=5)))
def test_capped_walk_is_named_and_full_is_dropped(self, monkeypatch):
text = self._summary(monkeypatch, n_elements=8, bound=8)
assert "accessibility walk capped at 8 elements; pass app= to narrow" in text
assert "full element tree" not in text
assert "element tree with untruncated labels saved to" in text
def test_uncapped_walk_still_promises_the_full_tree(self, monkeypatch):
text = self._summary(monkeypatch, n_elements=8, bound=50)
assert "accessibility walk capped" not in text
assert "full element tree with untruncated labels saved to" in text
def test_zero_bound_never_claims_a_cap(self, monkeypatch):
assert "accessibility walk capped" not in self._summary(monkeypatch, n_elements=8, bound=0)

View File

@@ -551,12 +551,24 @@ def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace:
lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible)
too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION
has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small
# The driver's own AX walk may have stopped at ``computer_use.ax_max_elements``: then the spill file is NOT
# the full tree, and the hint must not promise one.
ax_bound = _ax_walk_bound()
ax_capped = ax_bound > 0 and len(cap.elements) >= ax_bound
return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height,
truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note,
ax_capped=ax_bound if ax_capped else 0,
elements_file=_spill_elements_to_file(cap) if lost_detail else None,
screenshot_path=_persist_capture_image(cap) if has_image else None,
dims_omitted=dims if too_small else None, has_image=has_image)
def _ax_walk_bound() -> int:
"""``computer_use.ax_max_elements`` as the driver saw it (0 = unbounded / unknown)."""
with contextlib.suppress(Exception):
from tools.computer_use.cua_backend import _cua_configured_ax_max_elements
return int(_cua_configured_ax_max_elements())
return 0
def _capture_summary_lines(v: SimpleNamespace) -> List[str]:
"""Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the
summary names indices the model can't find."""
@@ -565,8 +577,10 @@ def _capture_summary_lines(v: SimpleNamespace) -> List[str]:
f"{v.bounds_scale} ≈ native coordinate)" if v.bounds_scale else ""),
v.screenshot_path and f"shareable screenshot saved to {v.screenshot_path}",
v.cap.note,
v.elements_file and (f"full element tree with untruncated labels saved to {v.elements_file} — "
"read_file/search_files it if you need dropped label text or elements beyond the cap"),
v.elements_file and (f"{'' if getattr(v, 'ax_capped', 0) else 'full '}element tree with untruncated labels "
f"saved to {v.elements_file} — read_file/search_files it if you need dropped label "
"text or elements beyond the cap"),
getattr(v, "ax_capped", 0) and (f"accessibility walk capped at {v.ax_capped} elements; pass app= to narrow"),
)
return [
f"capture mode={v.cap.mode} {v.width}x{v.height}"