fix(computer_use): a capped accessibility walk says so in the capture hint
`_capture_view`'s hint promised a "full element tree saved to elements_file", but once `computer_use.ax_max_elements` bounds the driver's walk the spill holds only the first N nodes. When the capture returned at least the bound (bound > 0), drop "full" and add "accessibility walk capped at N elements; pass app= to narrow" so the model knows how to reach the rest.
This commit is contained in:
@@ -75,3 +75,31 @@ class TestAxWalkBound:
|
||||
args = _StubCapture()._gws_args()
|
||||
assert args["pid"] == 607 and args["window_id"] == 382
|
||||
assert args["max_elements"] == 250
|
||||
|
||||
|
||||
class TestCappedWalkHint:
|
||||
"""When the driver's walk stopped at the bound, the spill file is not the full tree — say so."""
|
||||
|
||||
@staticmethod
|
||||
def _summary(monkeypatch, n_elements: int, bound: int) -> str:
|
||||
from tools.computer_use import tool
|
||||
from tools.computer_use.backend import CaptureResult, UIElement
|
||||
monkeypatch.setattr(cua_backend, "_cua_configured_ax_max_elements", lambda: bound)
|
||||
monkeypatch.setattr(tool, "_spill_elements_to_file", lambda cap: "/tmp/elements.json")
|
||||
cap = CaptureResult(mode="ax", width=800, height=600,
|
||||
elements=[UIElement(index=i + 1, role="AXButton", label=f"b{i}") for i in range(n_elements)])
|
||||
return "\n".join(tool._capture_summary_lines(tool._capture_view(cap, max_elements=5)))
|
||||
|
||||
def test_capped_walk_is_named_and_full_is_dropped(self, monkeypatch):
|
||||
text = self._summary(monkeypatch, n_elements=8, bound=8)
|
||||
assert "accessibility walk capped at 8 elements; pass app= to narrow" in text
|
||||
assert "full element tree" not in text
|
||||
assert "element tree with untruncated labels saved to" in text
|
||||
|
||||
def test_uncapped_walk_still_promises_the_full_tree(self, monkeypatch):
|
||||
text = self._summary(monkeypatch, n_elements=8, bound=50)
|
||||
assert "accessibility walk capped" not in text
|
||||
assert "full element tree with untruncated labels saved to" in text
|
||||
|
||||
def test_zero_bound_never_claims_a_cap(self, monkeypatch):
|
||||
assert "accessibility walk capped" not in self._summary(monkeypatch, n_elements=8, bound=0)
|
||||
|
||||
@@ -551,12 +551,24 @@ def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace:
|
||||
lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible)
|
||||
too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION
|
||||
has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small
|
||||
# The driver's own AX walk may have stopped at ``computer_use.ax_max_elements``: then the spill file is NOT
|
||||
# the full tree, and the hint must not promise one.
|
||||
ax_bound = _ax_walk_bound()
|
||||
ax_capped = ax_bound > 0 and len(cap.elements) >= ax_bound
|
||||
return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height,
|
||||
truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note,
|
||||
ax_capped=ax_bound if ax_capped else 0,
|
||||
elements_file=_spill_elements_to_file(cap) if lost_detail else None,
|
||||
screenshot_path=_persist_capture_image(cap) if has_image else None,
|
||||
dims_omitted=dims if too_small else None, has_image=has_image)
|
||||
|
||||
def _ax_walk_bound() -> int:
|
||||
"""``computer_use.ax_max_elements`` as the driver saw it (0 = unbounded / unknown)."""
|
||||
with contextlib.suppress(Exception):
|
||||
from tools.computer_use.cua_backend import _cua_configured_ax_max_elements
|
||||
return int(_cua_configured_ax_max_elements())
|
||||
return 0
|
||||
|
||||
def _capture_summary_lines(v: SimpleNamespace) -> List[str]:
|
||||
"""Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the
|
||||
summary names indices the model can't find."""
|
||||
@@ -565,8 +577,10 @@ def _capture_summary_lines(v: SimpleNamespace) -> List[str]:
|
||||
f"{v.bounds_scale} ≈ native coordinate)" if v.bounds_scale else ""),
|
||||
v.screenshot_path and f"shareable screenshot saved to {v.screenshot_path}",
|
||||
v.cap.note,
|
||||
v.elements_file and (f"full element tree with untruncated labels saved to {v.elements_file} — "
|
||||
"read_file/search_files it if you need dropped label text or elements beyond the cap"),
|
||||
v.elements_file and (f"{'' if getattr(v, 'ax_capped', 0) else 'full '}element tree with untruncated labels "
|
||||
f"saved to {v.elements_file} — read_file/search_files it if you need dropped label "
|
||||
"text or elements beyond the cap"),
|
||||
getattr(v, "ax_capped", 0) and (f"accessibility walk capped at {v.ax_capped} elements; pass app= to narrow"),
|
||||
)
|
||||
return [
|
||||
f"capture mode={v.cap.mode} {v.width}x{v.height}"
|
||||
|
||||
Reference in New Issue
Block a user