9b9dd6eeab
KDE/Qt apps report [0,0,0,0] bounds for elements that are perfectly clickable by index (live QA: all 50 of kcalc's zero-rect elements, including every radio button). Serializing that as a plausible rect invites a model to derive coordinate=[0,0] and click the screen corner. - _element_to_dict: zero rect -> bounds: null - _format_elements: '@ bounds-unknown (click by element index)' instead of the fake rect in the summary line - malformed bounds fail open (unchanged serialization) Live-proven on real kcalc (cua-driver 0.20.0): 50 elements now null, 0 zero-rect leftovers, summary annotated, real rects preserved, and a null-bounds radio button still clicks fine by index.
55 lines
1.7 KiB
Python
55 lines
1.7 KiB
Python
"""Zero-rect AX bounds must read as 'unknown', never as a clickable position.
|
|
|
|
Live QA (Aug 2026): KDE/Qt apps report [0,0,0,0] for elements that are
|
|
perfectly clickable by index (all of kcalc's radio buttons). A
|
|
plausible-looking zero rect invites coordinate=[0,0] derivation.
|
|
"""
|
|
from tools.computer_use.backend import UIElement
|
|
from tools.computer_use.tool import (
|
|
_bounds_unknown,
|
|
_element_to_dict,
|
|
_format_elements,
|
|
)
|
|
|
|
|
|
def _el(bounds, index=2, role="radio button", label="Deg"):
|
|
return UIElement(index=index, role=role, label=label, bounds=tuple(bounds), app="")
|
|
|
|
|
|
def test_zero_rect_is_unknown():
|
|
assert _bounds_unknown((0, 0, 0, 0)) is True
|
|
|
|
|
|
def test_real_rects_are_known():
|
|
assert _bounds_unknown((0, 0, 640, 29)) is False
|
|
assert _bounds_unknown((820, 433, 620, 19)) is False
|
|
|
|
|
|
def test_malformed_bounds_fail_open():
|
|
assert _bounds_unknown(None) is False
|
|
assert _bounds_unknown(("x", 0, 0, 0)) is False
|
|
|
|
|
|
def test_json_bounds_nulled_for_zero_rect():
|
|
out = _element_to_dict(_el((0, 0, 0, 0)))
|
|
assert out["bounds"] is None
|
|
assert out["index"] == 2 # index targeting stays intact
|
|
|
|
|
|
def test_json_bounds_preserved_for_real_rect():
|
|
out = _element_to_dict(_el((10, 20, 30, 40)))
|
|
assert out["bounds"] == [10, 20, 30, 40]
|
|
|
|
|
|
def test_summary_line_says_bounds_unknown():
|
|
lines = _format_elements([_el((0, 0, 0, 0))])
|
|
assert "bounds-unknown" in lines[0]
|
|
assert "click by element index" in lines[0]
|
|
assert "(0, 0, 0, 0)" not in lines[0]
|
|
|
|
|
|
def test_summary_line_normal_for_real_rect():
|
|
lines = _format_elements([_el((10, 20, 30, 40))])
|
|
assert "bounds-unknown" not in lines[0]
|
|
assert "(10, 20, 30, 40)" in lines[0]
|