review-fix(computer_use): restore per-axis scroll coordinate semantics

main guarded scroll's x/y independently (coordinate=[null,100] -> y=100),
while click treats a coordinate without x as no point. The shared _xy()
applied click semantics to scroll; split out _scroll_xy() with main's
per-axis guard and pin both behaviors in a test.
This commit is contained in:
Teknium
2026-09-03 09:32:38 -07:00
parent 0c62b297c6
commit febbe6ae12
2 changed files with 20 additions and 1 deletions

View File

@@ -124,6 +124,18 @@ class TestDispatch:
assert drag_kw["from_element"] == 1
assert drag_kw["to_element"] == 5
def test_scroll_coordinate_axes_are_independent(self, noop_backend):
"""scroll forwards each coordinate axis on its own (main parity):
``coordinate=[null, 100]`` scrolls at y=100 with x=None, whereas
click treats a coordinate without x as no point at all."""
from tools.computer_use.tool import handle_computer_use
handle_computer_use({"action": "scroll", "coordinate": [None, 100]})
scroll_kw = next(c[1] for c in noop_backend.calls if c[0] == "scroll")
assert (scroll_kw["x"], scroll_kw["y"]) == (None, 100)
handle_computer_use({"action": "click", "coordinate": [None, 100]})
click_kw = next(c[1] for c in noop_backend.calls if c[0] == "click")
assert (click_kw["x"], click_kw["y"]) == (None, None)
def test_capture_forwards_exact_pid_window_target(self, noop_backend):
from tools.computer_use.tool import handle_computer_use

View File

@@ -292,8 +292,15 @@ def _summarize_action(action: str, args: Dict[str, Any]) -> str:
# final str/dict result. `delivery` = delivery_mode + bring_to_front; only input actions use it.
def _xy(args: Dict[str, Any]) -> Dict[str, Any]:
"""Click semantics: a coordinate only counts when its x is set (a bare y is not a point)."""
return dict(x=coord[0], y=coord[1]) if (coord := args.get("coordinate")) and coord[0] is not None else dict(x=None, y=None)
def _scroll_xy(args: Dict[str, Any]) -> Dict[str, Any]:
"""Scroll semantics: axes are independent — ``coordinate=[null, 100]`` scrolls at y=100 with x unset."""
coord = args.get("coordinate") or (None, None)
return dict(x=coord[0] if coord and coord[0] is not None else None,
y=coord[1] if coord and coord[1] is not None else None)
def _do_click(backend, action, args, button=None, count=1, **delivery):
return backend.click(element=args.get("element"), **_xy(args), button=button or args.get("button") or "left",
click_count=count, modifiers=args.get("modifiers"), **delivery)
@@ -308,7 +315,7 @@ def _do_drag(backend, action, args, **delivery):
def _do_scroll(backend, action, args, **delivery):
return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)),
element=args.get("element"), **_xy(args), modifiers=args.get("modifiers"), **delivery)
element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery)
def _do_capture(backend, action, args, **_):
if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}: