From febbe6ae1211cbff4a133ca4d57afd8d91eec3af Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:32:38 -0700 Subject: [PATCH] review-fix(computer_use): restore per-axis scroll coordinate semantics main guarded scroll's x/y independently (coordinate=[null,100] -> y=100), while click treats a coordinate without x as no point. The shared _xy() applied click semantics to scroll; split out _scroll_xy() with main's per-axis guard and pin both behaviors in a test. --- tests/tools/test_computer_use.py | 12 ++++++++++++ tools/computer_use/tool.py | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_computer_use.py b/tests/tools/test_computer_use.py index 844bb3dffa..b5a0cfd1b4 100644 --- a/tests/tools/test_computer_use.py +++ b/tests/tools/test_computer_use.py @@ -124,6 +124,18 @@ class TestDispatch: assert drag_kw["from_element"] == 1 assert drag_kw["to_element"] == 5 + def test_scroll_coordinate_axes_are_independent(self, noop_backend): + """scroll forwards each coordinate axis on its own (main parity): + ``coordinate=[null, 100]`` scrolls at y=100 with x=None, whereas + click treats a coordinate without x as no point at all.""" + from tools.computer_use.tool import handle_computer_use + handle_computer_use({"action": "scroll", "coordinate": [None, 100]}) + scroll_kw = next(c[1] for c in noop_backend.calls if c[0] == "scroll") + assert (scroll_kw["x"], scroll_kw["y"]) == (None, 100) + handle_computer_use({"action": "click", "coordinate": [None, 100]}) + click_kw = next(c[1] for c in noop_backend.calls if c[0] == "click") + assert (click_kw["x"], click_kw["y"]) == (None, None) + def test_capture_forwards_exact_pid_window_target(self, noop_backend): from tools.computer_use.tool import handle_computer_use diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index e5bb9044b6..ec5b622312 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -292,8 +292,15 @@ def _summarize_action(action: str, args: Dict[str, Any]) -> str: # final str/dict result. `delivery` = delivery_mode + bring_to_front; only input actions use it. def _xy(args: Dict[str, Any]) -> Dict[str, Any]: + """Click semantics: a coordinate only counts when its x is set (a bare y is not a point).""" return dict(x=coord[0], y=coord[1]) if (coord := args.get("coordinate")) and coord[0] is not None else dict(x=None, y=None) +def _scroll_xy(args: Dict[str, Any]) -> Dict[str, Any]: + """Scroll semantics: axes are independent — ``coordinate=[null, 100]`` scrolls at y=100 with x unset.""" + coord = args.get("coordinate") or (None, None) + return dict(x=coord[0] if coord and coord[0] is not None else None, + y=coord[1] if coord and coord[1] is not None else None) + def _do_click(backend, action, args, button=None, count=1, **delivery): return backend.click(element=args.get("element"), **_xy(args), button=button or args.get("button") or "left", click_count=count, modifiers=args.get("modifiers"), **delivery) @@ -308,7 +315,7 @@ def _do_drag(backend, action, args, **delivery): def _do_scroll(backend, action, args, **delivery): return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)), - element=args.get("element"), **_xy(args), modifiers=args.get("modifiers"), **delivery) + element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery) def _do_capture(backend, action, args, **_): if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}: