From b651b77af5d44856dab2d1710fac60038448d31f Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 2 Sep 2026 23:12:47 -0700 Subject: [PATCH] refactor(computer_use): inline _view into its two builders; fold constant comments and _VERDICT_FIELDS --- tools/computer_use/tool.py | 49 +++++++++++++++----------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 7bfc8e660a..8dc363e789 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -45,8 +45,7 @@ _BLOCKED_KEY_COMBOS = { } _KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option", "windows": "win", "super": "win", "meta": "win"} -# Dangerous shell patterns for the `type` action (last one: fork bomb). -_BLOCKED_TYPE_PATTERNS = [re.compile(p, re.IGNORECASE) for p in ( +_BLOCKED_TYPE_PATTERNS = [re.compile(p, re.IGNORECASE) for p in ( # dangerous shell patterns for `type` (last one: fork bomb) r"curl\s+[^|]*\|\s*bash", r"curl\s+[^|]*\|\s*sh", r"wget\s+[^|]*\|\s*bash", r"\bsudo\s+rm\s+-[rf]", r"\brm\s+-rf\s+/\s*$", r":\s*\(\)\s*\{\s*:\|:\s*&\s*\}")] @@ -80,9 +79,8 @@ def _input_target_mismatch(backend, requested_app: str) -> Optional[str]: # ── Backend selection — env-swappable for tests ───────────────────────────── # Per-Hermes-session cached backends (own cua-driver session, native target, refs, grant namespace). -# `_backend` is the backward-compatible empty-session injection hook (older tests). _backend_lock = threading.Lock() -_backend: Optional[ComputerUseBackend] = None +_backend: Optional[ComputerUseBackend] = None # backward-compatible empty-session injection hook (older tests) _backends: Dict[str, ComputerUseBackend] = {} _backend_call_locks: Dict[str, threading.RLock] = {} _backend_permission_modes: Dict[str, str] = {} @@ -442,8 +440,6 @@ def _classify_action_result(res: ActionResult) -> Dict[str, Any]: return {"decision": "verify_fresh_state", "hint": "Transport succeeded but the effect is unproven. Re-capture and confirm before continuing."} -_VERDICT_FIELDS = ("verified", "effect", "escalation", "path", "degraded", "delivery_mode", "code") - def _present(**fields: Any) -> Dict[str, Any]: return {k: v for k, v in fields.items() if v} # only the truthy optional fields, in the given order @@ -451,21 +447,19 @@ def _action_payload(res: ActionResult) -> Dict[str, Any]: # cua-driver's structured verdict fields only when returned (None = old driver). ok is transport success; # effect/escalation are the semantic verdict. return {"ok": res.ok, "action": res.action, **_present(message=res.message), - **{k: v for k in _VERDICT_FIELDS if (v := getattr(res, k)) is not None}, **_present(meta=res.meta), + **{k: v for k in ("verified", "effect", "escalation", "path", "degraded", "delivery_mode", "code") + if (v := getattr(res, k)) is not None}, **_present(meta=res.meta), "verdict": _classify_action_result(res)} def _text_response(res: ActionResult) -> str: return json.dumps(_action_payload(res)) -# AX `elements` cap: dense UIs publish 500+ nodes (one capture would exhaust context); the full tree spills to a file. -_DEFAULT_MAX_ELEMENTS = 100 -# Some providers reject images below 8x8 before the model sees the result; such captures fall back to text. -_MIN_PROVIDER_IMAGE_DIMENSION = 8 +_DEFAULT_MAX_ELEMENTS = 100 # AX `elements` cap: dense UIs publish 500+ nodes (one capture would exhaust context); the full tree spills to a file +_MIN_PROVIDER_IMAGE_DIMENSION = 8 # some providers reject images below 8x8 before the model sees the result; such captures fall back to text # Some AX trees (Discord/Slack via UIA, Electron chat clients) expose ENTIRE message bodies as labels; uncapped # they blew the tool-result budget and leaked private chat text. Labels identify a control, not text extraction. _MAX_ELEMENT_LABEL_CHARS = 120 -# Bounded cache trails: every dense capture can spill, and CLI-only sessions never run the gateway's media cleanup. -_MAX_SPILL_FILES = _MAX_CAPTURE_FILES = 20 +_MAX_SPILL_FILES = _MAX_CAPTURE_FILES = 20 # bounded cache trails: every dense capture can spill, and CLI-only sessions never run the gateway's media cleanup def _capture_image_format(cap: CaptureResult) -> Tuple[str, str]: # (MIME, file extension): cua-driver's explicit MIME type, else sniff the base64 prefix (JPEG starts with /9j/, @@ -518,18 +512,12 @@ def _bounds_hints(elements: List[UIElement], image_width: int, image_height: int def _bounds_scale(elements: List[UIElement], image_width: int, image_height: int) -> Optional[float]: return _bounds_hints(elements, image_width, image_height)[0] - def _bounds_space_note(elements: List[UIElement], image_width: int, image_height: int) -> Optional[str]: return _bounds_hints(elements, image_width, image_height)[1] -def _view(cap: CaptureResult, visible: List[UIElement], width: int, height: int, **facts: Any) -> SimpleNamespace: - """One capture's derived facts, computed once for every response branch: ``visible`` is the capped element list, - ``dims_omitted`` an image below the provider minimum; ``facts`` override the None/False defaults.""" - return SimpleNamespace(**{**dict(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height, - truncated=len(cap.elements) - len(visible), bounds_scale=None, bounds_note=None, - elements_file=None, screenshot_path=None, dims_omitted=None, has_image=False), **facts}) - def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace: + """One capture's derived facts, computed once for every response branch: ``visible`` is the capped element list, + ``dims_omitted`` an image below the provider minimum.""" visible, dims = cap.elements[:max_elements], None with contextlib.suppress(Exception): # (width, height) of the inline PNG/JPEG screenshot, else the backend's dims = image_dimensions_from_bytes(base64.b64decode(cap.png_b64, validate=False)) if cap.png_b64 else None @@ -539,10 +527,11 @@ def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace: lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible) too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small - return _view(cap, visible, width, height, bounds_scale=scale, bounds_note=note, - elements_file=_spill_elements_to_file(cap) if lost_detail else None, - screenshot_path=_persist_capture_image(cap) if has_image else None, - dims_omitted=dims if too_small else None, has_image=has_image) + return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height, + truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note, + elements_file=_spill_elements_to_file(cap) if lost_detail else None, + screenshot_path=_persist_capture_image(cap) if has_image else None, + dims_omitted=dims if too_small else None, has_image=has_image) def _capture_summary_lines(v: SimpleNamespace) -> List[str]: """Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the @@ -688,9 +677,8 @@ def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]: # ── auxiliary.vision routing for captured screenshots ─────────────────────── -# Longest image side handed to the aux vision model. Full-resolution desktop captures tokenize heavily and can -# overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting vision latency. -_MAX_VISION_DIM = 1456 +_MAX_VISION_DIM = 1456 # longest image side handed to the aux vision model: full-resolution desktop captures tokenize heavily +# and can overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting vision latency. def _shrink_capture_for_vision(raw: bytes, ext: str, max_dim: int = _MAX_VISION_DIM) -> tuple[bytes, Optional[str]]: """Downscale encoded image bytes so the longest side is <= max_dim -> ``(bytes, scale_note)``. note is None when @@ -791,8 +779,9 @@ def _route_capture_through_aux_vision( return None # Same element cap as every other capture branch; dumping cap.elements in full would bypass max_elements # exactly for non-vision main models. Dimensions are the backend's on this branch. - view = _view(cap, cap.elements if visible_elements is None else visible_elements, cap.width, cap.height, - truncated=truncated_elements, elements_file=elements_file, screenshot_path=screenshot_path) + view = SimpleNamespace(cap=cap, visible=cap.elements if visible_elements is None else visible_elements, + total=len(cap.elements), width=cap.width, height=cap.height, truncated=truncated_elements, + elements_file=elements_file, screenshot_path=screenshot_path, bounds_scale=None) return _text_capture_payload(view, summary, {"vision_analysis": analysis_text, "vision_analysis_routed_via": "auxiliary.vision"})