diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts index dcee5341b9..ba37c70e89 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts @@ -143,12 +143,16 @@ describe('actOnActivePreview (drive_preview tool)', () => { const send = vi.fn() cleanups.push( - registerPreviewScriptRunner(tabId, async code => - code.includes('"kind":"locate"') + registerPreviewScriptRunner(tabId, async code => { + if (code.includes('hermes-focus-probe')) { + return JSON.stringify({ focused: true, success: true, tag: 'INPUT' }) + } + + return code.includes('"kind":"locate"') ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 120, y: 80 }, success: true }) : // `hit` is the page's witness that the real pointerdown arrived. JSON.stringify({ elements: [], hit: { tag: 'BUTTON', trusted: true }, success: true }) - ) + }) ) cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) @@ -175,6 +179,92 @@ describe('actOnActivePreview (drive_preview tool)', () => { expect(result.acted).toBe('clicked button "Save"') }) + /** A driven pane whose post-click focus probe can be answered independently of + * the locate and the read-back. The probe is how a type learns whether the + * located editable actually became document.activeElement. */ + const withTypedPane = (focus: { focused: boolean; tag?: string }, onSend?: (event: { type: string }) => void) => { + const tabId = openBrowserTab() + const send = vi.fn((event: { type: string }) => onSend?.(event)) + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => { + if (code.includes('hermes-focus-probe')) { + return JSON.stringify({ focused: focus.focused, success: true, tag: focus.tag ?? 'BODY' }) + } + + return code.includes('"kind":"locate"') + ? JSON.stringify({ + acted: 'looking at textbox "Comment"', + point: { x: 40, y: 20 }, + success: true, + tag: 'TEXTAREA', + typable: true + }) + : JSON.stringify({ elements: [], hit: { tag: 'TEXTAREA', trusted: true }, success: true }) + }) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + return send + } + + const keyEvents = (send: ReturnType) => + send.mock.calls + .map(([event]) => event) + .filter(event => event.type === 'keyDown' || event.type === 'char' || event.type === 'keyUp') + + it('refuses to type unless the located editable is document.activeElement', async () => { + const send = withTypedPane({ focused: false, tag: 'BODY' }) + + const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'hello' }) + + // The click may land; the characters must not. Focus stayed off the located + // field, so those keystrokes would be page input instead of text. + expect(result.success).toBe(false) + expect(result.error).toMatch(/not focused/i) + expect(result.error).toMatch(/nothing typed/i) + expect(keyEvents(send)).toEqual([]) + }) + + it('stops keystrokes still queued when the type times out', async () => { + const controller = new AbortController() + + const send = withTypedPane({ focused: true, tag: 'TEXTAREA' }, event => { + if (event.type === 'char') { + controller.abort('timeout') + } + }) + + const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'abcdefghij' }, controller.signal) + const chars = send.mock.calls + .map(([event]) => event) + .filter(event => event.type === 'char') + .map(event => event.keyCode) + + expect(chars.length).toBeGreaterThan(0) + expect(chars.length).toBeLessThan('abcdefghij'.length) + expect(result.success).toBe(false) + expect(result.error).toMatch(/timed out/i) + }) + + it('stops keystrokes still queued when the type is interrupted', async () => { + const controller = new AbortController() + + const send = withTypedPane({ focused: true, tag: 'TEXTAREA' }, event => { + if (event.type === 'char') { + controller.abort('interrupted') + } + }) + + const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'abcdefghij' }, controller.signal) + const chars = send.mock.calls.map(([event]) => event).filter(event => event.type === 'char') + + expect(chars.length).toBeGreaterThan(0) + expect(chars.length).toBeLessThan('abcdefghij'.length) + expect(result.success).toBe(false) + expect(result.error).toMatch(/interrupt/i) + }) + it('types by pressing keys, after selecting whatever the field held', async () => { const send = withDrivenPane() @@ -340,7 +430,109 @@ describe('actOnActivePreview (drive_preview tool)', () => { expect(result.note).toContain('elements') }) + it('refuses to type when a real click leaves focus off the located field', async () => { + document.body.innerHTML = '' + document.getElementById('other')!.focus() + + const rect = vi.spyOn(Element.prototype, 'getBoundingClientRect').mockReturnValue({ + bottom: 40, + height: 40, + left: 0, + right: 40, + top: 0, + width: 40, + x: 0, + y: 0, + toJSON: () => ({}) + }) + + const raf = vi.spyOn(window, 'requestAnimationFrame').mockImplementation(callback => { + callback(0) + + return 1 + }) + + const send = vi.fn() + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => { + const raw = new Function('return ' + code)() + + return types.isPromise(raw) ? await raw : raw + }) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + try { + const result = await actOnActivePreview({ kind: 'type', selector: '#comment', text: 'hello' }) + + expect(result.success).toBe(false) + expect(result.error).toMatch(/not focused/i) + expect(result.error).toMatch(/nothing typed/i) + expect(keyEvents(send)).toEqual([]) + expect(document.activeElement).toBe(document.getElementById('other')) + } finally { + raf.mockRestore() + rect.mockRestore() + document.body.replaceChildren() + delete (window as unknown as { __hermesActHolder?: unknown }).__hermesActHolder + } + }) + it('reports history verbs with no pane to drive', async () => { expect((await actOnActivePreview({ kind: 'reload' })).error).toContain('open_preview') }) + + const withPressedPane = (tag: string) => { + const tabId = openBrowserTab() + const send = vi.fn() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: `looking at ${tag}`, point: { x: 8, y: 8 }, success: true, tag }) + : JSON.stringify({ elements: [], success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + return send + } + + it('refuses a printable press on body or html unless the caller opts into a shortcut', async () => { + for (const tag of ['BODY', 'HTML']) { + const send = withPressedPane(tag) + const result = await actOnActivePreview({ key: 'x', kind: 'press', selector: tag.toLowerCase() }) + + expect(result.success).toBe(false) + expect(result.error).toMatch(/shortcut/i) + expect(keyEvents(send)).toEqual([]) + } + + const opted = withPressedPane('BODY') + + const allowed = await actOnActivePreview({ + allowShortcut: true, + key: 'x', + kind: 'press', + selector: 'body' + }) + + expect(allowed.success).toBe(true) + expect( + opted.mock.calls.map(([event]) => event).some(event => event.type === 'keyDown' && event.keyCode === 'x') + ).toBe(true) + }) + + it('still presses a named key on body', async () => { + const send = withPressedPane('BODY') + + const result = await actOnActivePreview({ key: 'Escape', kind: 'press', selector: 'body' }) + + expect(result.success).toBe(true) + expect( + send.mock.calls.map(([event]) => event).some(event => event.type === 'keyDown' && event.keyCode === 'Escape') + ).toBe(true) + }) }) diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.ts b/apps/desktop/src/app/chat/right-rail/preview-act.ts index 070edef84a..00636efd7f 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.ts @@ -171,6 +171,26 @@ ${preamble()} })()` } +/** After the locate click: the located editable must itself be + * document.activeElement. A click that only mounts the editor leaves focus + * on body, and the characters that follow would be page input. */ +function buildFocusProbeScript(): string { + return `(function () { +${preamble()} + // hermes-focus-probe + var aimed = holder.aimed; + var active = document.activeElement; + var tag = aimed && aimed.tagName ? aimed.tagName : ''; + var editable = !!(aimed && (aimed.isContentEditable === true || tag === 'INPUT' || tag === 'TEXTAREA')); + var focused = editable && active === aimed; + return Promise.resolve(JSON.stringify({ + focused: focused, + success: true, + tag: active && active.tagName ? active.tagName : '' + })); +})()` +} + /** Put up a mark that outlives the action that made it. Every other cue on the * overlay retires on a timer, which is right for narrating a click and no use * at all for holding a finding on screen while the agent keeps working. */ @@ -316,6 +336,30 @@ async function runJson(run: PreviewScriptRunner, code: string): Promise { return { kind: 'answered', result: JSON.parse(raw) as PreviewActResult } } +/** A single character is text. Named keys (Enter, Escape, ArrowDown) are not. */ +function isPrintableKey(key: string): boolean { + return key.length === 1 +} + +/** body/html, or a tag we never learned — a printable key there is a page + * shortcut, not text entry. */ +function isPageRoot(tag: string | undefined): boolean { + const normalized = (tag || '').toUpperCase() + + return !normalized || normalized === 'BODY' || normalized === 'HTML' +} + +/** The loop stopped because the tool timed out or the turn was interrupted. + * Say how much landed so the agent does not assume the whole string did. */ +function stoppedType(signal: AbortSignal, typed: number, total: number): PreviewActResult { + const reason = signal.reason === 'timeout' ? 'timed out' : 'was interrupted' + + return { + error: `Typing stopped after ${typed} of ${total} characters because the action ${reason}.`, + success: false + } +} + /** Past tense of the verb the agent asked for, against what it actually hit. */ function describeDone(action: PreviewActAction, target: string): string { if (action.kind === 'type') { @@ -337,7 +381,8 @@ function describeDone(action: PreviewActAction, target: string): string { async function driveAction( run: PreviewScriptRunner, input: PreviewInputHandle, - action: PreviewActAction + action: PreviewActAction, + signal?: AbortSignal ): Promise { // A key press must not be preceded by a click — that would activate the // control rather than type into it — so the page hands it focus instead. @@ -366,6 +411,8 @@ async function driveAction( if (action.kind === 'click') { await clickAt(input) } else if (action.kind === 'type') { + const text = action.text ?? '' + if (found.typable === false) { return { error: `${String(found.acted || 'That').replace(/^looking at /, '')} is not a text field, so typing into it would only select the text under the pointer. Click it if it opens one, then type into that.`, @@ -373,21 +420,63 @@ async function driveAction( } } + if (signal?.aborted) { + return stoppedType(signal, 0, text.length) + } + input.focus() await clickAt(input) + + if (signal?.aborted) { + return stoppedType(signal, 0, text.length) + } + + // The click is what is supposed to move DOM focus. webContents focus is + // not that — refuse unless the located editable is now activeElement, + // before select-all or any character. + const probe = await runJson(run, buildFocusProbeScript()) + const focused = probe.kind === 'answered' && (probe.result as { focused?: boolean }).focused === true + + if (!focused) { + const active = + probe.kind === 'answered' ? String((probe.result as { tag?: string }).tag || 'the page') : 'the page' + + return { + error: `target is not focused (${active}); nothing typed`, + success: false + } + } + + if (signal?.aborted) { + return stoppedType(signal, 0, text.length) + } + // Select-all inside the now-focused field, so typing replaces what is there // the way it would for a person. NOT a triple-click: that is a pointer // gesture and selects the paragraph under the cursor whenever the target // turns out not to be a field. await selectAll(input) - await typeText(input, action.text ?? '') + const typed = await typeText(input, text, signal) + + if (signal?.aborted) { + return stoppedType(signal, typed, text.length) + } if (action.submit) { await pressKey(input, 'Enter') } } else if (action.kind === 'press') { + const key = action.key || 'Enter' + + if (isPrintableKey(key) && action.allowShortcut !== true && isPageRoot(found.tag)) { + return { + error: `Refused to press a printable key on ${found.tag || 'body'}. That would be a page shortcut; pass allowShortcut to opt in.`, + success: false + } + } + input.focus() - await pressKey(input, action.key || 'Enter') + await pressKey(input, key) } // hover is the glide and nothing else — the pointer is already sitting on the // target, which is the whole request. @@ -488,7 +577,8 @@ async function driveScroll( * string: the verb arrives off the wire, and the history ones never reach * the in-page engine. */ export async function actOnActivePreview( - action: Omit & { kind: string } + action: Omit & { kind: string }, + signal?: AbortSignal ): Promise { const nav = NAV_ACTIONS.find(verb => verb === action.kind) @@ -538,7 +628,7 @@ export async function actOnActivePreview( const input = activePreviewInput() if (input && DRIVEN.indexOf(typed.kind) !== -1) { - return driveAction(run, input, typed) + return driveAction(run, input, typed, signal) } // A plain page scroll is a wheel gesture. Jumping to an end is not — no hand diff --git a/apps/desktop/src/app/chat/right-rail/preview-drive.ts b/apps/desktop/src/app/chat/right-rail/preview-drive.ts index 18b9dbf24e..993d2689a0 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-drive.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-drive.ts @@ -130,9 +130,27 @@ export async function selectAll(input: PreviewInputHandle): Promise { await wait(KEY_MS) } -/** Type `text` a character at a time into whatever currently has focus. */ -export async function typeText(input: PreviewInputHandle, text: string): Promise { +/** Type `text` a character at a time into whatever currently has focus. + * + * Checks `signal` between characters. A timeout or interrupt aborts the + * signal while this loop is still queued; without the check those keystrokes + * keep landing after the tool has already given up. The character in flight + * has already been sent — the rest are not. */ +export async function typeText(input: PreviewInputHandle, text: string, signal?: AbortSignal): Promise { + let typed = 0 + for (const character of text) { + if (signal?.aborted) { + return typed + } + await pressKey(input, character) + typed += 1 + + if (signal?.aborted) { + return typed + } } + + return typed } diff --git a/apps/desktop/src/app/chat/right-rail/preview-typing-abort.ts b/apps/desktop/src/app/chat/right-rail/preview-typing-abort.ts new file mode 100644 index 0000000000..91e62f9b99 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-typing-abort.ts @@ -0,0 +1,43 @@ +/** + * PREVIEW TYPING ABORT — the keystroke loop outlives the tool that started it. + * + * `drive_preview` blocks on `preview.act` for 45s. When that wait times out, or + * the turn is interrupted, the backend withdraws the request with + * `request.cancel`. The renderer used to ignore that for typing and keep + * sending the rest of the string into whatever still had focus. This map is the + * only link: the act handler registers the in-flight controller, and cancel + * aborts it so the loop stops at the next character. + */ + +const controllers = new Map() + +/** Arm a controller for one `preview.act` request. A second arm for the same id + * aborts the previous loop — a replayed request must not leave the first one + * typing. */ +export function trackPreviewTyping(requestId: string): AbortSignal { + controllers.get(requestId)?.abort('interrupted') + + const controller = new AbortController() + + controllers.set(requestId, controller) + + return controller.signal +} + +/** Stop the loop for `requestId`, if one is still running. `reason` is the + * cancel's reason (`timeout` or `interrupted`); the loop reports it. */ +export function abortPreviewTyping(requestId: string, reason = 'interrupted'): void { + const controller = controllers.get(requestId) + + if (!controller || controller.signal.aborted) { + return + } + + controller.abort(reason) +} + +/** Drop the controller once the action has settled, so a late cancel is a no-op + * rather than aborting a controller the next action might reuse. */ +export function releasePreviewTyping(requestId: string): void { + controllers.delete(requestId) +} diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts index 9209b065f9..1a991c02ec 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/input-requests.ts @@ -1,6 +1,7 @@ import type { ConnectionRequestPayload, ConnectionUpdatePayload, GatewayEvent } from '@hermes/shared' import { applyAccountConnectionUpdate } from '@/app/capabilities/connectors/data/account-operations' +import { abortPreviewTyping } from '@/app/chat/right-rail/preview-typing-abort' import { pendingClarifyToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-clarify' import { connectionRequestToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-connection' import { translateNow } from '@/i18n' @@ -98,6 +99,10 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean { return true } + // preview.act has no card. A timeout or interrupt still has to stop keystrokes + // already queued for that type. + abortPreviewTyping(id, typeof payload?.reason === 'string' ? payload.reason : 'interrupted') + forgetServerRequest(id) const key = sessionId ?? '' diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/server-requests.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/server-requests.ts index c93db37a93..e09945af96 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/server-requests.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/server-requests.ts @@ -1,4 +1,9 @@ import { readActivePreview } from '@/app/chat/right-rail/preview-reader' +import { + abortPreviewTyping, + releasePreviewTyping, + trackPreviewTyping +} from '@/app/chat/right-rail/preview-typing-abort' import { readActiveTerminal } from '@/app/right-sidebar/terminal/buffer' import { pendingClarifyToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-clarify' import { translateNow } from '@/i18n' @@ -336,7 +341,7 @@ const previewRead: Handler = ({ request }) => { ) } -const previewAct: Handler = ({ isActiveSession, request }) => { +const previewAct: Handler = ({ deps, isActiveSession, request, sessionId }) => { // drive_preview tool: click/type/scroll/press inside the guest page. Active // session only: a background turn (including one in a tile this window hosts) // must never reach into the page the user is working in (desktop AGENTS.md: @@ -353,24 +358,48 @@ const previewAct: Handler = ({ isActiveSession, request }) => { return } + // The keystroke loop has to be able to stop when this request is withdrawn + // (tool timeout or turn interrupt). The local interrupted flag can flip + // before request.cancel arrives; poll it so Stop cuts the loop off too. + const signal = trackPreviewTyping(request.id) + + const watch = sessionId + ? setInterval(() => { + if (deps.sessionInterrupted(sessionId)) { + abortPreviewTyping(request.id, 'interrupted') + } + }, 50) + : undefined + void loadPreviewEngine() .then(run => - run({ - amount: p.amount as never, - key: p.key as never, - kind: (str(p.action) || '') as never, - max: p.max as never, - ref: p.ref as never, - selector: p.selector as never, - submit: p.submit as never, - text: p.text as never, - to: p.to as PreviewActAction['to'] - }) + run( + { + allowShortcut: p.allow_shortcut === true, + amount: p.amount as never, + key: p.key as never, + kind: (str(p.action) || '') as never, + max: p.max as never, + ref: p.ref as never, + selector: p.selector as never, + submit: p.submit as never, + text: p.text as never, + to: p.to as PreviewActAction['to'] + }, + signal + ) ) .then( result => answerValue(request, result), error => answerValue(request, { error: error instanceof Error ? error.message : String(error), success: false }) ) + .finally(() => { + if (watch !== undefined) { + clearInterval(watch) + } + + releasePreviewTyping(request.id) + }) } const windowRead: Handler = ({ request }) => { diff --git a/apps/desktop/src/lib/preview-act/act-in-page.ts b/apps/desktop/src/lib/preview-act/act-in-page.ts index 4901059bce..1d80c52bc2 100644 --- a/apps/desktop/src/lib/preview-act/act-in-page.ts +++ b/apps/desktop/src/lib/preview-act/act-in-page.ts @@ -494,7 +494,8 @@ export function actInPageCore( acted: 'looking at ' + describe(el), point: { x: spot.clientX, y: spot.clientY }, success: true, - // Real typing starts with a triple-click to clear the field. On anything + tag, + // Real typing starts with a select-all to clear the field. On anything // that is not a field that gesture selects the paragraph under it // instead, which is how the agent ended up highlighting whole pages. typable: tag === 'TEXTAREA' || tag === 'INPUT' || el.isContentEditable === true @@ -606,6 +607,19 @@ export function actInPageCore( return fail('Pass the key to press, e.g. "Enter" or "Escape".') } + // A printable key on the page root is a shortcut, not text entry. Named + // keys (Enter, Escape, arrows) stay allowed; the caller opts in for the + // rest. + const root = el.tagName === 'BODY' || el.tagName === 'HTML' + + if (key.length === 1 && root && action.allowShortcut !== true) { + return fail( + 'Refused to press a printable key on ' + + el.tagName + + '. That would be a page shortcut; pass allowShortcut to opt in.' + ) + } + const init = { bubbles: true, cancelable: true, code: key.length === 1 ? 'Key' + key.toUpperCase() : key, key } el.focus() diff --git a/apps/desktop/src/lib/preview-act/types.ts b/apps/desktop/src/lib/preview-act/types.ts index b87b5255ab..369d67f6b5 100644 --- a/apps/desktop/src/lib/preview-act/types.ts +++ b/apps/desktop/src/lib/preview-act/types.ts @@ -78,6 +78,9 @@ export interface PreviewActAction { selector?: string /** type: press Enter (and submit the owning form) after entering text. */ submit?: boolean + /** press: send a printable key even when the located target is body/html. + * Off by default — that key would otherwise be a page shortcut. */ + allowShortcut?: boolean text?: string to?: 'bottom' | 'top' } @@ -96,6 +99,9 @@ export interface PreviewActResult { /** Viewport centre of a located target, for aiming real pointer input at it. */ point?: { x: number; y: number } success: boolean + /** locate: the target's tag, so a press can refuse body/html without another + * round trip. */ + tag?: string title?: string /** locate: whether the target actually takes typed text. */ typable?: boolean diff --git a/apps/shared/src/gateway-contract.generated.ts b/apps/shared/src/gateway-contract.generated.ts index 17139f1b60..392a71bc52 100644 --- a/apps/shared/src/gateway-contract.generated.ts +++ b/apps/shared/src/gateway-contract.generated.ts @@ -4220,6 +4220,7 @@ export interface PreviewActRequestParams { to?: string | null amount?: number | null max?: number | null + allow_shortcut?: boolean | null } /** ``tools/tour_tool.py`` field set. */ export interface TourRequestParams { diff --git a/apps/shared/src/gateway-contract.openrpc.json b/apps/shared/src/gateway-contract.openrpc.json index 5d48c90797..6b1374a20f 100644 --- a/apps/shared/src/gateway-contract.openrpc.json +++ b/apps/shared/src/gateway-contract.openrpc.json @@ -21847,6 +21847,18 @@ ], "default": null, "title": "Max" + }, + "allow_shortcut": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Allow Shortcut" } }, "required": [ diff --git a/tools/drive_preview_tool.py b/tools/drive_preview_tool.py index 436bb8febd..a5503398d0 100644 --- a/tools/drive_preview_tool.py +++ b/tools/drive_preview_tool.py @@ -25,6 +25,7 @@ def drive_preview_tool( action: str = "", ref: Optional[str] = None, selector: Optional[str] = None, text: Optional[str] = None, key: Optional[str] = None, submit: Optional[bool] = None, amount: Optional[int] = None, to: Optional[str] = None, limit: Optional[int] = None, full: Optional[bool] = None, + allow_shortcut: Optional[bool] = None, callback: Optional[Callable] = None) -> str: """Dispatch one interaction to the desktop renderer and return its outcome.""" if callback is None: @@ -43,7 +44,7 @@ def drive_preview_tool( try: fields = ( ("action", verb), ("ref", ref), ("selector", selector), ("text", text), ("key", key), - ("submit", submit), ("full", full), ("to", to), + ("submit", submit), ("full", full), ("to", to), ("allow_shortcut", allow_shortcut), ("amount", None if amount is None else int(amount)), ("max", None if limit is None else int(limit)), ) except (TypeError, ValueError): @@ -80,7 +81,8 @@ ACT_PREVIEW_SCHEMA = { "also presses Enter), scroll, press, strobe (visual flourish only — " "one call runs a multi-second burst; never loop it), back/forward/" "reload. Moves draw live and fade; annotate_preview leaves a lasting " - "mark. Page text only: desktop_preview action=read. Separate automated " + "mark. A printable press on body/html is refused unless allow_shortcut " + "is true. Page text only: desktop_preview action=read. Separate automated " "browser: browser_* tools." ), "parameters": { @@ -108,6 +110,10 @@ ACT_PREVIEW_SCHEMA = { "type": "string", "description": "press: key name ('Enter', 'Escape', 'ArrowDown').", }, + "allow_shortcut": { + "type": "boolean", + "description": "press: allow a printable key on body/html. Off by default.", + }, "amount": { "type": "integer", "description": "scroll: pixels (negative = up; default ~one screen).", @@ -137,7 +143,8 @@ registry.register( schema=ACT_PREVIEW_SCHEMA, handler=lambda args, **kw: drive_preview_tool( action=args.get("action", ""), limit=args.get("max"), callback=kw.get("callback"), - **{k: args.get(k) for k in ("ref", "selector", "text", "key", "submit", "amount", "to", "full")}, + **{k: args.get(k) for k in ( + "ref", "selector", "text", "key", "submit", "amount", "to", "full", "allow_shortcut")}, ), emoji="🖱️") diff --git a/tui_gateway/contracts/server_requests.py b/tui_gateway/contracts/server_requests.py index 3a57dc4a6f..92cba6fcff 100644 --- a/tui_gateway/contracts/server_requests.py +++ b/tui_gateway/contracts/server_requests.py @@ -178,6 +178,7 @@ class PreviewActRequestParams(ServerRequestParams): to: str | None = None amount: int | None = None max: int | None = None + allow_shortcut: bool | None = None server_request("preview.act", params=PreviewActRequestParams, result=ValueResult,