fix(preview): refuse unfocused type and abort leftover keystrokes
After the locate click, type now refuses unless the located editable is document.activeElement, so characters are not delivered as page input when focus stayed on body. The keystroke loop checks an abort signal between characters; preview.act cancel (timeout or interrupt) and a local Stop both set it, so queued keystrokes stop. A printable press on body/html is refused unless allow_shortcut is set.
This commit is contained in:
@@ -143,12 +143,16 @@ describe('actOnActivePreview (drive_preview tool)', () => {
|
||||
const send = vi.fn()
|
||||
|
||||
cleanups.push(
|
||||
registerPreviewScriptRunner(tabId, async code =>
|
||||
code.includes('"kind":"locate"')
|
||||
registerPreviewScriptRunner(tabId, async code => {
|
||||
if (code.includes('hermes-focus-probe')) {
|
||||
return JSON.stringify({ focused: true, success: true, tag: 'INPUT' })
|
||||
}
|
||||
|
||||
return code.includes('"kind":"locate"')
|
||||
? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 120, y: 80 }, success: true })
|
||||
: // `hit` is the page's witness that the real pointerdown arrived.
|
||||
JSON.stringify({ elements: [], hit: { tag: 'BUTTON', trusted: true }, success: true })
|
||||
)
|
||||
})
|
||||
)
|
||||
cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send }))
|
||||
|
||||
@@ -175,6 +179,92 @@ describe('actOnActivePreview (drive_preview tool)', () => {
|
||||
expect(result.acted).toBe('clicked button "Save"')
|
||||
})
|
||||
|
||||
/** A driven pane whose post-click focus probe can be answered independently of
|
||||
* the locate and the read-back. The probe is how a type learns whether the
|
||||
* located editable actually became document.activeElement. */
|
||||
const withTypedPane = (focus: { focused: boolean; tag?: string }, onSend?: (event: { type: string }) => void) => {
|
||||
const tabId = openBrowserTab()
|
||||
const send = vi.fn((event: { type: string }) => onSend?.(event))
|
||||
|
||||
cleanups.push(
|
||||
registerPreviewScriptRunner(tabId, async code => {
|
||||
if (code.includes('hermes-focus-probe')) {
|
||||
return JSON.stringify({ focused: focus.focused, success: true, tag: focus.tag ?? 'BODY' })
|
||||
}
|
||||
|
||||
return code.includes('"kind":"locate"')
|
||||
? JSON.stringify({
|
||||
acted: 'looking at textbox "Comment"',
|
||||
point: { x: 40, y: 20 },
|
||||
success: true,
|
||||
tag: 'TEXTAREA',
|
||||
typable: true
|
||||
})
|
||||
: JSON.stringify({ elements: [], hit: { tag: 'TEXTAREA', trusted: true }, success: true })
|
||||
})
|
||||
)
|
||||
cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send }))
|
||||
|
||||
return send
|
||||
}
|
||||
|
||||
const keyEvents = (send: ReturnType<typeof vi.fn>) =>
|
||||
send.mock.calls
|
||||
.map(([event]) => event)
|
||||
.filter(event => event.type === 'keyDown' || event.type === 'char' || event.type === 'keyUp')
|
||||
|
||||
it('refuses to type unless the located editable is document.activeElement', async () => {
|
||||
const send = withTypedPane({ focused: false, tag: 'BODY' })
|
||||
|
||||
const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'hello' })
|
||||
|
||||
// The click may land; the characters must not. Focus stayed off the located
|
||||
// field, so those keystrokes would be page input instead of text.
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error).toMatch(/not focused/i)
|
||||
expect(result.error).toMatch(/nothing typed/i)
|
||||
expect(keyEvents(send)).toEqual([])
|
||||
})
|
||||
|
||||
it('stops keystrokes still queued when the type times out', async () => {
|
||||
const controller = new AbortController()
|
||||
|
||||
const send = withTypedPane({ focused: true, tag: 'TEXTAREA' }, event => {
|
||||
if (event.type === 'char') {
|
||||
controller.abort('timeout')
|
||||
}
|
||||
})
|
||||
|
||||
const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'abcdefghij' }, controller.signal)
|
||||
const chars = send.mock.calls
|
||||
.map(([event]) => event)
|
||||
.filter(event => event.type === 'char')
|
||||
.map(event => event.keyCode)
|
||||
|
||||
expect(chars.length).toBeGreaterThan(0)
|
||||
expect(chars.length).toBeLessThan('abcdefghij'.length)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error).toMatch(/timed out/i)
|
||||
})
|
||||
|
||||
it('stops keystrokes still queued when the type is interrupted', async () => {
|
||||
const controller = new AbortController()
|
||||
|
||||
const send = withTypedPane({ focused: true, tag: 'TEXTAREA' }, event => {
|
||||
if (event.type === 'char') {
|
||||
controller.abort('interrupted')
|
||||
}
|
||||
})
|
||||
|
||||
const result = await actOnActivePreview({ kind: 'type', ref: '@e1', text: 'abcdefghij' }, controller.signal)
|
||||
const chars = send.mock.calls.map(([event]) => event).filter(event => event.type === 'char')
|
||||
|
||||
expect(chars.length).toBeGreaterThan(0)
|
||||
expect(chars.length).toBeLessThan('abcdefghij'.length)
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error).toMatch(/interrupt/i)
|
||||
})
|
||||
|
||||
it('types by pressing keys, after selecting whatever the field held', async () => {
|
||||
const send = withDrivenPane()
|
||||
|
||||
@@ -340,7 +430,109 @@ describe('actOnActivePreview (drive_preview tool)', () => {
|
||||
expect(result.note).toContain('elements')
|
||||
})
|
||||
|
||||
it('refuses to type when a real click leaves focus off the located field', async () => {
|
||||
document.body.innerHTML = '<textarea id="comment">old</textarea><button id="other">Other</button>'
|
||||
document.getElementById('other')!.focus()
|
||||
|
||||
const rect = vi.spyOn(Element.prototype, 'getBoundingClientRect').mockReturnValue({
|
||||
bottom: 40,
|
||||
height: 40,
|
||||
left: 0,
|
||||
right: 40,
|
||||
top: 0,
|
||||
width: 40,
|
||||
x: 0,
|
||||
y: 0,
|
||||
toJSON: () => ({})
|
||||
})
|
||||
|
||||
const raf = vi.spyOn(window, 'requestAnimationFrame').mockImplementation(callback => {
|
||||
callback(0)
|
||||
|
||||
return 1
|
||||
})
|
||||
|
||||
const send = vi.fn()
|
||||
const tabId = openBrowserTab()
|
||||
|
||||
cleanups.push(
|
||||
registerPreviewScriptRunner(tabId, async code => {
|
||||
const raw = new Function('return ' + code)()
|
||||
|
||||
return types.isPromise(raw) ? await raw : raw
|
||||
})
|
||||
)
|
||||
cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send }))
|
||||
|
||||
try {
|
||||
const result = await actOnActivePreview({ kind: 'type', selector: '#comment', text: 'hello' })
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error).toMatch(/not focused/i)
|
||||
expect(result.error).toMatch(/nothing typed/i)
|
||||
expect(keyEvents(send)).toEqual([])
|
||||
expect(document.activeElement).toBe(document.getElementById('other'))
|
||||
} finally {
|
||||
raf.mockRestore()
|
||||
rect.mockRestore()
|
||||
document.body.replaceChildren()
|
||||
delete (window as unknown as { __hermesActHolder?: unknown }).__hermesActHolder
|
||||
}
|
||||
})
|
||||
|
||||
it('reports history verbs with no pane to drive', async () => {
|
||||
expect((await actOnActivePreview({ kind: 'reload' })).error).toContain('open_preview')
|
||||
})
|
||||
|
||||
const withPressedPane = (tag: string) => {
|
||||
const tabId = openBrowserTab()
|
||||
const send = vi.fn()
|
||||
|
||||
cleanups.push(
|
||||
registerPreviewScriptRunner(tabId, async code =>
|
||||
code.includes('"kind":"locate"')
|
||||
? JSON.stringify({ acted: `looking at ${tag}`, point: { x: 8, y: 8 }, success: true, tag })
|
||||
: JSON.stringify({ elements: [], success: true })
|
||||
)
|
||||
)
|
||||
cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send }))
|
||||
|
||||
return send
|
||||
}
|
||||
|
||||
it('refuses a printable press on body or html unless the caller opts into a shortcut', async () => {
|
||||
for (const tag of ['BODY', 'HTML']) {
|
||||
const send = withPressedPane(tag)
|
||||
const result = await actOnActivePreview({ key: 'x', kind: 'press', selector: tag.toLowerCase() })
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
expect(result.error).toMatch(/shortcut/i)
|
||||
expect(keyEvents(send)).toEqual([])
|
||||
}
|
||||
|
||||
const opted = withPressedPane('BODY')
|
||||
|
||||
const allowed = await actOnActivePreview({
|
||||
allowShortcut: true,
|
||||
key: 'x',
|
||||
kind: 'press',
|
||||
selector: 'body'
|
||||
})
|
||||
|
||||
expect(allowed.success).toBe(true)
|
||||
expect(
|
||||
opted.mock.calls.map(([event]) => event).some(event => event.type === 'keyDown' && event.keyCode === 'x')
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('still presses a named key on body', async () => {
|
||||
const send = withPressedPane('BODY')
|
||||
|
||||
const result = await actOnActivePreview({ key: 'Escape', kind: 'press', selector: 'body' })
|
||||
|
||||
expect(result.success).toBe(true)
|
||||
expect(
|
||||
send.mock.calls.map(([event]) => event).some(event => event.type === 'keyDown' && event.keyCode === 'Escape')
|
||||
).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -171,6 +171,26 @@ ${preamble()}
|
||||
})()`
|
||||
}
|
||||
|
||||
/** After the locate click: the located editable must itself be
|
||||
* document.activeElement. A click that only mounts the editor leaves focus
|
||||
* on body, and the characters that follow would be page input. */
|
||||
function buildFocusProbeScript(): string {
|
||||
return `(function () {
|
||||
${preamble()}
|
||||
// hermes-focus-probe
|
||||
var aimed = holder.aimed;
|
||||
var active = document.activeElement;
|
||||
var tag = aimed && aimed.tagName ? aimed.tagName : '';
|
||||
var editable = !!(aimed && (aimed.isContentEditable === true || tag === 'INPUT' || tag === 'TEXTAREA'));
|
||||
var focused = editable && active === aimed;
|
||||
return Promise.resolve(JSON.stringify({
|
||||
focused: focused,
|
||||
success: true,
|
||||
tag: active && active.tagName ? active.tagName : ''
|
||||
}));
|
||||
})()`
|
||||
}
|
||||
|
||||
/** Put up a mark that outlives the action that made it. Every other cue on the
|
||||
* overlay retires on a timer, which is right for narrating a click and no use
|
||||
* at all for holding a finding on screen while the agent keeps working. */
|
||||
@@ -316,6 +336,30 @@ async function runJson(run: PreviewScriptRunner, code: string): Promise<Trip> {
|
||||
return { kind: 'answered', result: JSON.parse(raw) as PreviewActResult }
|
||||
}
|
||||
|
||||
/** A single character is text. Named keys (Enter, Escape, ArrowDown) are not. */
|
||||
function isPrintableKey(key: string): boolean {
|
||||
return key.length === 1
|
||||
}
|
||||
|
||||
/** body/html, or a tag we never learned — a printable key there is a page
|
||||
* shortcut, not text entry. */
|
||||
function isPageRoot(tag: string | undefined): boolean {
|
||||
const normalized = (tag || '').toUpperCase()
|
||||
|
||||
return !normalized || normalized === 'BODY' || normalized === 'HTML'
|
||||
}
|
||||
|
||||
/** The loop stopped because the tool timed out or the turn was interrupted.
|
||||
* Say how much landed so the agent does not assume the whole string did. */
|
||||
function stoppedType(signal: AbortSignal, typed: number, total: number): PreviewActResult {
|
||||
const reason = signal.reason === 'timeout' ? 'timed out' : 'was interrupted'
|
||||
|
||||
return {
|
||||
error: `Typing stopped after ${typed} of ${total} characters because the action ${reason}.`,
|
||||
success: false
|
||||
}
|
||||
}
|
||||
|
||||
/** Past tense of the verb the agent asked for, against what it actually hit. */
|
||||
function describeDone(action: PreviewActAction, target: string): string {
|
||||
if (action.kind === 'type') {
|
||||
@@ -337,7 +381,8 @@ function describeDone(action: PreviewActAction, target: string): string {
|
||||
async function driveAction(
|
||||
run: PreviewScriptRunner,
|
||||
input: PreviewInputHandle,
|
||||
action: PreviewActAction
|
||||
action: PreviewActAction,
|
||||
signal?: AbortSignal
|
||||
): Promise<PreviewActResult> {
|
||||
// A key press must not be preceded by a click — that would activate the
|
||||
// control rather than type into it — so the page hands it focus instead.
|
||||
@@ -366,6 +411,8 @@ async function driveAction(
|
||||
if (action.kind === 'click') {
|
||||
await clickAt(input)
|
||||
} else if (action.kind === 'type') {
|
||||
const text = action.text ?? ''
|
||||
|
||||
if (found.typable === false) {
|
||||
return {
|
||||
error: `${String(found.acted || 'That').replace(/^looking at /, '')} is not a text field, so typing into it would only select the text under the pointer. Click it if it opens one, then type into that.`,
|
||||
@@ -373,21 +420,63 @@ async function driveAction(
|
||||
}
|
||||
}
|
||||
|
||||
if (signal?.aborted) {
|
||||
return stoppedType(signal, 0, text.length)
|
||||
}
|
||||
|
||||
input.focus()
|
||||
await clickAt(input)
|
||||
|
||||
if (signal?.aborted) {
|
||||
return stoppedType(signal, 0, text.length)
|
||||
}
|
||||
|
||||
// The click is what is supposed to move DOM focus. webContents focus is
|
||||
// not that — refuse unless the located editable is now activeElement,
|
||||
// before select-all or any character.
|
||||
const probe = await runJson(run, buildFocusProbeScript())
|
||||
const focused = probe.kind === 'answered' && (probe.result as { focused?: boolean }).focused === true
|
||||
|
||||
if (!focused) {
|
||||
const active =
|
||||
probe.kind === 'answered' ? String((probe.result as { tag?: string }).tag || 'the page') : 'the page'
|
||||
|
||||
return {
|
||||
error: `target is not focused (${active}); nothing typed`,
|
||||
success: false
|
||||
}
|
||||
}
|
||||
|
||||
if (signal?.aborted) {
|
||||
return stoppedType(signal, 0, text.length)
|
||||
}
|
||||
|
||||
// Select-all inside the now-focused field, so typing replaces what is there
|
||||
// the way it would for a person. NOT a triple-click: that is a pointer
|
||||
// gesture and selects the paragraph under the cursor whenever the target
|
||||
// turns out not to be a field.
|
||||
await selectAll(input)
|
||||
await typeText(input, action.text ?? '')
|
||||
const typed = await typeText(input, text, signal)
|
||||
|
||||
if (signal?.aborted) {
|
||||
return stoppedType(signal, typed, text.length)
|
||||
}
|
||||
|
||||
if (action.submit) {
|
||||
await pressKey(input, 'Enter')
|
||||
}
|
||||
} else if (action.kind === 'press') {
|
||||
const key = action.key || 'Enter'
|
||||
|
||||
if (isPrintableKey(key) && action.allowShortcut !== true && isPageRoot(found.tag)) {
|
||||
return {
|
||||
error: `Refused to press a printable key on ${found.tag || 'body'}. That would be a page shortcut; pass allowShortcut to opt in.`,
|
||||
success: false
|
||||
}
|
||||
}
|
||||
|
||||
input.focus()
|
||||
await pressKey(input, action.key || 'Enter')
|
||||
await pressKey(input, key)
|
||||
}
|
||||
// hover is the glide and nothing else — the pointer is already sitting on the
|
||||
// target, which is the whole request.
|
||||
@@ -488,7 +577,8 @@ async function driveScroll(
|
||||
* string: the verb arrives off the wire, and the history ones never reach
|
||||
* the in-page engine. */
|
||||
export async function actOnActivePreview(
|
||||
action: Omit<PreviewActAction, 'kind'> & { kind: string }
|
||||
action: Omit<PreviewActAction, 'kind'> & { kind: string },
|
||||
signal?: AbortSignal
|
||||
): Promise<PreviewActResult> {
|
||||
const nav = NAV_ACTIONS.find(verb => verb === action.kind)
|
||||
|
||||
@@ -538,7 +628,7 @@ export async function actOnActivePreview(
|
||||
const input = activePreviewInput()
|
||||
|
||||
if (input && DRIVEN.indexOf(typed.kind) !== -1) {
|
||||
return driveAction(run, input, typed)
|
||||
return driveAction(run, input, typed, signal)
|
||||
}
|
||||
|
||||
// A plain page scroll is a wheel gesture. Jumping to an end is not — no hand
|
||||
|
||||
@@ -130,9 +130,27 @@ export async function selectAll(input: PreviewInputHandle): Promise<void> {
|
||||
await wait(KEY_MS)
|
||||
}
|
||||
|
||||
/** Type `text` a character at a time into whatever currently has focus. */
|
||||
export async function typeText(input: PreviewInputHandle, text: string): Promise<void> {
|
||||
/** Type `text` a character at a time into whatever currently has focus.
|
||||
*
|
||||
* Checks `signal` between characters. A timeout or interrupt aborts the
|
||||
* signal while this loop is still queued; without the check those keystrokes
|
||||
* keep landing after the tool has already given up. The character in flight
|
||||
* has already been sent — the rest are not. */
|
||||
export async function typeText(input: PreviewInputHandle, text: string, signal?: AbortSignal): Promise<number> {
|
||||
let typed = 0
|
||||
|
||||
for (const character of text) {
|
||||
if (signal?.aborted) {
|
||||
return typed
|
||||
}
|
||||
|
||||
await pressKey(input, character)
|
||||
typed += 1
|
||||
|
||||
if (signal?.aborted) {
|
||||
return typed
|
||||
}
|
||||
}
|
||||
|
||||
return typed
|
||||
}
|
||||
|
||||
43
apps/desktop/src/app/chat/right-rail/preview-typing-abort.ts
Normal file
43
apps/desktop/src/app/chat/right-rail/preview-typing-abort.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* PREVIEW TYPING ABORT — the keystroke loop outlives the tool that started it.
|
||||
*
|
||||
* `drive_preview` blocks on `preview.act` for 45s. When that wait times out, or
|
||||
* the turn is interrupted, the backend withdraws the request with
|
||||
* `request.cancel`. The renderer used to ignore that for typing and keep
|
||||
* sending the rest of the string into whatever still had focus. This map is the
|
||||
* only link: the act handler registers the in-flight controller, and cancel
|
||||
* aborts it so the loop stops at the next character.
|
||||
*/
|
||||
|
||||
const controllers = new Map<string, AbortController>()
|
||||
|
||||
/** Arm a controller for one `preview.act` request. A second arm for the same id
|
||||
* aborts the previous loop — a replayed request must not leave the first one
|
||||
* typing. */
|
||||
export function trackPreviewTyping(requestId: string): AbortSignal {
|
||||
controllers.get(requestId)?.abort('interrupted')
|
||||
|
||||
const controller = new AbortController()
|
||||
|
||||
controllers.set(requestId, controller)
|
||||
|
||||
return controller.signal
|
||||
}
|
||||
|
||||
/** Stop the loop for `requestId`, if one is still running. `reason` is the
|
||||
* cancel's reason (`timeout` or `interrupted`); the loop reports it. */
|
||||
export function abortPreviewTyping(requestId: string, reason = 'interrupted'): void {
|
||||
const controller = controllers.get(requestId)
|
||||
|
||||
if (!controller || controller.signal.aborted) {
|
||||
return
|
||||
}
|
||||
|
||||
controller.abort(reason)
|
||||
}
|
||||
|
||||
/** Drop the controller once the action has settled, so a late cancel is a no-op
|
||||
* rather than aborting a controller the next action might reuse. */
|
||||
export function releasePreviewTyping(requestId: string): void {
|
||||
controllers.delete(requestId)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { ConnectionRequestPayload, ConnectionUpdatePayload, GatewayEvent } from '@hermes/shared'
|
||||
|
||||
import { applyAccountConnectionUpdate } from '@/app/capabilities/connectors/data/account-operations'
|
||||
import { abortPreviewTyping } from '@/app/chat/right-rail/preview-typing-abort'
|
||||
import { pendingClarifyToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-clarify'
|
||||
import { connectionRequestToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-connection'
|
||||
import { translateNow } from '@/i18n'
|
||||
@@ -98,6 +99,10 @@ export function handleInputRequestEvent(ctx: GatewayEventContext): boolean {
|
||||
return true
|
||||
}
|
||||
|
||||
// preview.act has no card. A timeout or interrupt still has to stop keystrokes
|
||||
// already queued for that type.
|
||||
abortPreviewTyping(id, typeof payload?.reason === 'string' ? payload.reason : 'interrupted')
|
||||
|
||||
forgetServerRequest(id)
|
||||
|
||||
const key = sessionId ?? ''
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { readActivePreview } from '@/app/chat/right-rail/preview-reader'
|
||||
import {
|
||||
abortPreviewTyping,
|
||||
releasePreviewTyping,
|
||||
trackPreviewTyping
|
||||
} from '@/app/chat/right-rail/preview-typing-abort'
|
||||
import { readActiveTerminal } from '@/app/right-sidebar/terminal/buffer'
|
||||
import { pendingClarifyToolPayload } from '@/app/session/hooks/use-session-actions/restore-pending-clarify'
|
||||
import { translateNow } from '@/i18n'
|
||||
@@ -336,7 +341,7 @@ const previewRead: Handler = ({ request }) => {
|
||||
)
|
||||
}
|
||||
|
||||
const previewAct: Handler = ({ isActiveSession, request }) => {
|
||||
const previewAct: Handler = ({ deps, isActiveSession, request, sessionId }) => {
|
||||
// drive_preview tool: click/type/scroll/press inside the guest page. Active
|
||||
// session only: a background turn (including one in a tile this window hosts)
|
||||
// must never reach into the page the user is working in (desktop AGENTS.md:
|
||||
@@ -353,24 +358,48 @@ const previewAct: Handler = ({ isActiveSession, request }) => {
|
||||
return
|
||||
}
|
||||
|
||||
// The keystroke loop has to be able to stop when this request is withdrawn
|
||||
// (tool timeout or turn interrupt). The local interrupted flag can flip
|
||||
// before request.cancel arrives; poll it so Stop cuts the loop off too.
|
||||
const signal = trackPreviewTyping(request.id)
|
||||
|
||||
const watch = sessionId
|
||||
? setInterval(() => {
|
||||
if (deps.sessionInterrupted(sessionId)) {
|
||||
abortPreviewTyping(request.id, 'interrupted')
|
||||
}
|
||||
}, 50)
|
||||
: undefined
|
||||
|
||||
void loadPreviewEngine()
|
||||
.then(run =>
|
||||
run({
|
||||
amount: p.amount as never,
|
||||
key: p.key as never,
|
||||
kind: (str(p.action) || '') as never,
|
||||
max: p.max as never,
|
||||
ref: p.ref as never,
|
||||
selector: p.selector as never,
|
||||
submit: p.submit as never,
|
||||
text: p.text as never,
|
||||
to: p.to as PreviewActAction['to']
|
||||
})
|
||||
run(
|
||||
{
|
||||
allowShortcut: p.allow_shortcut === true,
|
||||
amount: p.amount as never,
|
||||
key: p.key as never,
|
||||
kind: (str(p.action) || '') as never,
|
||||
max: p.max as never,
|
||||
ref: p.ref as never,
|
||||
selector: p.selector as never,
|
||||
submit: p.submit as never,
|
||||
text: p.text as never,
|
||||
to: p.to as PreviewActAction['to']
|
||||
},
|
||||
signal
|
||||
)
|
||||
)
|
||||
.then(
|
||||
result => answerValue(request, result),
|
||||
error => answerValue(request, { error: error instanceof Error ? error.message : String(error), success: false })
|
||||
)
|
||||
.finally(() => {
|
||||
if (watch !== undefined) {
|
||||
clearInterval(watch)
|
||||
}
|
||||
|
||||
releasePreviewTyping(request.id)
|
||||
})
|
||||
}
|
||||
|
||||
const windowRead: Handler = ({ request }) => {
|
||||
|
||||
@@ -494,7 +494,8 @@ export function actInPageCore(
|
||||
acted: 'looking at ' + describe(el),
|
||||
point: { x: spot.clientX, y: spot.clientY },
|
||||
success: true,
|
||||
// Real typing starts with a triple-click to clear the field. On anything
|
||||
tag,
|
||||
// Real typing starts with a select-all to clear the field. On anything
|
||||
// that is not a field that gesture selects the paragraph under it
|
||||
// instead, which is how the agent ended up highlighting whole pages.
|
||||
typable: tag === 'TEXTAREA' || tag === 'INPUT' || el.isContentEditable === true
|
||||
@@ -606,6 +607,19 @@ export function actInPageCore(
|
||||
return fail('Pass the key to press, e.g. "Enter" or "Escape".')
|
||||
}
|
||||
|
||||
// A printable key on the page root is a shortcut, not text entry. Named
|
||||
// keys (Enter, Escape, arrows) stay allowed; the caller opts in for the
|
||||
// rest.
|
||||
const root = el.tagName === 'BODY' || el.tagName === 'HTML'
|
||||
|
||||
if (key.length === 1 && root && action.allowShortcut !== true) {
|
||||
return fail(
|
||||
'Refused to press a printable key on ' +
|
||||
el.tagName +
|
||||
'. That would be a page shortcut; pass allowShortcut to opt in.'
|
||||
)
|
||||
}
|
||||
|
||||
const init = { bubbles: true, cancelable: true, code: key.length === 1 ? 'Key' + key.toUpperCase() : key, key }
|
||||
|
||||
el.focus()
|
||||
|
||||
@@ -78,6 +78,9 @@ export interface PreviewActAction {
|
||||
selector?: string
|
||||
/** type: press Enter (and submit the owning form) after entering text. */
|
||||
submit?: boolean
|
||||
/** press: send a printable key even when the located target is body/html.
|
||||
* Off by default — that key would otherwise be a page shortcut. */
|
||||
allowShortcut?: boolean
|
||||
text?: string
|
||||
to?: 'bottom' | 'top'
|
||||
}
|
||||
@@ -96,6 +99,9 @@ export interface PreviewActResult {
|
||||
/** Viewport centre of a located target, for aiming real pointer input at it. */
|
||||
point?: { x: number; y: number }
|
||||
success: boolean
|
||||
/** locate: the target's tag, so a press can refuse body/html without another
|
||||
* round trip. */
|
||||
tag?: string
|
||||
title?: string
|
||||
/** locate: whether the target actually takes typed text. */
|
||||
typable?: boolean
|
||||
|
||||
@@ -4220,6 +4220,7 @@ export interface PreviewActRequestParams {
|
||||
to?: string | null
|
||||
amount?: number | null
|
||||
max?: number | null
|
||||
allow_shortcut?: boolean | null
|
||||
}
|
||||
/** ``tools/tour_tool.py`` field set. */
|
||||
export interface TourRequestParams {
|
||||
|
||||
@@ -21847,6 +21847,18 @@
|
||||
],
|
||||
"default": null,
|
||||
"title": "Max"
|
||||
},
|
||||
"allow_shortcut": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null,
|
||||
"title": "Allow Shortcut"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -25,6 +25,7 @@ def drive_preview_tool(
|
||||
action: str = "", ref: Optional[str] = None, selector: Optional[str] = None, text: Optional[str] = None,
|
||||
key: Optional[str] = None, submit: Optional[bool] = None, amount: Optional[int] = None,
|
||||
to: Optional[str] = None, limit: Optional[int] = None, full: Optional[bool] = None,
|
||||
allow_shortcut: Optional[bool] = None,
|
||||
callback: Optional[Callable] = None) -> str:
|
||||
"""Dispatch one interaction to the desktop renderer and return its outcome."""
|
||||
if callback is None:
|
||||
@@ -43,7 +44,7 @@ def drive_preview_tool(
|
||||
try:
|
||||
fields = (
|
||||
("action", verb), ("ref", ref), ("selector", selector), ("text", text), ("key", key),
|
||||
("submit", submit), ("full", full), ("to", to),
|
||||
("submit", submit), ("full", full), ("to", to), ("allow_shortcut", allow_shortcut),
|
||||
("amount", None if amount is None else int(amount)), ("max", None if limit is None else int(limit)),
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
@@ -80,7 +81,8 @@ ACT_PREVIEW_SCHEMA = {
|
||||
"also presses Enter), scroll, press, strobe (visual flourish only — "
|
||||
"one call runs a multi-second burst; never loop it), back/forward/"
|
||||
"reload. Moves draw live and fade; annotate_preview leaves a lasting "
|
||||
"mark. Page text only: desktop_preview action=read. Separate automated "
|
||||
"mark. A printable press on body/html is refused unless allow_shortcut "
|
||||
"is true. Page text only: desktop_preview action=read. Separate automated "
|
||||
"browser: browser_* tools."
|
||||
),
|
||||
"parameters": {
|
||||
@@ -108,6 +110,10 @@ ACT_PREVIEW_SCHEMA = {
|
||||
"type": "string",
|
||||
"description": "press: key name ('Enter', 'Escape', 'ArrowDown').",
|
||||
},
|
||||
"allow_shortcut": {
|
||||
"type": "boolean",
|
||||
"description": "press: allow a printable key on body/html. Off by default.",
|
||||
},
|
||||
"amount": {
|
||||
"type": "integer",
|
||||
"description": "scroll: pixels (negative = up; default ~one screen).",
|
||||
@@ -137,7 +143,8 @@ registry.register(
|
||||
schema=ACT_PREVIEW_SCHEMA,
|
||||
handler=lambda args, **kw: drive_preview_tool(
|
||||
action=args.get("action", ""), limit=args.get("max"), callback=kw.get("callback"),
|
||||
**{k: args.get(k) for k in ("ref", "selector", "text", "key", "submit", "amount", "to", "full")},
|
||||
**{k: args.get(k) for k in (
|
||||
"ref", "selector", "text", "key", "submit", "amount", "to", "full", "allow_shortcut")},
|
||||
),
|
||||
emoji="🖱️")
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ class PreviewActRequestParams(ServerRequestParams):
|
||||
to: str | None = None
|
||||
amount: int | None = None
|
||||
max: int | None = None
|
||||
allow_shortcut: bool | None = None
|
||||
|
||||
|
||||
server_request("preview.act", params=PreviewActRequestParams, result=ValueResult,
|
||||
|
||||
Reference in New Issue
Block a user