fix(desktop): read table headers aloud instead of dropping tables silently
Read Aloud removed Markdown tables with no spoken trace (#86602). Speak the header row in their place ("Model, Price.") and keep skipping the body rows, so the listener hears that a table is on screen and what it compares without cell-by-cell data. The header is the reply's own text, so it is always in the voice's language; a localized "table omitted" notice would follow the UI locale instead and reintroduce the mismatch when the two differ. A table whose header cells are all empty stays silent. Also keep the sentence punctuation after an inline MEDIA: token on both the desktop and backend paths ("see MEDIA:/x.py. Then" no longer loses its full stop), per review on #89367.
This commit is contained in:
@@ -16,7 +16,7 @@ describe('sanitizeTextForSpeech', () => {
|
|||||||
expect(sanitizeTextForSpeech('Use `git status` after the change.')).toBe('Use git status after the change.')
|
expect(sanitizeTextForSpeech('Use `git status` after the change.')).toBe('Use git status after the change.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips markdown table data while preserving surrounding human text', () => {
|
it('reads the table header, not its data, between surrounding human text', () => {
|
||||||
const text = `Here is the quick takeaway: the totals remain unchanged.
|
const text = `Here is the quick takeaway: the totals remain unchanged.
|
||||||
|
|
||||||
| Item | Value | Notes |
|
| Item | Value | Notes |
|
||||||
@@ -27,10 +27,45 @@ describe('sanitizeTextForSpeech', () => {
|
|||||||
Full detail stays visible on screen.`
|
Full detail stays visible on screen.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe(
|
expect(sanitizeTextForSpeech(text)).toBe(
|
||||||
'Here is the quick takeaway: the totals remain unchanged. Full detail stays visible on screen.'
|
'Here is the quick takeaway: the totals remain unchanged. Item, Value, Notes. Full detail stays visible on screen.'
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('speaks a non-English reply with no English words injected (#86602)', () => {
|
||||||
|
// Placeholders used to follow the code, not the reply's language: a
|
||||||
|
// Chinese voice read "code block omitted" / "link" and skipped tables
|
||||||
|
// entirely. Code and URLs are silence; the table header is read in the
|
||||||
|
// reply's own language.
|
||||||
|
const text = `对比如下:
|
||||||
|
|
||||||
|
| 模型 | 价格 |
|
||||||
|
| --- | ---: |
|
||||||
|
| 甲 | 10 |
|
||||||
|
|
||||||
|
代码:
|
||||||
|
\`\`\`py
|
||||||
|
print(1)
|
||||||
|
\`\`\`
|
||||||
|
详情见 https://example.com/docs`
|
||||||
|
|
||||||
|
const spoken = sanitizeTextForSpeech(text)
|
||||||
|
|
||||||
|
expect(spoken).toContain('模型, 价格')
|
||||||
|
expect(spoken).not.toMatch(/[A-Za-z]/)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('stays silent for a table whose header cells are all empty', () => {
|
||||||
|
const text = `Before the table.
|
||||||
|
|
||||||
|
| | |
|
||||||
|
| --- | --- |
|
||||||
|
| a | b |
|
||||||
|
|
||||||
|
After the table.`
|
||||||
|
|
||||||
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
||||||
|
})
|
||||||
|
|
||||||
it('does not strip prose that merely contains a pipe character', () => {
|
it('does not strip prose that merely contains a pipe character', () => {
|
||||||
const text = 'Use the summary first | keep the table on screen when it matters.'
|
const text = 'Use the summary first | keep the table on screen when it matters.'
|
||||||
|
|
||||||
@@ -53,6 +88,10 @@ Second sentence.`
|
|||||||
expect(sanitizeTextForSpeech(text)).toBe('The files are below. Bye.')
|
expect(sanitizeTextForSpeech(text)).toBe('The files are below. Bye.')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('keeps the sentence break after an inline MEDIA token', () => {
|
||||||
|
expect(sanitizeTextForSpeech('See MEDIA:/tmp/report-2026-q3.xlsx. Then reply.')).toBe('See. Then reply.')
|
||||||
|
})
|
||||||
|
|
||||||
it('does not speak a placeholder word for URLs', () => {
|
it('does not speak a placeholder word for URLs', () => {
|
||||||
// Used to say the English word "link" (#86602); URLs are silence now.
|
// Used to say the English word "link" (#86602); URLs are silence now.
|
||||||
expect(sanitizeTextForSpeech('See https://example.com/a-huge-page for details')).toBe(
|
expect(sanitizeTextForSpeech('See https://example.com/a-huge-page for details')).toBe(
|
||||||
@@ -90,7 +129,7 @@ Second sentence.`
|
|||||||
expect(sanitizeTextForSpeech(text)).toBe(expected)
|
expect(sanitizeTextForSpeech(text)).toBe(expected)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips markdown tables without leading and trailing pipes', () => {
|
it('reads only the header of markdown tables without leading and trailing pipes', () => {
|
||||||
const text = `Main takeaway: total is unchanged.
|
const text = `Main takeaway: total is unchanged.
|
||||||
|
|
||||||
Item | Value
|
Item | Value
|
||||||
@@ -100,10 +139,10 @@ Example B | 20
|
|||||||
|
|
||||||
Done.`
|
Done.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Main takeaway: total is unchanged. Done.')
|
expect(sanitizeTextForSpeech(text)).toBe('Main takeaway: total is unchanged. Item, Value. Done.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips markdown tables nested inside blockquotes', () => {
|
it('reads only the header of markdown tables nested inside blockquotes', () => {
|
||||||
const text = `Before the table.
|
const text = `Before the table.
|
||||||
|
|
||||||
> | Item | Value |
|
> | Item | Value |
|
||||||
@@ -113,7 +152,7 @@ Done.`
|
|||||||
|
|
||||||
After the table.`
|
After the table.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. Item, Value. After the table.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('allows marker padding plus three spaces in blockquoted tables', () => {
|
it('allows marker padding plus three spaces in blockquoted tables', () => {
|
||||||
@@ -125,10 +164,10 @@ After the table.`
|
|||||||
|
|
||||||
After the table.`
|
After the table.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. Item, Value. After the table.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips explicit single-column markdown tables', () => {
|
it('reads only the header of explicit single-column markdown tables', () => {
|
||||||
const text = `Before the table.
|
const text = `Before the table.
|
||||||
|
|
||||||
| Item |
|
| Item |
|
||||||
@@ -137,7 +176,7 @@ After the table.`
|
|||||||
|
|
||||||
After the table.`
|
After the table.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. Item. After the table.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('preserves rows outside a table blockquote', () => {
|
it('preserves rows outside a table blockquote', () => {
|
||||||
@@ -146,7 +185,7 @@ After the table.`
|
|||||||
> | Example A | 10 |
|
> | Example A | 10 |
|
||||||
Outside | prose`
|
Outside | prose`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Outside | prose')
|
expect(sanitizeTextForSpeech(text)).toBe('Item, Value. Outside | prose')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('preserves malformed tables with mismatched column counts', () => {
|
it('preserves malformed tables with mismatched column counts', () => {
|
||||||
@@ -167,10 +206,10 @@ Keep this prose.`
|
|||||||
|
|
||||||
After the table.`
|
After the table.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. Item, Value. After the table.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('skips tables containing escaped pipe characters', () => {
|
it('reads headers containing escaped pipe characters', () => {
|
||||||
const text = `Before the table.
|
const text = `Before the table.
|
||||||
|
|
||||||
| Item \\| detail | Value |
|
| Item \\| detail | Value |
|
||||||
@@ -179,7 +218,7 @@ After the table.`
|
|||||||
|
|
||||||
After the table.`
|
After the table.`
|
||||||
|
|
||||||
expect(sanitizeTextForSpeech(text)).toBe('Before the table. After the table.')
|
expect(sanitizeTextForSpeech(text)).toBe('Before the table. Item detail, Value. After the table.')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('preserves indented code that resembles a table', () => {
|
it('preserves indented code that resembles a table', () => {
|
||||||
|
|||||||
@@ -7,7 +7,10 @@ const PARAGRAPH_BREAK_RE = /[ \t]*\n{2,}[ \t]*/g
|
|||||||
const PUNCTUATED_PARAGRAPH_BREAK_RE = /([.!?])([*_~`>"'’”)}\]]*)[ \t]*\n{2,}[ \t]*/g
|
const PUNCTUATED_PARAGRAPH_BREAK_RE = /([.!?])([*_~`>"'’”)}\]]*)[ \t]*\n{2,}[ \t]*/g
|
||||||
const SOFT_BREAK_RE = /[ \t]*\n[ \t]*/g
|
const SOFT_BREAK_RE = /[ \t]*\n[ \t]*/g
|
||||||
|
|
||||||
const MEDIA_PATH_RE = /MEDIA:\S+/g
|
// A file-link token ("MEDIA:/path/to/report.xlsx") renders as a chip on
|
||||||
|
// screen; spoken, its hyphenated slug makes voices loop. It is silence, but a
|
||||||
|
// sentence-final period/comma after it is kept ("see MEDIA:/x.py. Then").
|
||||||
|
const MEDIA_PATH_RE = /[ \t]*MEDIA:\S+?(?=[.,;:!?)\]]*(?:\s|$))/g
|
||||||
const LINE_FINAL_COLON_RE = /:\s*$/gm
|
const LINE_FINAL_COLON_RE = /:\s*$/gm
|
||||||
|
|
||||||
const THINKING_PREFIX_RE =
|
const THINKING_PREFIX_RE =
|
||||||
@@ -101,9 +104,23 @@ function parseMarkdownTableRow(line: string): MarkdownTableRow | null {
|
|||||||
return { blockquoteDepth, cells }
|
return { blockquoteDepth, cells }
|
||||||
}
|
}
|
||||||
|
|
||||||
function stripMarkdownTables(text: string): string {
|
// The header row is spoken in place of the table ("Model, Price, Context."):
|
||||||
|
// the listener learns a table is on screen and what it compares, in the reply's
|
||||||
|
// own language, without the body data being read cell by cell (#86602). A
|
||||||
|
// table with an empty header stays silent — there is nothing to announce.
|
||||||
|
function speakableTableHeader(cells: string[]): string {
|
||||||
|
const header = cells
|
||||||
|
.map(cell => cell.replace(/\\\|/g, ' ').trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(', ')
|
||||||
|
|
||||||
|
return header && !/[.!?:]$/.test(header) ? `${header}.` : header
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarizeMarkdownTables(text: string): string {
|
||||||
const lines = text.replace(/\r\n?/g, '\n').split('\n')
|
const lines = text.replace(/\r\n?/g, '\n').split('\n')
|
||||||
const tableLines = new Set<number>()
|
const tableLines = new Set<number>()
|
||||||
|
const headers = new Map<number, string>()
|
||||||
|
|
||||||
let index = 1
|
let index = 1
|
||||||
|
|
||||||
@@ -125,6 +142,7 @@ function stripMarkdownTables(text: string): string {
|
|||||||
|
|
||||||
tableLines.add(index - 1)
|
tableLines.add(index - 1)
|
||||||
tableLines.add(index)
|
tableLines.add(index)
|
||||||
|
headers.set(index - 1, speakableTableHeader(headerRow.cells))
|
||||||
|
|
||||||
let rowIndex = index + 1
|
let rowIndex = index + 1
|
||||||
|
|
||||||
@@ -141,7 +159,17 @@ function stripMarkdownTables(text: string): string {
|
|||||||
index = rowIndex
|
index = rowIndex
|
||||||
}
|
}
|
||||||
|
|
||||||
return lines.filter((_, index) => !tableLines.has(index)).join('\n')
|
return lines
|
||||||
|
.flatMap((line, index) => {
|
||||||
|
if (!tableLines.has(index)) {
|
||||||
|
return [line]
|
||||||
|
}
|
||||||
|
|
||||||
|
const header = headers.get(index)
|
||||||
|
|
||||||
|
return header ? [header] : []
|
||||||
|
})
|
||||||
|
.join('\n')
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeLineBreaks(text: string): string {
|
function normalizeLineBreaks(text: string): string {
|
||||||
@@ -156,13 +184,15 @@ function normalizeLineBreaks(text: string): string {
|
|||||||
export function sanitizeTextForSpeech(text: string): string {
|
export function sanitizeTextForSpeech(text: string): string {
|
||||||
// Tables first: their right-align marker is a trailing colon (":-"), and
|
// Tables first: their right-align marker is a trailing colon (":-"), and
|
||||||
// closing colons before the table detector runs would mangle it.
|
// closing colons before the table detector runs would mangle it.
|
||||||
const withoutTables = stripMarkdownTables(String(text))
|
const withoutTables = summarizeMarkdownTables(String(text))
|
||||||
|
|
||||||
// Close line-final colons BEFORE newlines are flattened: "the regex list:"
|
// Close line-final colons BEFORE newlines are flattened: "the regex list:"
|
||||||
// followed by a code block keeps its colon if this runs after the flatten,
|
// followed by a code block keeps its colon if this runs after the flatten,
|
||||||
// and the voice hangs on it. Closing early turns it into "the regex list.".
|
// and the voice hangs on it. Closing early turns it into "the regex list.".
|
||||||
const pre = withoutTables.replace(LINE_FINAL_COLON_RE, '.')
|
const pre = withoutTables.replace(LINE_FINAL_COLON_RE, '.')
|
||||||
|
|
||||||
|
// Unspeakable tokens are silence, never a placeholder word: an English
|
||||||
|
// "code block omitted" / "link" is wrong for every non-English voice.
|
||||||
return normalizeLineBreaks(pre)
|
return normalizeLineBreaks(pre)
|
||||||
.replace(FENCED_CODE_RE, '')
|
.replace(FENCED_CODE_RE, '')
|
||||||
.replace(THINKING_PREFIX_RE, ' ')
|
.replace(THINKING_PREFIX_RE, ' ')
|
||||||
|
|||||||
@@ -61,6 +61,12 @@ def test_prepare_spoken_text_strips_media_file_links():
|
|||||||
assert "Bye" in spoken
|
assert "Bye" in spoken
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepare_spoken_text_keeps_sentence_break_after_inline_media_link():
|
||||||
|
spoken = prepare_spoken_text("See MEDIA:/tmp/report-2026-q3.xlsx. Then reply.")
|
||||||
|
assert "report" not in spoken
|
||||||
|
assert spoken == "See. Then reply."
|
||||||
|
|
||||||
|
|
||||||
def test_prepare_spoken_text_closes_trailing_colons():
|
def test_prepare_spoken_text_closes_trailing_colons():
|
||||||
# "the regex list:" + a now-removed raw token would leave the voice hanging
|
# "the regex list:" + a now-removed raw token would leave the voice hanging
|
||||||
# on an open colon-pause (the "aaaa" stutter). Close it with a period.
|
# on an open colon-pause (the "aaaa" stutter). Close it with a period.
|
||||||
|
|||||||
@@ -32,8 +32,9 @@ _MD_TABLE_PIPE_RE = re.compile(r"\s*\|\s*")
|
|||||||
_URL_RE = re.compile(r"https?://\S+")
|
_URL_RE = re.compile(r"https?://\S+")
|
||||||
# Local file links ("MEDIA:/Users/me/file.xlsx") are click targets on screen, not
|
# Local file links ("MEDIA:/Users/me/file.xlsx") are click targets on screen, not
|
||||||
# speech: voices loop on the hyphenated slug ("eeeeee"). The token is silence; the
|
# speech: voices loop on the hyphenated slug ("eeeeee"). The token is silence; the
|
||||||
# assistant's prose already says "the files are below".
|
# assistant's prose already says "the files are below". Trailing sentence
|
||||||
_MEDIA_PATH_RE = re.compile(r"MEDIA:\S+")
|
# punctuation is left in place so "see MEDIA:/x.py. Then" keeps its full stop.
|
||||||
|
_MEDIA_PATH_RE = re.compile(r"MEDIA:\S+?(?=[.,;:!?)\]]*(?:\s|$))")
|
||||||
|
|
||||||
_DEGREE_UNITS = (("C", "Celsius"), ("F", "Fahrenheit"))
|
_DEGREE_UNITS = (("C", "Celsius"), ("F", "Fahrenheit"))
|
||||||
# Unit suffix (regex, after a digit) -> spoken word; km/h variants before the bare "m".
|
# Unit suffix (regex, after a digit) -> spoken word; km/h variants before the bare "m".
|
||||||
|
|||||||
Reference in New Issue
Block a user