diff --git a/apps/desktop/src/lib/markdown-preprocess.citation.test.ts b/apps/desktop/src/lib/markdown-preprocess.citation.test.ts new file mode 100644 index 0000000000..69c8a18da3 --- /dev/null +++ b/apps/desktop/src/lib/markdown-preprocess.citation.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' + +import { preprocessMarkdown } from './markdown-preprocess' + +// Web-citation transport markers (Gemini-style grounding) arrive in model +// output as private-use-delimited id lists. Desktop must never paint them: +// the U+E200..U+E202 glyphs render as replacement boxes ("triple bars") and +// the `turn…search…` ids are protocol noise no reader can follow to a source +// (#120587). A marker that cannot be resolved to a source is dropped, never +// invented into a link. +const P = { E200: '', E201: '', E202: '' } + +describe('web citation transport markers', () => { + it('strips a single-id marker from prose', () => { + expect(preprocessMarkdown(`Ice floats because it is less dense than liquid water.${P.E200}citeturn0search0${P.E201}`)).toBe( + 'Ice floats because it is less dense than liquid water.' + ) + }) + + it('strips a multi-id marker list', () => { + const input = `answer here${P.E200}citeturn0search11${P.E202}turn2search0${P.E201} and continues` + expect(preprocessMarkdown(input)).toBe('answer here and continues') + }) + + it('strips the no-delimiter shape inside a table cell', () => { + // The reporter's table-cell case: the marker abuts a `|` cell wall. + const input = `| Firm${P.E200}citeturn0search1${P.E201} | US |` + expect(preprocessMarkdown(input)).not.toContain('citeturn') + expect(preprocessMarkdown(input)).toBe('| Firm | US |') + }) + + it('strips a bare citeturn token the model emitted without delimiters', () => { + expect(preprocessMarkdown('See citeturn3search7 for details.')).toBe('See for details.') + expect(preprocessMarkdown('A citeturn0search0.')).toBe('A .') + }) + + it('strips a marker that runs to the very end of the text mid-stream', () => { + // preprocessMarkdown runs per streaming flush; the closing U+E201 has not + // arrived yet. + expect(preprocessMarkdown(`Still typing${P.E200}citeturn0search0`)).toBe('Still typing') + }) + + it('fuses the words a stripped in-word marker separated, like the thinking strip does', () => { + // The marker is transport noise, not content: deleting it inside a word + // joins what it split (same contract as the strip's "does not + // fuse the words a stripped block separated" inverse — there the block is + // replaced by a space; here the marker is INVISIBLE noise, so the visible + // letters on either side were always adjacent). + expect(preprocessMarkdown(`word${P.E200}citeturn0search0${P.E201}word2`)).toBe('wordword2') + }) + + it('does not touch other private-use or icon-font characters', () => { + const icon = '' + expect(preprocessMarkdown(`Label ${icon} stays`)).toBe(`Label ${icon} stays`) + }) + + it('keeps marker text inside inline code and math spans intact', () => { + // The shield machinery (normalizeVisibleProse) splits on inline code and + // math before prose rewrites; the marker rule must ride the same path. + expect(preprocessMarkdown(`run \`x = ${P.E200}citeturn0search0${P.E201}\` please`)).toContain('citeturn') + expect(preprocessMarkdown(`$\\sqrt[3]{8}$ and ${P.E200}citeturn0search0${P.E201}`)).toBe('$\\sqrt[3]{8}$ and ') + }) + + it('keeps stripping numeric [n] markers alongside the transport shape', () => { + expect(preprocessMarkdown(`Both shapes[1] and ${P.E200}citeturn0search0${P.E201} go away.`)).toBe( + 'Both shapes and go away.' + ) + }) +}) diff --git a/apps/desktop/src/lib/markdown-preprocess.ts b/apps/desktop/src/lib/markdown-preprocess.ts index 0243631071..0e2eed7831 100644 --- a/apps/desktop/src/lib/markdown-preprocess.ts +++ b/apps/desktop/src/lib/markdown-preprocess.ts @@ -169,6 +169,19 @@ const SAFE_HTML_TAG_NAMES = new Set([ ]) const CITATION_MARKER_RE = /(?<=[\p{L}\p{N})\].,!?:;"'”’])\[(?:\d+(?:\s*,\s*\d+)*)\](?!\()/gu + +// Web-citation transport markers (Gemini-style grounding): private-use +// delimiters U+E200/U+E201 wrap a `citeturnsearch` id list, with U+E202 +// separating ids — `\uE200citeturn0search11\uE202turn2search0\uE201`, or the +// single-id `\uE200citeturn0search0\uE201`. The delimiters paint as +// replacement glyphs (the reported "triple bars") and the ids are protocol +// noise a reader cannot follow to a source (#120587). Strip the whole marker; +// a marker that cannot be resolved is dropped, never invented into a link. +// The bare no-delimiter alternative only fires with the `cite` head, so plain +// prose can't trip it. Scoped to this shape: stray private-use characters +// (icon fonts, user content) are left alone. +const CITATION_TRANSPORT_MARKER_RE = + /\uE200(?:cite)?(?:\uE202?turn\d+search\d+)+\uE201?|citeturn\d+search\d+(?:turn\d+search\d+)*/gu // Markdown links whose target is a filesystem path on the agent's machine: // `[report](/home/user/report.md)`, `[notes](file:///srv/notes.txt)`, // `[todo](~/todo.md)`, `[log](C:\logs\run.txt)`. Negative lookbehind keeps @@ -421,7 +434,11 @@ function rewriteProseSegment(segment: string): string { autoLinkRawUrls( routeFileLinksToPreview( escapeUnknownHtmlLikeTags( - segment.replace(/`{3,}/g, '').replace(LOCAL_PREVIEW_URL_RE, '$1').replace(CITATION_MARKER_RE, '') + segment + .replace(/`{3,}/g, '') + .replace(LOCAL_PREVIEW_URL_RE, '$1') + .replace(CITATION_TRANSPORT_MARKER_RE, '') + .replace(CITATION_MARKER_RE, '') ) ) )