From a6dadb83ba0f2c8dff82daff01f8f1732b381524 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Thu, 24 Sep 2026 19:08:56 -0500 Subject: [PATCH] fix(desktop): strip citeturn web-citation transport markers from prose (#120587) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model output can arrive carrying Gemini-style grounding citation markers: private-use delimiters U+E200/U+E201 wrap a `citeturnsearch` id list (U+E202 separates ids) - e.g. `\uE200citeturn0search11\uE202turn2search0\uE201`. Desktop had no rule for that shape (CITATION_MARKER_RE only strips bare numeric `[n]` markers), so the private-use code points painted as replacement glyphs - the reported "triple bars" - and the `turn…search…` ids rendered as literal prose, wrapping across table cells and obscuring the answer. Add CITATION_TRANSPORT_MARKER_RE and strip it in rewriteProseSegment, the same shielded path the numeric marker rule rides: inline code and math spans split out first, so `$\sqrt[3]{8}$` and quoted marker text are untouched, and the bare no-delimiter alternative only fires on the `cite` head so plain prose and stray private-use characters (icon fonts) are left alone. A marker that cannot be resolved to a source is dropped, never invented into a link - matching the reporter's own expectation. Mid-stream flushes (closing U+E201 not yet arrived) are covered by the optional-tail shape. Backend-side emission (which search provider leaks the markers into model text) remains unisolated, as the report itself notes; the display-layer strip is justified regardless. Fixes #120587. No external PR existed (the catalog's linked PR #120592 is a dead reference). --- .../lib/markdown-preprocess.citation.test.ts | 69 +++++++++++++++++++ apps/desktop/src/lib/markdown-preprocess.ts | 19 ++++- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 apps/desktop/src/lib/markdown-preprocess.citation.test.ts diff --git a/apps/desktop/src/lib/markdown-preprocess.citation.test.ts b/apps/desktop/src/lib/markdown-preprocess.citation.test.ts new file mode 100644 index 0000000000..69c8a18da3 --- /dev/null +++ b/apps/desktop/src/lib/markdown-preprocess.citation.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' + +import { preprocessMarkdown } from './markdown-preprocess' + +// Web-citation transport markers (Gemini-style grounding) arrive in model +// output as private-use-delimited id lists. Desktop must never paint them: +// the U+E200..U+E202 glyphs render as replacement boxes ("triple bars") and +// the `turn…search…` ids are protocol noise no reader can follow to a source +// (#120587). A marker that cannot be resolved to a source is dropped, never +// invented into a link. +const P = { E200: '', E201: '', E202: '' } + +describe('web citation transport markers', () => { + it('strips a single-id marker from prose', () => { + expect(preprocessMarkdown(`Ice floats because it is less dense than liquid water.${P.E200}citeturn0search0${P.E201}`)).toBe( + 'Ice floats because it is less dense than liquid water.' + ) + }) + + it('strips a multi-id marker list', () => { + const input = `answer here${P.E200}citeturn0search11${P.E202}turn2search0${P.E201} and continues` + expect(preprocessMarkdown(input)).toBe('answer here and continues') + }) + + it('strips the no-delimiter shape inside a table cell', () => { + // The reporter's table-cell case: the marker abuts a `|` cell wall. + const input = `| Firm${P.E200}citeturn0search1${P.E201} | US |` + expect(preprocessMarkdown(input)).not.toContain('citeturn') + expect(preprocessMarkdown(input)).toBe('| Firm | US |') + }) + + it('strips a bare citeturn token the model emitted without delimiters', () => { + expect(preprocessMarkdown('See citeturn3search7 for details.')).toBe('See for details.') + expect(preprocessMarkdown('A citeturn0search0.')).toBe('A .') + }) + + it('strips a marker that runs to the very end of the text mid-stream', () => { + // preprocessMarkdown runs per streaming flush; the closing U+E201 has not + // arrived yet. + expect(preprocessMarkdown(`Still typing${P.E200}citeturn0search0`)).toBe('Still typing') + }) + + it('fuses the words a stripped in-word marker separated, like the thinking strip does', () => { + // The marker is transport noise, not content: deleting it inside a word + // joins what it split (same contract as the strip's "does not + // fuse the words a stripped block separated" inverse — there the block is + // replaced by a space; here the marker is INVISIBLE noise, so the visible + // letters on either side were always adjacent). + expect(preprocessMarkdown(`word${P.E200}citeturn0search0${P.E201}word2`)).toBe('wordword2') + }) + + it('does not touch other private-use or icon-font characters', () => { + const icon = '' + expect(preprocessMarkdown(`Label ${icon} stays`)).toBe(`Label ${icon} stays`) + }) + + it('keeps marker text inside inline code and math spans intact', () => { + // The shield machinery (normalizeVisibleProse) splits on inline code and + // math before prose rewrites; the marker rule must ride the same path. + expect(preprocessMarkdown(`run \`x = ${P.E200}citeturn0search0${P.E201}\` please`)).toContain('citeturn') + expect(preprocessMarkdown(`$\\sqrt[3]{8}$ and ${P.E200}citeturn0search0${P.E201}`)).toBe('$\\sqrt[3]{8}$ and ') + }) + + it('keeps stripping numeric [n] markers alongside the transport shape', () => { + expect(preprocessMarkdown(`Both shapes[1] and ${P.E200}citeturn0search0${P.E201} go away.`)).toBe( + 'Both shapes and go away.' + ) + }) +}) diff --git a/apps/desktop/src/lib/markdown-preprocess.ts b/apps/desktop/src/lib/markdown-preprocess.ts index 0243631071..0e2eed7831 100644 --- a/apps/desktop/src/lib/markdown-preprocess.ts +++ b/apps/desktop/src/lib/markdown-preprocess.ts @@ -169,6 +169,19 @@ const SAFE_HTML_TAG_NAMES = new Set([ ]) const CITATION_MARKER_RE = /(?<=[\p{L}\p{N})\].,!?:;"'”’])\[(?:\d+(?:\s*,\s*\d+)*)\](?!\()/gu + +// Web-citation transport markers (Gemini-style grounding): private-use +// delimiters U+E200/U+E201 wrap a `citeturnsearch` id list, with U+E202 +// separating ids — `\uE200citeturn0search11\uE202turn2search0\uE201`, or the +// single-id `\uE200citeturn0search0\uE201`. The delimiters paint as +// replacement glyphs (the reported "triple bars") and the ids are protocol +// noise a reader cannot follow to a source (#120587). Strip the whole marker; +// a marker that cannot be resolved is dropped, never invented into a link. +// The bare no-delimiter alternative only fires with the `cite` head, so plain +// prose can't trip it. Scoped to this shape: stray private-use characters +// (icon fonts, user content) are left alone. +const CITATION_TRANSPORT_MARKER_RE = + /\uE200(?:cite)?(?:\uE202?turn\d+search\d+)+\uE201?|citeturn\d+search\d+(?:turn\d+search\d+)*/gu // Markdown links whose target is a filesystem path on the agent's machine: // `[report](/home/user/report.md)`, `[notes](file:///srv/notes.txt)`, // `[todo](~/todo.md)`, `[log](C:\logs\run.txt)`. Negative lookbehind keeps @@ -421,7 +434,11 @@ function rewriteProseSegment(segment: string): string { autoLinkRawUrls( routeFileLinksToPreview( escapeUnknownHtmlLikeTags( - segment.replace(/`{3,}/g, '').replace(LOCAL_PREVIEW_URL_RE, '$1').replace(CITATION_MARKER_RE, '') + segment + .replace(/`{3,}/g, '') + .replace(LOCAL_PREVIEW_URL_RE, '$1') + .replace(CITATION_TRANSPORT_MARKER_RE, '') + .replace(CITATION_MARKER_RE, '') ) ) )