fix(desktop): attachment-tolerant transcript reconciliation (#120978)

Pasted-attachment user rows carry no rowId until hydration, so
hydratedIdFor returned undefined and matchesTailUserInNext required
exact text+refs — the conservative append duplicated the turn. Match
on attachment identity (normalized refs + text) when the rowId is
absent, reusing the new attachment-turn helpers.
This commit is contained in:
Hermes Agent
2026-09-24 19:58:07 -05:00
committed by brooklyn!
parent fc91608d85
commit 4c1f111de3
6 changed files with 370 additions and 12 deletions

View File

@@ -2152,3 +2152,57 @@ describe('preserveEquivalentTranscript', () => {
expect(preserveEquivalentTranscript(current, next)).toBe(next)
})
})
describe('preserveLocalPendingTurnMessages attachment rewrites (#120978)', () => {
it('drops the rowId-less pasted-attachment prompt once the rewritten copy commits', () => {
// A pasted clipboard image is rewritten on the durable side (marker lines,
// no data: ref) while the optimistic local row keeps the bare caption and
// the data: ref — exact text/refs equality can never match them and the
// optimistic row was re-appended below the newest turn.
const previous = [
msg('1-user', 'user', 'first'),
msg('2-assistant', 'assistant', 'first answer'),
msg('user-1790168309-ab12cd', 'user', 'unable to publish', {
attachmentRefs: ['data:image/png;base64,AAAA']
})
]
const next = [
msg('1-user-stored', 'user', 'first', { rowId: 1 }),
msg('2-assistant-stored', 'assistant', 'first answer', { rowId: 2 }),
msg('3-user-stored', 'user', 'unable to publish\n\n[Image attached at: C:\\img\\shot.png]\n[screenshot]', {
rowId: 3
})
]
expect(preserveLocalPendingTurnMessages(next, previous).map(message => message.id)).toEqual([
'1-user-stored',
'2-assistant-stored',
'3-user-stored'
])
})
it('never tolerance-matches a plain repeat prompt without attachment evidence', () => {
// The gating invariant: rewrite markers on the stored side AND attachment
// evidence on the local side. A bare repeated caption is a genuine new
// question and must survive.
const previous = [
msg('1-user', 'user', 'first'),
msg('2-assistant', 'assistant', 'first answer'),
msg('user-plain-repeat', 'user', 'unable to publish')
]
const next = [
msg('1-user-stored', 'user', 'first', { rowId: 1 }),
msg('2-assistant-stored', 'assistant', 'first answer', { rowId: 2 }),
msg('3-user-stored', 'user', 'unable to publish\n\n[Image attached at: C:\\img\\shot.png]', { rowId: 3 })
]
expect(preserveLocalPendingTurnMessages(next, previous).map(message => message.id)).toEqual([
'1-user-stored',
'2-assistant-stored',
'3-user-stored',
'user-plain-repeat'
])
})
})

View File

@@ -1,6 +1,7 @@
import { resolveSessionRpcOwner } from '@/app/contrib/wiring-routing'
import { textWithoutReferenceLines } from '@/components/assistant-ui/reference-kinds'
import { getSession } from '@/hermes'
import { sameAttachmentTurn, spliceOlderPreservedRows } from '@/lib/chat-messages'
import {
assistantTextPart,
type ChatMessage,
@@ -885,8 +886,9 @@ export function preserveLocalPendingTurnMessages(
isOptimisticUser &&
acknowledgedUserCandidates.some(
candidate =>
!conflictingTranscriptIdentity(message, candidate) &&
textWithoutReferenceLines(chatMessageText(candidate)) === textWithoutReferenceLines(chatMessageText(message))
(!conflictingTranscriptIdentity(message, candidate) &&
textWithoutReferenceLines(chatMessageText(candidate)) === textWithoutReferenceLines(chatMessageText(message))) ||
sameAttachmentTurn(candidate, message)
)
) {
continue
@@ -1002,7 +1004,10 @@ export function preserveLocalPendingTurnMessages(
const withReplacements =
replacements.size > 0 ? nextMessages.map(message => replacements.get(message.id) ?? message) : nextMessages
return preserved.length ? [...withReplacements, ...preserved] : withReplacements
// #120978: a kept run whose rowIds predate the whole hydrated page belongs
// earlier — splice it in front of the first newer row instead of appending it
// below the newest turn (non-qualifying runs keep the trailing behavior).
return preserved.length ? spliceOlderPreservedRows(withReplacements, preserved) : withReplacements
}
/**

View File

@@ -0,0 +1,70 @@
import type { ChatMessage } from '@/lib/chat-messages'
import { chatMessageText } from '@/lib/chat-messages/parts'
/**
* Attachment-rewrite tolerance for pasted attachments (#120978, follow-up to
* #119326).
*
* A pasted clipboard attachment has no rowId on the optimistic local row, and
* the backend rewrites the durable prompt around it: the caption survives but
* the refs become `[Image attached at: <path>]` / `[screenshot]` markers, and a
* `<memory-context>…</memory-context>` block can be injected. Exact text+refs
* equality can never bridge that, so `hydratedIdFor()` returns undefined and
* the conservative append path paints the local pair below the newest turn.
*
* The tolerance is gated on ATTACHMENT EVIDENCE on both sides so a plain
* repeated prompt — or a second attempt under the same caption — is never
* swallowed: the stored row must carry rewrite markers AND the local row must
* carry its own attachment evidence (refs or markers).
*/
// Rewrite markers the backend stamps onto the durable prompt of a pasted
// attachment turn. Kept in one pattern so a new marker shape lands here once.
const ATTACHMENT_REWRITE_MARKER_RE = /\[Image attached at:[^\]]*\]|\[(?:screenshot|image|attachment|file)\]/i
const MEMORY_CONTEXT_RE = /<memory-context>[\s\S]*?(?:<\/memory-context>|$)/gi
const IMAGE_ATTACHED_RE = /\[Image attached at:[^\]]*\]/gi
const SHORT_MARKER_RE = /\[(?:screenshot|image|attachment|file)\]/gi
/** The stored row carries backend rewrite markers for a pasted attachment. */
export const carriesAttachmentRewrite = (value: string): boolean => ATTACHMENT_REWRITE_MARKER_RE.test(value)
/**
* The caption a rewrite buried: memory-context block, path markers and short
* markers stripped, whitespace collapsed. Empty when the row was markers only.
*/
export const attachmentTolerantUserText = (value: string): string =>
value
.replace(MEMORY_CONTEXT_RE, ' ')
.replace(IMAGE_ATTACHED_RE, ' ')
.replace(SHORT_MARKER_RE, ' ')
.replace(/\s+/g, ' ')
.trim()
/** Either side showing attachment evidence: real refs or rewrite markers. */
const carriesAttachmentEvidence = (message: ChatMessage): boolean =>
(message.attachmentRefs ?? []).some(ref => ref.trim().length > 0) ||
carriesAttachmentRewrite(chatMessageText(message))
/**
* Same attachment turn, tolerantly: the stored row carries rewrite markers,
* the local row carries attachment evidence, and the tolerant captions are
* equal. The marker gate means a plain repeat of the caption never matches.
*/
export const sameAttachmentTurn = (stored: ChatMessage, local: ChatMessage): boolean => {
if (stored.role !== 'user' || local.role !== 'user') {
return false
}
const storedText = chatMessageText(stored)
if (!carriesAttachmentRewrite(storedText)) {
return false
}
if (!carriesAttachmentEvidence(local)) {
return false
}
return attachmentTolerantUserText(storedText) === attachmentTolerantUserText(chatMessageText(local))
}

View File

@@ -1,3 +1,4 @@
export { sameAttachmentTurn } from './attachment-turn'
export { toChatMessages } from './hydration'
export {
appendAssistantTextPart,
@@ -14,7 +15,7 @@ export {
textPart
} from './parts'
export type { UnspokenTurnSpeech } from './parts'
export { branchGroupForUser, preserveLocalAssistantErrors } from './reconciliation'
export { branchGroupForUser, preserveLocalAssistantErrors, spliceOlderPreservedRows } from './reconciliation'
export {
restorePendingBlockingToolCall,
restorePendingClarifyToolCall,

View File

@@ -215,3 +215,123 @@ it('moves a local error onto the durable row it already represents (#119326)', (
expect(merged.map(message => message.id)).toEqual(['9-0-user', '9-1-assistant'])
expect(merged[1]).toMatchObject({ error: 'upstream timeout', pending: false })
})
it('does not re-append a rowId-less pasted-attachment prompt rewritten by the backend (#120978)', () => {
// The pasted clipboard image has no rowId on the optimistic local row, and
// the durable prompt is rewritten to marker lines + an injected
// memory-context block, so no exact text/refs compare can tie them.
const merged = preserveLocalAssistantErrors(
[
row('9-0-user', 'user', 'first', { rowId: 1 }),
row('9-1-assistant', 'assistant', 'first answer', { rowId: 2 }),
row('9-2-user', 'user', 'unable to publish\n\n[Image attached at: C:\\img\\shot.png]\n[screenshot]', { rowId: 3 })
],
[
row('1-0-user', 'user', 'first', { rowId: 1 }),
row('1-1-assistant', 'assistant', 'first answer', { rowId: 2 }),
row('user-1790168309-ab12cd', 'user', 'unable to publish', {
attachmentRefs: ['data:image/png;base64,AAAA']
})
]
)
expect(merged.map(message => message.id)).toEqual(['9-0-user', '9-1-assistant', '9-2-user'])
})
it('folds a preserved attachment error onto the durable reply via the tolerant caption (#120978)', () => {
// The errored assistant's hydrated row exists, but neither it nor the user
// row can be matched by rowId (the local pair carries none) — the tolerant
// caption match must fold the error onto the durable reply and drop the
// optimistic pair instead of preserving both at the tail.
const merged = preserveLocalAssistantErrors(
[
row('9-0-user', 'user', 'unable to publish\n\n[Image attached at: C:\\img\\shot.png]', { rowId: 18711 }),
row('9-1-assistant', 'assistant', 'partial', { rowId: 18715 }),
row('9-2-user', 'user', 'later question', { rowId: 18817 }),
row('9-3-assistant', 'assistant', 'later answer', { rowId: 18822 })
],
[
row('user-1790168309-ab12cd', 'user', 'unable to publish', {
attachmentRefs: ['data:image/png;base64,AAAA']
}),
row('assistant-stream-deadbeef', 'assistant', 'partial', { error: 'upstream timeout' }),
row('9-2-user', 'user', 'later question', { rowId: 18817 }),
row('9-3-assistant', 'assistant', 'later answer', { rowId: 18822 })
]
)
expect(merged.map(message => message.id)).toEqual([
'9-0-user',
'9-1-assistant',
'9-2-user',
'9-3-assistant'
])
expect(merged[1]).toMatchObject({ error: 'upstream timeout', pending: false })
})
it('never tolerance-matches a plain repeated prompt without attachment evidence (#120978)', () => {
// Gating: the stored row carries rewrite markers but the local repeat is a
// bare caption with no refs — a genuine repeat must survive.
const merged = preserveLocalAssistantErrors(
[
row('9-0-user', 'user', 'unable to publish\n\n[Image attached at: C:\\img\\shot.png]', { rowId: 18711 }),
row('9-1-assistant', 'assistant', 'stored reply', { rowId: 18712 })
],
[
row('1-0-user', 'user', 'earlier', { rowId: 100 }),
row('1-1-assistant', 'assistant', 'earlier answer', { rowId: 101 }),
row('user-repeat', 'user', 'unable to publish'),
row('assistant-stream-x', 'assistant', 'stored reply', { error: 'upstream timeout' })
]
)
expect(merged.map(message => message.id)).toEqual([
'9-0-user',
'9-1-assistant',
'user-repeat',
'assistant-stream-x'
])
})
it('splices an older-rowId preserved run in front of the first newer hydrated row (#120978)', () => {
// The windowed hydrated page starts past the failed turn; the kept pair
// (user 210 + errored assistant 211) must land ABOVE the newer turn, not
// below it.
const merged = preserveLocalAssistantErrors(
[
row('9-0-user', 'user', 'newer question', { rowId: 220 }),
row('9-1-assistant', 'assistant', 'newer answer', { rowId: 221 })
],
[
row('user-210', 'user', 'older question', { rowId: 210 }),
row('assistant-stream-211', 'assistant', 'older partial', { rowId: 211, error: 'upstream timeout' }),
row('9-0-user', 'user', 'newer question', { rowId: 220 }),
row('9-1-assistant', 'assistant', 'newer answer', { rowId: 221 })
]
)
expect(merged.map(message => message.rowId)).toEqual([210, 211, 220, 221])
expect(merged[1]).toMatchObject({ error: 'upstream timeout', pending: false })
})
it('keeps a rowId-less preserved run trailing (#118002 behavior unchanged)', () => {
const merged = preserveLocalAssistantErrors(
[
row('9-0-user', 'user', 'newer question', { rowId: 220 }),
row('9-1-assistant', 'assistant', 'newer answer', { rowId: 221 })
],
[
row('user-no-row', 'user', 'older question'),
row('assistant-stream-x', 'assistant', 'older partial', { error: 'upstream timeout' }),
row('9-0-user', 'user', 'newer question', { rowId: 220 }),
row('9-1-assistant', 'assistant', 'newer answer', { rowId: 221 })
]
)
expect(merged.map(message => message.id)).toEqual([
'9-0-user',
'9-1-assistant',
'user-no-row',
'assistant-stream-x'
])
})

View File

@@ -1,3 +1,4 @@
import { sameAttachmentTurn } from './attachment-turn'
import { chatMessageText } from './parts'
import type { ChatMessage, ChatMessagePart } from './types'
@@ -43,8 +44,13 @@ const assistantTimelineMatch = (stored: ChatMessage, local: ChatMessage) => {
const userTurnMatch = (stored: ChatMessage, local: ChatMessage) =>
stored.role === 'user' &&
local.role === 'user' &&
normalizedTimelineText(stored) === normalizedTimelineText(local) &&
(stored.attachmentRefs ?? []).join('\n') === (local.attachmentRefs ?? []).join('\n')
((normalizedTimelineText(stored) === normalizedTimelineText(local) &&
(stored.attachmentRefs ?? []).join('\n') === (local.attachmentRefs ?? []).join('\n')) ||
// A pasted attachment turn is rewritten on the durable side while the
// optimistic local row keeps the bare caption + a data: ref (#120978);
// the marker-gated tolerant compare bridges the two without ever
// matching a plain repeated prompt.
sameAttachmentTurn(stored, local))
/**
* Find the hydrated assistant representing a local failed tail turn.
@@ -99,6 +105,46 @@ const tailTurnAssistantMatchIndex = (
return storedMessages.findLastIndex(visibleAssistant)
}
/**
* #120978: find the hydrated assistant representing a rowId-less errored
* attachment turn whose user row cannot be matched exactly. The local user
* row directly before the errored assistant is compared tolerantly
* (attachment-rewrite gated) against every hydrated user row; the first
* settled assistant reply after the match is the durable twin of the failed
* turn. Returns -1 when no rewrite-marker pair exists — plain turns keep the
* conservative preserve path.
*/
const attachmentTurnAssistantMatchIndex = (
storedMessages: ChatMessage[],
localMessages: ChatMessage[],
localAssistantIndex: number
) => {
const localUser = localMessages
.slice(0, localAssistantIndex)
.reverse()
.find(message => message.role === 'user' && !message.hidden)
if (!localUser) {
return -1
}
const storedUserIndex = storedMessages.findIndex(stored => stored.role === 'user' && sameAttachmentTurn(stored, localUser))
if (storedUserIndex === -1) {
return -1
}
const reply = storedMessages
.slice(storedUserIndex + 1)
.find(message => message.role === 'assistant' && !message.hidden && !message.pending && !message.interim)
if (!reply) {
return -1
}
return storedMessages.indexOf(reply)
}
const timelinePartMatch = (stored: ChatMessagePart, local: ChatMessagePart) => {
if (stored.type !== local.type) {
return false
@@ -208,6 +254,41 @@ function mergeStoredAssistantErrors(nextMessages: ChatMessage[], currentMessages
const normalizedMessageText = (message: ChatMessage): string => chatMessageText(message).replace(/\s+/g, ' ').trim()
/**
* Older-rowId preserved runs (#120978): a kept run whose rows ALL carry
* rowIds older than every hydrated rowId belongs EARLIER in the transcript —
* the windowed page simply starts past them. Appending at the tail paints
* them below the newest turn; splice them in front of the first hydrated row
* newer than the whole run instead. Runs that do not qualify (rowId-less
* optimistic rows, rowIds interleaved with the hydrated page) keep the
* trailing behavior (#118002).
*/
export function spliceOlderPreservedRows(merged: ChatMessage[], preserved: ChatMessage[]): ChatMessage[] {
if (!preserved.length) {
return merged
}
const keptRowIds = preserved.map(row => row.rowId)
if (keptRowIds.some(id => id === undefined)) {
return [...merged, ...preserved]
}
const maxKept = Math.max(...(keptRowIds as number[]))
const hydratedRowIds = merged.flatMap(row => (row.rowId !== undefined ? [row.rowId] : []))
if (!hydratedRowIds.length || hydratedRowIds.some(id => id <= maxKept)) {
return [...merged, ...preserved]
}
const out = [...merged]
const at = out.findIndex(row => (row.rowId ?? -Infinity) > maxKept)
out.splice(at === -1 ? out.length : at, 0, ...preserved)
return out
}
// Renderer ids are positional, so a hydrated page can carry a local row under
// a new id; its durable rowId still names the same row (#119326).
function hydratedIdResolver(mergedNextMessages: ChatMessage[]): (message: ChatMessage) => string | undefined {
@@ -237,10 +318,16 @@ function localAssistantErrorIdsToPreserve(
const tailUserText = tailUserInNext ? normalizedMessageText(tailUserInNext) : ''
const tailUserRefs = tailUserInNext ? (tailUserInNext.attachmentRefs ?? []).join('\n') : ''
// A pasted attachment is rewritten on the durable side (marker lines + injected
// memory-context) while the optimistic local row keeps the bare caption and a
// data: ref, with no rowId to bridge them (#120978). The tolerant arm is
// gated on rewrite markers + local attachment evidence so plain repeats are
// never swallowed.
const matchesTailUserInNext = (candidate: ChatMessage): boolean =>
Boolean(tailUserInNext) &&
normalizedMessageText(candidate) === tailUserText &&
(candidate.attachmentRefs ?? []).join('\n') === tailUserRefs
((normalizedMessageText(candidate) === tailUserText &&
(candidate.attachmentRefs ?? []).join('\n') === tailUserRefs) ||
(tailUserInNext ? sameAttachmentTurn(tailUserInNext, candidate) : false))
for (let index = 0; index < currentMessages.length; index += 1) {
const message = currentMessages[index]
@@ -256,6 +343,27 @@ function localAssistantErrorIdsToPreserve(
? tailTurnAssistantMatchIndex(mergedNextMessages, currentMessages, index)
: mergedNextMessages.findIndex(candidate => candidate.id === hydratedId && candidate.role === 'assistant')
// #120978: a rowId-less errored attachment turn cannot be matched by rowId
// and the tail-anchor match bails when newer turns already committed. The
// preceding local user row's caption, compared tolerantly against every
// hydrated user row, still names the turn: fold the error onto the first
// settled assistant reply after that row.
const hydratedAttachmentAssistantIndex =
hydratedAssistantIndex === -1
? attachmentTurnAssistantMatchIndex(mergedNextMessages, currentMessages, index)
: -1
if (hydratedAttachmentAssistantIndex !== -1) {
mergedNextMessages[hydratedAttachmentAssistantIndex] = {
...mergedNextMessages[hydratedAttachmentAssistantIndex],
error: message.error,
...(message.errorSurface ? { errorSurface: message.errorSurface } : {}),
pending: false
}
continue
}
if (hydratedAssistantIndex !== -1) {
mergedNextMessages[hydratedAssistantIndex] = {
...mergedNextMessages[hydratedAssistantIndex],
@@ -340,10 +448,10 @@ function insertPreservedErrorRuns(
keptAfter.set(after, [...(keptAfter.get(after) ?? []), ...rows])
}
return [
...mergedNextMessages.flatMap(message => [message, ...(keptAfter.get(message.id) ?? [])]),
...(keptAfter.get(undefined) ?? [])
]
return spliceOlderPreservedRows(
mergedNextMessages.flatMap(message => [message, ...(keptAfter.get(message.id) ?? [])]),
keptAfter.get(undefined) ?? []
)
}
export function preserveLocalAssistantErrors(