From 2bfa29b697304fef4b27f7fd7cf2b5cc83446aeb Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Wed, 23 Sep 2026 04:03:20 -0700 Subject: [PATCH] test(desktop-core): one live socket per backend; sampler names the duplicate bubbles The non-default-profile step now asserts the renderer holds exactly one live socket to the host backend (red when 0bb539b4725, #120006, is reverted); the earlier per-turn check alone missed that regression. Transient-duplicate violations now report which bubble ids carry the copies. --- apps/desktop/e2e/core/oracle.ts | 40 ++++++++++-- apps/desktop/e2e/core/provider.ts | 6 +- .../e2e/core/transcript-integrity.spec.ts | 63 ++++++++++++++++--- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/apps/desktop/e2e/core/oracle.ts b/apps/desktop/e2e/core/oracle.ts index 0680e71144..09c5acc859 100644 --- a/apps/desktop/e2e/core/oracle.ts +++ b/apps/desktop/e2e/core/oracle.ts @@ -88,7 +88,25 @@ export async function installDuplicateSampler(page: Page): Promise { for (const [marker, count] of counts) { if (count > 1 && w.__coreSampler.violations.length < 20) { - w.__coreSampler.violations.push({ marker, count, text: text.replace(/\s+/g, ' ').slice(0, 600) }) + // Where the copies live: one bubble with doubled text vs two bubbles. + const bubbles = ( + [ + ...viewport.querySelectorAll( + '[data-slot="aui_user-message-root"], [data-slot="aui_assistant-message-root"]' + ) + ] as HTMLElement[] + ) + .filter(el => el.innerText.includes(marker)) + .map(el => `${el.getAttribute('data-slot')}#${el.getAttribute('data-message-id') ?? el.id ?? ''}`) + + w.__coreSampler.violations.push({ + marker, + count, + at: Math.round(performance.now()), + route: location.hash, + bubbles, + text: text.replace(/\s+/g, ' ').slice(0, 600) + }) } } } @@ -268,16 +286,19 @@ function wireViolations( if (turn.complete === null && !gapped) { problems.push(`wire ${textSegments.at(-1)!.marker}: turn never completed`) + continue } const seen = new Set() + const check = (kind: string, segs: { marker: string; text: string }[], pick: (c: RecordedCompletion) => string) => { segs.forEach((seg, i) => { const completion = byOpening.get(seg.marker) if (!completion) { problems.push(`wire: ${kind} segment ${seg.marker} was never streamed by the provider`) + return } @@ -291,10 +312,14 @@ function wireViolations( if (gapped) { if (!isSubsequence(seg.text.split(' '), sent.split(' '))) { - problems.push(`wire: ${kind} ${JSON.stringify(seg.text)} is not an in-order subsequence of ${JSON.stringify(sent)}`) + problems.push( + `wire: ${kind} ${JSON.stringify(seg.text)} is not an in-order subsequence of ${JSON.stringify(sent)}` + ) } } else if (partialAllowed ? !sent.startsWith(seg.text) : seg.text !== sent) { - problems.push(`wire: ${kind} ${JSON.stringify(seg.text)} != provider ${JSON.stringify(sent)}${completion.aborted ? ' (aborted)' : ''}`) + problems.push( + `wire: ${kind} ${JSON.stringify(seg.text)} != provider ${JSON.stringify(sent)}${completion.aborted ? ' (aborted)' : ''}` + ) } }) } @@ -305,7 +330,9 @@ function wireViolations( const final = byOpening.get(textSegments.at(-1)!.marker) if (final && turn.complete !== null && norm(turn.complete) !== norm(final.sentText)) { - problems.push(`wire: message.complete ${JSON.stringify(turn.complete)} != final completion ${JSON.stringify(final.sentText)}`) + problems.push( + `wire: message.complete ${JSON.stringify(turn.complete)} != final completion ${JSON.stringify(final.sentText)}` + ) } } @@ -354,6 +381,7 @@ function transcriptViolations(persisted: PersistedMessage[], view: RenderedView, if (occurrences !== 1) { problems.push(`rendered ${occurrences}x (want 1): ${row.role} ${JSON.stringify(content.slice(0, 80))}`) + continue } @@ -432,7 +460,9 @@ export async function assertTranscriptOracle( }) const transient = await samplerViolations(page) - expect(transient.violations, `transient duplicate render during [${label}] (${transient.samples} samples)`).toEqual([]) + expect(transient.violations, `transient duplicate render during [${label}] (${transient.samples} samples)`).toEqual( + [] + ) const wire = wireViolations(ws, provider, new Set(target.expectUserMarkers), new Set(target.lossyWire ?? [])) expect(wire, `wire integrity [${label}]`).toEqual([]) diff --git a/apps/desktop/e2e/core/provider.ts b/apps/desktop/e2e/core/provider.ts index 15d6105721..414988edd9 100644 --- a/apps/desktop/e2e/core/provider.ts +++ b/apps/desktop/e2e/core/provider.ts @@ -22,6 +22,7 @@ export interface Gate { export function gate(): Gate { let open = () => {} + const opened = new Promise(resolve => { open = resolve }) @@ -86,6 +87,7 @@ function turnPosition(messages: any[]): { marker: null | string; step: number } for (let i = messages.length - 1; i >= 0; i--) { if (messages[i]?.role === 'user') { lastUser = i + break } } @@ -240,7 +242,9 @@ export function startScriptedProvider(): Promise { const server = http.createServer((req, res) => { if (req.method === 'GET' && req.url?.startsWith('/v1/models')) { res.writeHead(200, { 'Content-Type': 'application/json' }) - res.end(JSON.stringify({ object: 'list', data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'core' }] })) + res.end( + JSON.stringify({ object: 'list', data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'core' }] }) + ) return } diff --git a/apps/desktop/e2e/core/transcript-integrity.spec.ts b/apps/desktop/e2e/core/transcript-integrity.spec.ts index e8cc0333d5..882508232e 100644 --- a/apps/desktop/e2e/core/transcript-integrity.spec.ts +++ b/apps/desktop/e2e/core/transcript-integrity.spec.ts @@ -38,13 +38,20 @@ import { import { assertTranscriptOracle, installDuplicateSampler, type OracleTarget } from './oracle' import { gate, startScriptedProvider } from './provider' -const nonce = Math.random().toString(36).slice(2, 8).replace(/[^a-z0-9]/g, 'x').padEnd(4, 'q') +const nonce = Math.random() + .toString(36) + .slice(2, 8) + .replace(/[^a-z0-9]/g, 'x') + .padEnd(4, 'q') const U = (n: number) => `U${n}-${nonce}` const A = (n: number) => `A${n}-${nonce}` const AI = (n: number) => `A${n}i-${nonce}` const R = (n: number) => `R${n}-${nonce}` -const words = (marker: string, ...rest: string[]) => [`${marker} `, ...rest.map((w, i) => (i === rest.length - 1 ? w : `${w} `))] +const words = (marker: string, ...rest: string[]) => [ + `${marker} `, + ...rest.map((w, i) => (i === rest.length - 1 ? w : `${w} `)) +] function viewport(page: Page) { return page.locator('[data-slot="aui_thread-viewport"]').filter({ visible: true }).first() @@ -94,7 +101,10 @@ test('transcript oracle holds across every transition', async () => { await test.step('tool-call turn: interim text + real terminal tool + final', async () => { provider.script(U(2), [ - { text: words(AI(2), 'checking', 'first'), toolCalls: [{ name: 'terminal', args: { command: 'echo core-tool-ok' } }] }, + { + text: words(AI(2), 'checking', 'first'), + toolCalls: [{ name: 'terminal', args: { command: 'echo core-tool-ok' } }] + }, { text: words(A(2), 'tool', 'said', 'ok') } ]) await send(page, `${U(2)} run a tool`, 'Enter', ws) @@ -104,7 +114,9 @@ test('transcript oracle holds across every transition', async () => { }) await test.step('reasoning turn: reasoning deltas never leak into or double the reply', async () => { - provider.script(U(3), [{ reasoning: words(R(3), 'weighing', 'options'), text: words(A(3), 'reasoned', 'answer') }]) + provider.script(U(3), [ + { reasoning: words(R(3), 'weighing', 'options'), text: words(A(3), 'reasoned', 'answer') } + ]) await send(page, `${U(3)} think first`, 'Enter', ws) await finished(U(3)) sessionA.expectUserMarkers.push(U(3)) @@ -216,12 +228,20 @@ test('transcript oracle holds across every transition', async () => { // Sockets that delivered the message.complete of the turn whose reply opens with `marker`. const completeSockets = (marker: string) => - new Set(ws.events.filter(e => e.type === 'message.complete' && String(e.payload?.text ?? '').startsWith(marker)).map(e => e.socket)) + new Set( + ws.events + .filter(e => e.type === 'message.complete' && String(e.payload?.text ?? '').startsWith(marker)) + .map(e => e.socket) + ) let sessionP2: OracleTarget = { sessionId: '', profile: 'p2', expectUserMarkers: [] } await test.step('non-default profile on the shared host backend: one socket per session', async () => { - await page.getByRole('button', { name: 'Bots', exact: true }).or(page.getByRole('tab', { name: 'Bots', exact: true })).first().click() + await page + .getByRole('button', { name: 'Bots', exact: true }) + .or(page.getByRole('tab', { name: 'Bots', exact: true })) + .first() + .click() const row = page.locator('[data-slot="bots-roster"] [data-roster-key="local::p2"]') await expect(row).toBeVisible({ timeout: 60_000 }) await row.click() @@ -232,7 +252,9 @@ test('transcript oracle holds across every transition', async () => { await expect(viewport(page)).toContainText(A(9)) let p2Session: null | string = null await expect - .poll(() => (p2Session = storedSessionForMarker(sandbox, 'p2', U(9))), { message: 'p2 turn persisted in the p2 state.db' }) + .poll(() => (p2Session = storedSessionForMarker(sandbox, 'p2', U(9))), { + message: 'p2 turn persisted in the p2 state.db' + }) .not.toBeNull() sessionP2 = { sessionId: p2Session!, profile: 'p2', expectUserMarkers: [U(9)] } await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 first turn') @@ -245,7 +267,22 @@ test('transcript oracle holds across every transition', async () => { sessionP2.expectUserMarkers.push(U(13)) await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 second turn') expect([...completeSockets(A(9))].length, 'p2 turn 1 delivered on exactly one socket').toBe(1) - expect([...completeSockets(A(13))].length, 'p2 turn 2 delivered on exactly one socket (no 2nd socket to one backend)').toBe(1) + expect([...completeSockets(A(13))].length, 'p2 turn 2 delivered on exactly one socket').toBe(1) + // One backend process, one socket: the host backend serves p2 too, so + // the renderer must not hold a second live socket to it (#120006). + const sameBackend = new Set([String(backendPort), String(proxy.port)]) + await expect + .poll( + () => + ws.sockets + .filter(s => !s.closed && sameBackend.has(new URL(s.url).port)) + .map(s => s.url.replace(/token=[^&]+/, 'token=…')), + { + timeout: 30_000, + message: 'live sockets to the one host backend' + } + ) + .toHaveLength(1) }) await test.step('forced second socket to the same backend: every event still renders once', async () => { @@ -256,7 +293,10 @@ test('transcript oracle holds across every transition', async () => { const socketsBefore = ws.sockets.length await splitProfileRoute(app, 'p2') provider.script(U(10), [ - { text: words(AI(10), 'looking', 'it', 'up'), toolCalls: [{ name: 'terminal', args: { command: 'echo core-two-sockets' } }] }, + { + text: words(AI(10), 'looking', 'it', 'up'), + toolCalls: [{ name: 'terminal', args: { command: 'echo core-two-sockets' } }] + }, { text: words(A(10), 'two', 'sockets', 'one', 'render') } ]) await send(page, `${U(10)} tool on p2`, 'Enter', ws) @@ -264,7 +304,10 @@ test('transcript oracle holds across every transition', async () => { await expect.poll(() => ws.sockets.length).toBeGreaterThan(socketsBefore) // Precondition of the scenario: the turn really was fanned out to 2+ sockets. await expect - .poll(() => completeSockets(A(10)).size, { timeout: 60_000, message: 'the forced turn reached more than one socket' }) + .poll(() => completeSockets(A(10)).size, { + timeout: 60_000, + message: 'the forced turn reached more than one socket' + }) .toBeGreaterThan(1) sessionP2.expectUserMarkers.push(U(10)) await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 tool turn over two sockets')