hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

test(desktop-core): one live socket per backend; sampler names the duplicate bubbles

Browse Source 
The non-default-profile step now asserts the renderer holds exactly one live
socket to the host backend (red when 0bb539b472, #120006, is reverted); the
earlier per-turn check alone missed that regression. Transient-duplicate
violations now report which bubble ids carry the copies.
This commit is contained in:
teknium1
2026-09-23 04:03:20 -07:00
committed by Teknium
parent 75908c1d23
commit 2bfa29b697
3 changed files with 93 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
apps/desktop/e2e/core/oracle.ts
View File

@@ -88,7 +88,25 @@ export async function installDuplicateSampler(page: Page): Promise<void> {
for (const [marker, count] of counts) {
if (count > 1 && w.__coreSampler.violations.length < 20) {
w.__coreSampler.violations.push({ marker, count, text: text.replace(/\s+/g, ' ').slice(0, 600) })
// Where the copies live: one bubble with doubled text vs two bubbles.
const bubbles = (
[
...viewport.querySelectorAll(
'[data-slot="aui_user-message-root"], [data-slot="aui_assistant-message-root"]'
)
] as HTMLElement[]
)
.filter(el => el.innerText.includes(marker))
.map(el => `${el.getAttribute('data-slot')}#${el.getAttribute('data-message-id') ?? el.id ?? ''}`)
w.__coreSampler.violations.push({
marker,
count,
at: Math.round(performance.now()),
route: location.hash,
bubbles,
text: text.replace(/\s+/g, ' ').slice(0, 600)
})
}
}
}
@@ -268,16 +286,19 @@ function wireViolations(
if (turn.complete === null && !gapped) {
problems.push(`wire ${textSegments.at(-1)!.marker}: turn never completed`)
continue
}
const seen = new Set<string>()
const check = (kind: string, segs: { marker: string; text: string }[], pick: (c: RecordedCompletion) => string) => {
segs.forEach((seg, i) => {
const completion = byOpening.get(seg.marker)
if (!completion) {
problems.push(`wire: ${kind} segment ${seg.marker} was never streamed by the provider`)
return
}
@@ -291,10 +312,14 @@ function wireViolations(
if (gapped) {
if (!isSubsequence(seg.text.split(' '), sent.split(' '))) {
problems.push(`wire: ${kind} ${JSON.stringify(seg.text)} is not an in-order subsequence of ${JSON.stringify(sent)}`)
problems.push(
`wire: ${kind} ${JSON.stringify(seg.text)} is not an in-order subsequence of ${JSON.stringify(sent)}`
)
}
} else if (partialAllowed ? !sent.startsWith(seg.text) : seg.text !== sent) {
problems.push(`wire: ${kind} ${JSON.stringify(seg.text)} != provider ${JSON.stringify(sent)}${completion.aborted ? ' (aborted)' : ''}`)
problems.push(
`wire: ${kind} ${JSON.stringify(seg.text)} != provider ${JSON.stringify(sent)}${completion.aborted ? ' (aborted)' : ''}`
)
}
})
}
@@ -305,7 +330,9 @@ function wireViolations(
const final = byOpening.get(textSegments.at(-1)!.marker)
if (final && turn.complete !== null && norm(turn.complete) !== norm(final.sentText)) {
problems.push(`wire: message.complete ${JSON.stringify(turn.complete)} != final completion ${JSON.stringify(final.sentText)}`)
problems.push(
`wire: message.complete ${JSON.stringify(turn.complete)} != final completion ${JSON.stringify(final.sentText)}`
)
}
}
@@ -354,6 +381,7 @@ function transcriptViolations(persisted: PersistedMessage[], view: RenderedView,
if (occurrences !== 1) {
problems.push(`rendered ${occurrences}x (want 1): ${row.role} ${JSON.stringify(content.slice(0, 80))}`)
continue
}
@@ -432,7 +460,9 @@ export async function assertTranscriptOracle(
})
const transient = await samplerViolations(page)
expect(transient.violations, `transient duplicate render during [${label}] (${transient.samples} samples)`).toEqual([])
expect(transient.violations, `transient duplicate render during [${label}] (${transient.samples} samples)`).toEqual(
[]
)
const wire = wireViolations(ws, provider, new Set(target.expectUserMarkers), new Set(target.lossyWire ?? []))
expect(wire, `wire integrity [${label}]`).toEqual([])

6
apps/desktop/e2e/core/provider.ts
View File

@@ -22,6 +22,7 @@ export interface Gate {
export function gate(): Gate {
let open = () => {}
const opened = new Promise<void>(resolve => {
open = resolve
})
@@ -86,6 +87,7 @@ function turnPosition(messages: any[]): { marker: null | string; step: number }
for (let i = messages.length - 1; i >= 0; i--) {
if (messages[i]?.role === 'user') {
lastUser = i
break
}
}
@@ -240,7 +242,9 @@ export function startScriptedProvider(): Promise<ScriptedProvider> {
const server = http.createServer((req, res) => {
if (req.method === 'GET' && req.url?.startsWith('/v1/models')) {
res.writeHead(200, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ object: 'list', data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'core' }] }))
res.end(
JSON.stringify({ object: 'list', data: [{ id: 'mock-model', object: 'model', created: 0, owned_by: 'core' }] })
)
return
}

63
apps/desktop/e2e/core/transcript-integrity.spec.ts
View File

@@ -38,13 +38,20 @@ import {
import { assertTranscriptOracle, installDuplicateSampler, type OracleTarget } from './oracle'
import { gate, startScriptedProvider } from './provider'
const nonce = Math.random().toString(36).slice(2, 8).replace(/[^a-z0-9]/g, 'x').padEnd(4, 'q')
const nonce = Math.random()
.toString(36)
.slice(2, 8)
.replace(/[^a-z0-9]/g, 'x')
.padEnd(4, 'q')
const U = (n: number) => `U${n}-${nonce}`
const A = (n: number) => `A${n}-${nonce}`
const AI = (n: number) => `A${n}i-${nonce}`
const R = (n: number) => `R${n}-${nonce}`
const words = (marker: string, ...rest: string[]) => [`${marker} `, ...rest.map((w, i) => (i === rest.length - 1 ? w : `${w} `))]
const words = (marker: string, ...rest: string[]) => [
`${marker} `,
...rest.map((w, i) => (i === rest.length - 1 ? w : `${w} `))
]
function viewport(page: Page) {
return page.locator('[data-slot="aui_thread-viewport"]').filter({ visible: true }).first()
@@ -94,7 +101,10 @@ test('transcript oracle holds across every transition', async () => {
await test.step('tool-call turn: interim text + real terminal tool + final', async () => {
provider.script(U(2), [
{ text: words(AI(2), 'checking', 'first'), toolCalls: [{ name: 'terminal', args: { command: 'echo core-tool-ok' } }] },
{
text: words(AI(2), 'checking', 'first'),
toolCalls: [{ name: 'terminal', args: { command: 'echo core-tool-ok' } }]
},
{ text: words(A(2), 'tool', 'said', 'ok') }
])
await send(page, `${U(2)} run a tool`, 'Enter', ws)
@@ -104,7 +114,9 @@ test('transcript oracle holds across every transition', async () => {
})
await test.step('reasoning turn: reasoning deltas never leak into or double the reply', async () => {
provider.script(U(3), [{ reasoning: words(R(3), 'weighing', 'options'), text: words(A(3), 'reasoned', 'answer') }])
provider.script(U(3), [
{ reasoning: words(R(3), 'weighing', 'options'), text: words(A(3), 'reasoned', 'answer') }
])
await send(page, `${U(3)} think first`, 'Enter', ws)
await finished(U(3))
sessionA.expectUserMarkers.push(U(3))
@@ -216,12 +228,20 @@ test('transcript oracle holds across every transition', async () => {
// Sockets that delivered the message.complete of the turn whose reply opens with `marker`.
const completeSockets = (marker: string) =>
new Set(ws.events.filter(e => e.type === 'message.complete' && String(e.payload?.text ?? '').startsWith(marker)).map(e => e.socket))
new Set(
ws.events
.filter(e => e.type === 'message.complete' && String(e.payload?.text ?? '').startsWith(marker))
.map(e => e.socket)
)
let sessionP2: OracleTarget = { sessionId: '', profile: 'p2', expectUserMarkers: [] }
await test.step('non-default profile on the shared host backend: one socket per session', async () => {
await page.getByRole('button', { name: 'Bots', exact: true }).or(page.getByRole('tab', { name: 'Bots', exact: true })).first().click()
await page
.getByRole('button', { name: 'Bots', exact: true })
.or(page.getByRole('tab', { name: 'Bots', exact: true }))
.first()
.click()
const row = page.locator('[data-slot="bots-roster"] [data-roster-key="local::p2"]')
await expect(row).toBeVisible({ timeout: 60_000 })
await row.click()
@@ -232,7 +252,9 @@ test('transcript oracle holds across every transition', async () => {
await expect(viewport(page)).toContainText(A(9))
let p2Session: null | string = null
await expect
.poll(() => (p2Session = storedSessionForMarker(sandbox, 'p2', U(9))), { message: 'p2 turn persisted in the p2 state.db' })
.poll(() => (p2Session = storedSessionForMarker(sandbox, 'p2', U(9))), {
message: 'p2 turn persisted in the p2 state.db'
})
.not.toBeNull()
sessionP2 = { sessionId: p2Session!, profile: 'p2', expectUserMarkers: [U(9)] }
await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 first turn')
@@ -245,7 +267,22 @@ test('transcript oracle holds across every transition', async () => {
sessionP2.expectUserMarkers.push(U(13))
await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 second turn')
expect([...completeSockets(A(9))].length, 'p2 turn 1 delivered on exactly one socket').toBe(1)
expect([...completeSockets(A(13))].length, 'p2 turn 2 delivered on exactly one socket (no 2nd socket to one backend)').toBe(1)
expect([...completeSockets(A(13))].length, 'p2 turn 2 delivered on exactly one socket').toBe(1)
// One backend process, one socket: the host backend serves p2 too, so
// the renderer must not hold a second live socket to it (#120006).
const sameBackend = new Set([String(backendPort), String(proxy.port)])
await expect
.poll(
() =>
ws.sockets
.filter(s => !s.closed && sameBackend.has(new URL(s.url).port))
.map(s => s.url.replace(/token=[^&]+/, 'token=…')),
{
timeout: 30_000,
message: 'live sockets to the one host backend'
}
)
.toHaveLength(1)
})
await test.step('forced second socket to the same backend: every event still renders once', async () => {
@@ -256,7 +293,10 @@ test('transcript oracle holds across every transition', async () => {
const socketsBefore = ws.sockets.length
await splitProfileRoute(app, 'p2')
provider.script(U(10), [
{ text: words(AI(10), 'looking', 'it', 'up'), toolCalls: [{ name: 'terminal', args: { command: 'echo core-two-sockets' } }] },
{
text: words(AI(10), 'looking', 'it', 'up'),
toolCalls: [{ name: 'terminal', args: { command: 'echo core-two-sockets' } }]
},
{ text: words(A(10), 'two', 'sockets', 'one', 'render') }
])
await send(page, `${U(10)} tool on p2`, 'Enter', ws)
@@ -264,7 +304,10 @@ test('transcript oracle holds across every transition', async () => {
await expect.poll(() => ws.sockets.length).toBeGreaterThan(socketsBefore)
// Precondition of the scenario: the turn really was fanned out to 2+ sockets.
await expect
.poll(() => completeSockets(A(10)).size, { timeout: 60_000, message: 'the forced turn reached more than one socket' })
.poll(() => completeSockets(A(10)).size, {
timeout: 60_000,
message: 'the forced turn reached more than one socket'
})
.toBeGreaterThan(1)
sessionP2.expectUserMarkers.push(U(10))
await assertTranscriptOracle(page, ws, provider, sessionP2, 'p2 tool turn over two sockets')