fix(desktop): attach renderer-lifecycle diagnostics to all BrowserWindow instances (#81290)
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -197,3 +197,6 @@ native/fts5_cjk/*.so
|
||||
# interrupted; consumed by launch-time recovery. Never commit it (was tracked
|
||||
# by accident via 3a69e34702, removed in the #72002 salvage).
|
||||
.lazy-refresh-incomplete
|
||||
|
||||
# OMC working artifacts
|
||||
.omc/
|
||||
|
||||
@@ -220,6 +220,7 @@ import { fetchMarketplaceThemes, searchMarketplaceThemes } from './vscode-market
|
||||
import { createWakeIndicatorWindowController } from './wake-indicator-window'
|
||||
import { readWindowBelow } from './window-below'
|
||||
import { createWindowRevealController } from './window-reveal'
|
||||
import { installWindowRendererLifecycle } from './window-renderer-lifecycle'
|
||||
import {
|
||||
bindGeometryPersistence,
|
||||
computeWindowOptions,
|
||||
@@ -1087,13 +1088,15 @@ const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDL
|
||||
// killing one to honor the soft cap would abort a running agent.
|
||||
const POOL_KEEPALIVE_FRESH_MS = 90_000
|
||||
let poolIdleReaper = null
|
||||
// Auto-reload budget for renderer crashes. A deterministic startup crash would
|
||||
// otherwise loop forever (reload → crash → reload), pinning CPU and spamming
|
||||
// logs. Allow a few reloads per rolling window, then stop and leave the dead
|
||||
// window so the user can read the error / quit.
|
||||
// Auto-reload budget for renderer crashes, shared by EVERY window (primary,
|
||||
// secondary session, instance) so a crash loop anywhere is suppressed after
|
||||
// the same budget instead of reloading per-window forever. A deterministic
|
||||
// startup crash would otherwise loop forever (reload → crash → reload),
|
||||
// pinning CPU and spamming logs. Allow a few reloads per rolling window, then
|
||||
// stop and leave the dead window so the user can read the error / quit.
|
||||
const RENDERER_RELOAD_WINDOW_MS = 60_000
|
||||
const RENDERER_RELOAD_MAX = 3
|
||||
let rendererReloadTimes = []
|
||||
const rendererReloadTimesRef: { current: number[] } = { current: [] }
|
||||
// Latched bootstrap failure: when the first-launch install fails, we hold
|
||||
// onto the error so subsequent startHermes() calls (e.g. the renderer's
|
||||
// ensureGatewayOpen retrying after the WS won't open) return the same error
|
||||
@@ -8919,6 +8922,23 @@ function spawnSecondaryWindow({ sessionId, watch }: { sessionId?: string; watch?
|
||||
wireCommonWindowHandlers(win, zoomWiringForWindowKind('chat'))
|
||||
attachRendererConsoleCapture(win, 'session-window', rememberLog)
|
||||
|
||||
// Renderer lifecycle diagnostics + recovery (#81290): a dead session-window
|
||||
// renderer used to log nothing and stay black; now it logs with its window
|
||||
// kind and reloads under the shared crash-loop budget, exactly like the
|
||||
// primary window, without touching any other window.
|
||||
installWindowRendererLifecycle(win, {
|
||||
kind: 'secondary',
|
||||
callbacks: {
|
||||
log: rememberLog,
|
||||
reload: () => {
|
||||
win.webContents.reload()
|
||||
}
|
||||
},
|
||||
reloadWindowMs: RENDERER_RELOAD_WINDOW_MS,
|
||||
reloadMax: RENDERER_RELOAD_MAX,
|
||||
recentReloadTimesRef: rendererReloadTimesRef
|
||||
})
|
||||
|
||||
loadWindowUrl(
|
||||
win,
|
||||
buildSessionWindowUrl(sessionId, {
|
||||
@@ -8999,6 +9019,22 @@ function createInstanceWindow() {
|
||||
streamThrottle.register(win)
|
||||
wireCommonWindowHandlers(win, zoomWiringForWindowKind('chat'))
|
||||
|
||||
// Renderer lifecycle diagnostics + recovery (#81290), same policy as the
|
||||
// primary and session windows: a crashed instance renderer logs with its
|
||||
// window kind and reloads under the shared crash-loop budget.
|
||||
installWindowRendererLifecycle(win, {
|
||||
kind: 'instance',
|
||||
callbacks: {
|
||||
log: rememberLog,
|
||||
reload: () => {
|
||||
win.webContents.reload()
|
||||
}
|
||||
},
|
||||
reloadWindowMs: RENDERER_RELOAD_WINDOW_MS,
|
||||
reloadMax: RENDERER_RELOAD_MAX,
|
||||
recentReloadTimesRef: rendererReloadTimesRef
|
||||
})
|
||||
|
||||
win.on('closed', () => {
|
||||
instanceWindows.delete(win)
|
||||
})
|
||||
@@ -9015,6 +9051,7 @@ const wakeIndicatorController = createWakeIndicatorWindowController({
|
||||
devServer: DEV_SERVER,
|
||||
isMac: IS_MAC,
|
||||
loadWindowUrl,
|
||||
log: rememberLog,
|
||||
preloadPath: PRELOAD_PATH,
|
||||
rendererIndex: resolveRendererIndex,
|
||||
wireWindow: window => wireCommonWindowHandlers(window, zoomWiringForWindowKind('wakeIndicator'))
|
||||
@@ -9110,6 +9147,10 @@ function spawnPetOverlayWindow(bounds) {
|
||||
|
||||
wireWindowReveal(win, { show: () => win.showInactive() })
|
||||
|
||||
// Log-only renderer lifecycle (#81290): a dead overlay must never resurrect
|
||||
// itself over the app, but its loss belongs in desktop.log.
|
||||
installWindowRendererLifecycle(win, { kind: 'overlay', callbacks: { log: rememberLog } })
|
||||
|
||||
win.on('closed', () => {
|
||||
if (petOverlayWindow === win) {
|
||||
petOverlayWindow = null
|
||||
@@ -9670,6 +9711,10 @@ function spawnQuickEntryWindow() {
|
||||
// its own OS window and a zoomed composer would overflow it.
|
||||
wireCommonWindowHandlers(win, zoomWiringForWindowKind('quickEntry'))
|
||||
|
||||
// Log-only renderer lifecycle (#81290): a dead quick-entry window must never
|
||||
// resurrect itself over the app, but its loss belongs in desktop.log.
|
||||
installWindowRendererLifecycle(win, { kind: 'quick', callbacks: { log: rememberLog } })
|
||||
|
||||
// Hide on blur. The window must never hold the user's focus captive — losing
|
||||
// focus is the cheapest, least surprising dismiss (matches Spotlight).
|
||||
win.on('blur', () => {
|
||||
@@ -9908,71 +9953,60 @@ function createWindow() {
|
||||
streamThrottle.register(mainWindow)
|
||||
wireCommonWindowHandlers(mainWindow, zoomWiringForWindowKind('chat'))
|
||||
|
||||
mainWindow.webContents.on('render-process-gone', (_event, details) => {
|
||||
rememberLog(`[renderer] render-process-gone reason=${details?.reason} exitCode=${details?.exitCode}`)
|
||||
|
||||
if (details?.reason === 'crashed' || details?.reason === 'oom') {
|
||||
const now = Date.now()
|
||||
rendererReloadTimes = rendererReloadTimes.filter(t => now - t < RENDERER_RELOAD_WINDOW_MS)
|
||||
|
||||
if (rendererReloadTimes.length >= RENDERER_RELOAD_MAX) {
|
||||
rememberLog(
|
||||
`[renderer] suppressing reload: ${rendererReloadTimes.length} crashes within ${RENDERER_RELOAD_WINDOW_MS}ms (likely a crash loop)`
|
||||
)
|
||||
|
||||
// Per-window renderer lifecycle diagnostics + recovery (#81290). The reload
|
||||
// policy (crashed/oom → bounded reload via the shared rolling budget, then
|
||||
// the #38216 Windows sandbox relaunch check on suppression) is the same
|
||||
// policy this window used before it moved into the shared helper, so a
|
||||
// crashed peer renderer now logs and recovers exactly like the primary one.
|
||||
installWindowRendererLifecycle(mainWindow, {
|
||||
kind: 'main',
|
||||
callbacks: {
|
||||
log: rememberLog,
|
||||
reload: () => {
|
||||
mainWindow.webContents.reload()
|
||||
},
|
||||
onCrashLoopSuppressed: details => {
|
||||
// #38216 renderer flavor (same recovery as #56726, credit @Sahil-SS9):
|
||||
// a deterministic Windows renderer crash loop with the sandbox
|
||||
// breakpoint signature gets one --no-sandbox relaunch instead of a
|
||||
// dead window. Gated on the exit code so unrelated crash loops don't
|
||||
// silently drop the sandbox.
|
||||
if (
|
||||
shouldRelaunchForRendererSandboxCrashLoop({
|
||||
!shouldRelaunchForRendererSandboxCrashLoop({
|
||||
reason: details?.reason,
|
||||
exitCode: details?.exitCode,
|
||||
alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env),
|
||||
relaunchAttempted: windowsNoSandboxRelaunchAttempted
|
||||
})
|
||||
) {
|
||||
windowsNoSandboxRelaunchAttempted = true
|
||||
windowsSandboxFallbackActive = true
|
||||
windowsSandboxFallbackSticky = true
|
||||
windowsSandboxFallbackReason = 'renderer-crash-loop'
|
||||
|
||||
try {
|
||||
writeSandboxMarker(app.getPath('userData'), fallbackMarker('renderer-crash-loop', app.getVersion()))
|
||||
} catch {
|
||||
void 0
|
||||
}
|
||||
|
||||
rememberLog('[renderer] Windows sandbox crash loop detected; relaunching once with --no-sandbox (#38216)')
|
||||
|
||||
try {
|
||||
app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) })
|
||||
app.exit(0)
|
||||
} catch (err) {
|
||||
rememberLog(`[renderer] --no-sandbox relaunch failed: ${err?.message || err}`)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
rendererReloadTimes.push(now)
|
||||
setImmediate(() => {
|
||||
if (!mainWindow || mainWindow.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
mainWindow.webContents.reload()
|
||||
} catch (err) {
|
||||
rememberLog(`[renderer] reload after crash failed: ${err?.message || err}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
windowsNoSandboxRelaunchAttempted = true
|
||||
windowsSandboxFallbackActive = true
|
||||
windowsSandboxFallbackSticky = true
|
||||
windowsSandboxFallbackReason = 'renderer-crash-loop'
|
||||
|
||||
mainWindow.webContents.on('unresponsive', () => rememberLog('[renderer] webContents became unresponsive'))
|
||||
try {
|
||||
writeSandboxMarker(app.getPath('userData'), fallbackMarker('renderer-crash-loop', app.getVersion()))
|
||||
} catch {
|
||||
void 0
|
||||
}
|
||||
|
||||
rememberLog('[renderer] Windows sandbox crash loop detected; relaunching once with --no-sandbox (#38216)')
|
||||
|
||||
try {
|
||||
app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) })
|
||||
app.exit(0)
|
||||
} catch (err) {
|
||||
rememberLog(`[renderer] --no-sandbox relaunch failed: ${err?.message || err}`)
|
||||
}
|
||||
}
|
||||
},
|
||||
reloadWindowMs: RENDERER_RELOAD_WINDOW_MS,
|
||||
reloadMax: RENDERER_RELOAD_MAX,
|
||||
recentReloadTimesRef: rendererReloadTimesRef
|
||||
})
|
||||
|
||||
// Electron always passes the event first. The canonical (Electron 36+) shape
|
||||
// is (event, messageDetails); the deprecated positional shape is
|
||||
|
||||
@@ -9,11 +9,13 @@ import {
|
||||
type WakeIndicatorState,
|
||||
wakeIndicatorWindowBounds
|
||||
} from './wake-indicator'
|
||||
import { installWindowRendererLifecycle } from './window-renderer-lifecycle'
|
||||
|
||||
interface WakeIndicatorWindowOptions {
|
||||
devServer?: string
|
||||
isMac: boolean
|
||||
loadWindowUrl: (window: BrowserWindow, url: string, label: string) => void
|
||||
log: (message: string) => void
|
||||
preloadPath: string
|
||||
rendererIndex: () => string
|
||||
wireWindow: (window: BrowserWindow) => void
|
||||
@@ -23,6 +25,7 @@ export function createWakeIndicatorWindowController({
|
||||
devServer,
|
||||
isMac,
|
||||
loadWindowUrl,
|
||||
log,
|
||||
preloadPath,
|
||||
rendererIndex,
|
||||
wireWindow
|
||||
@@ -100,6 +103,10 @@ export function createWakeIndicatorWindowController({
|
||||
|
||||
wireWindow(next)
|
||||
|
||||
// Log-only renderer lifecycle (#81290): the wake cue is ambient and
|
||||
// macOS-only; its loss belongs in desktop.log, never resurrected.
|
||||
installWindowRendererLifecycle(next, { kind: 'wake', callbacks: { log } })
|
||||
|
||||
next.webContents.on('did-finish-load', sendState)
|
||||
next.once('ready-to-show', () => {
|
||||
if (!next.isDestroyed() && state !== 'hidden') {
|
||||
|
||||
@@ -124,6 +124,7 @@ describe('wake indicator window controller', () => {
|
||||
const controller = createWakeIndicatorWindowController({
|
||||
isMac: true,
|
||||
loadWindowUrl: vi.fn(),
|
||||
log: () => {},
|
||||
preloadPath: '/tmp/preload.cjs',
|
||||
rendererIndex: () => '/tmp/index.html',
|
||||
wireWindow: vi.fn()
|
||||
|
||||
391
apps/desktop/electron/window-renderer-lifecycle.test.ts
Normal file
391
apps/desktop/electron/window-renderer-lifecycle.test.ts
Normal file
@@ -0,0 +1,391 @@
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import {
|
||||
consoleMessageLog,
|
||||
describeRendererLifecycleEvent,
|
||||
installWindowRendererLifecycle,
|
||||
pruneReloadTimes,
|
||||
pushReloadTime,
|
||||
shouldReloadAfterRendererGone
|
||||
} from './window-renderer-lifecycle'
|
||||
|
||||
// Fake Electron surface — real listener wiring, no Electron import. Mirrors
|
||||
// how the rest of electron/*.test.ts exercises Electron-free modules.
|
||||
function makeFakeWindow(overrides: { destroyed?: boolean } = {}) {
|
||||
const listeners = new Map<string, ((...args: any[]) => void)[]>()
|
||||
const reloadCalls: number[] = []
|
||||
let destroyed = overrides.destroyed ?? false
|
||||
|
||||
const win = {
|
||||
isDestroyed: () => destroyed,
|
||||
setDestroyed: (value: boolean) => {
|
||||
destroyed = value
|
||||
},
|
||||
webContents: {
|
||||
on: (event: string, listener: (...args: any[]) => void) => {
|
||||
const list = listeners.get(event) ?? []
|
||||
|
||||
list.push(listener)
|
||||
listeners.set(event, list)
|
||||
},
|
||||
removeListener: (event: string, listener: (...args: any[]) => void) => {
|
||||
const list = listeners.get(event) ?? []
|
||||
|
||||
listeners.set(
|
||||
event,
|
||||
list.filter(candidate => candidate !== listener)
|
||||
)
|
||||
},
|
||||
emit: (event: string, ...args: unknown[]) => {
|
||||
for (const listener of listeners.get(event) ?? []) {
|
||||
listener(...args)
|
||||
}
|
||||
},
|
||||
reload: () => {
|
||||
reloadCalls.push(1)
|
||||
},
|
||||
listenerCount: (event: string) => (listeners.get(event) ?? []).length
|
||||
},
|
||||
reloadCalls
|
||||
}
|
||||
|
||||
return win
|
||||
}
|
||||
|
||||
function makeOptions(win: ReturnType<typeof makeFakeWindow>, kind = 'secondary', extra: Record<string, unknown> = {}) {
|
||||
const logs: string[] = []
|
||||
|
||||
const options = {
|
||||
kind,
|
||||
callbacks: {
|
||||
log: (message: string) => {
|
||||
logs.push(message)
|
||||
},
|
||||
reload: () => {
|
||||
win.webContents.reload()
|
||||
}
|
||||
},
|
||||
...extra
|
||||
}
|
||||
|
||||
return { logs, options }
|
||||
}
|
||||
|
||||
// The reload fires on setImmediate (never from inside the event handler), so
|
||||
// tests flush the deferred queue before asserting.
|
||||
function flushDeferred(): Promise<void> {
|
||||
return new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
|
||||
test('pruneReloadTimes drops timestamps outside the rolling window', () => {
|
||||
const now = 100_000
|
||||
|
||||
assert.deepEqual(pruneReloadTimes([100_000, 90_000, 39_999], now, 60_000), [100_000, 90_000])
|
||||
assert.deepEqual(pruneReloadTimes([], now, 60_000), [])
|
||||
})
|
||||
|
||||
test('pushReloadTime records the timestamp', () => {
|
||||
const times: number[] = []
|
||||
|
||||
assert.deepEqual(pushReloadTime(times, 42), [42])
|
||||
})
|
||||
|
||||
test('shouldReloadAfterRendererGone reloads crashed/oom on a live window', () => {
|
||||
assert.deepEqual(
|
||||
shouldReloadAfterRendererGone({ reason: 'crashed', isDestroyed: false, recentReloadTimes: [] }),
|
||||
{ reload: true }
|
||||
)
|
||||
assert.deepEqual(
|
||||
shouldReloadAfterRendererGone({ reason: 'oom', isDestroyed: false, recentReloadTimes: [] }),
|
||||
{ reload: true }
|
||||
)
|
||||
})
|
||||
|
||||
test('shouldReloadAfterRendererGone never reloads expected teardown or unknown reasons', () => {
|
||||
// A window the user closed reports reason 'killed' — reloading would pop it
|
||||
// back up after close.
|
||||
assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'killed', isDestroyed: true, recentReloadTimes: [] }), {
|
||||
reload: false,
|
||||
suppressedReason: 'expected-teardown'
|
||||
})
|
||||
// Killed on a live window is a process-initiated loss (e.g. OS reclaim);
|
||||
// the primary window never reloaded it, so peers don't either.
|
||||
assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'killed', isDestroyed: false, recentReloadTimes: [] }), {
|
||||
reload: false,
|
||||
suppressedReason: 'unrecoverable-reason'
|
||||
})
|
||||
assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'launch-failed', isDestroyed: false, recentReloadTimes: [] }), {
|
||||
reload: false,
|
||||
suppressedReason: 'unrecoverable-reason'
|
||||
})
|
||||
assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'unknown-reason', isDestroyed: false, recentReloadTimes: [] }), {
|
||||
reload: false,
|
||||
suppressedReason: 'unrecoverable-reason'
|
||||
})
|
||||
assert.deepEqual(shouldReloadAfterRendererGone({ reason: undefined, isDestroyed: false, recentReloadTimes: [] }), {
|
||||
reload: false,
|
||||
suppressedReason: 'unrecoverable-reason'
|
||||
})
|
||||
})
|
||||
|
||||
test('shouldReloadAfterRendererGone suppresses past the shared crash-loop budget', () => {
|
||||
const recentReloadTimes = [100, 50, 10]
|
||||
|
||||
assert.deepEqual(
|
||||
shouldReloadAfterRendererGone({
|
||||
reason: 'crashed',
|
||||
isDestroyed: false,
|
||||
recentReloadTimes,
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 3,
|
||||
now: () => 200
|
||||
}),
|
||||
{ reload: false, suppressedReason: 'crash-loop' }
|
||||
)
|
||||
|
||||
// An expired budget entry frees a reload slot.
|
||||
const stale = [100, 50, 10]
|
||||
|
||||
assert.deepEqual(
|
||||
shouldReloadAfterRendererGone({
|
||||
reason: 'crashed',
|
||||
isDestroyed: false,
|
||||
recentReloadTimes: stale,
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 3,
|
||||
now: () => 100_000
|
||||
}),
|
||||
{ reload: true }
|
||||
)
|
||||
})
|
||||
|
||||
test('installWindowRendererLifecycle logs and reloads a crashed secondary window', async () => {
|
||||
const win = makeFakeWindow()
|
||||
|
||||
const { logs, options } = makeOptions(win, 'secondary', {
|
||||
callbacks: {
|
||||
log: (message: string) => {
|
||||
logs.push(message)
|
||||
},
|
||||
reload: () => {
|
||||
win.webContents.reload()
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
|
||||
assert.equal(win.reloadCalls.length, 1)
|
||||
assert.match(logs[0], /\[renderer:secondary\] render-process-gone reason=crashed exitCode=3/)
|
||||
})
|
||||
|
||||
test('installWindowRendererLifecycle logs expected teardown without reloading', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'secondary')
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
win.setDestroyed(true)
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'killed', exitCode: 1 })
|
||||
|
||||
assert.equal(win.reloadCalls.length, 0)
|
||||
assert.match(logs[0], /render-process-gone reason=killed exitCode=1 \(expected teardown\)/)
|
||||
})
|
||||
|
||||
test('installWindowRendererLifecycle suppresses a peer crash loop after the budget', async () => {
|
||||
const win = makeFakeWindow()
|
||||
|
||||
const { logs, options } = makeOptions(win, 'instance', {
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 3,
|
||||
now: () => 1000
|
||||
})
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
}
|
||||
|
||||
await flushDeferred()
|
||||
assert.equal(win.reloadCalls.length, 3)
|
||||
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
|
||||
assert.equal(win.reloadCalls.length, 3)
|
||||
assert.match(logs[logs.length - 1], /suppressing reload: 3 crashes within 60000ms/)
|
||||
})
|
||||
|
||||
test('windows share one crash-loop budget via recentReloadTimesRef', async () => {
|
||||
const shared = { current: [] as number[] }
|
||||
const main = makeFakeWindow()
|
||||
const secondary = makeFakeWindow()
|
||||
|
||||
const { logs: mainLogs, options: mainOptions } = makeOptions(main, 'main', {
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 3,
|
||||
now: () => 1000,
|
||||
recentReloadTimesRef: shared
|
||||
})
|
||||
|
||||
const { logs: secondaryLogs, options: secondaryOptions } = makeOptions(secondary, 'secondary', {
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 3,
|
||||
now: () => 1000,
|
||||
recentReloadTimesRef: shared
|
||||
})
|
||||
|
||||
installWindowRendererLifecycle(main, mainOptions)
|
||||
installWindowRendererLifecycle(secondary, secondaryOptions)
|
||||
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
main.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
}
|
||||
|
||||
// The secondary window's crash spends the last budget slot.
|
||||
secondary.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
assert.equal(secondary.reloadCalls.length, 1)
|
||||
|
||||
// A fourth crash anywhere is suppressed.
|
||||
main.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
assert.equal(main.reloadCalls.length, 2)
|
||||
assert.match(mainLogs[mainLogs.length - 1], /suppressing reload/)
|
||||
assert.equal(secondaryLogs.length, 1)
|
||||
})
|
||||
|
||||
test('log-only mode never reloads', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'overlay')
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
|
||||
assert.equal(win.reloadCalls.length, 0)
|
||||
assert.match(logs[0], /\[renderer:overlay\] render-process-gone reason=crashed exitCode=3/)
|
||||
})
|
||||
|
||||
test('unresponsive is logged, never reloaded', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'secondary')
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
win.webContents.emit('unresponsive')
|
||||
|
||||
assert.equal(win.reloadCalls.length, 0)
|
||||
assert.equal(logs[0], '[renderer:secondary] webContents became unresponsive')
|
||||
})
|
||||
|
||||
test('did-fail-load on the main frame is logged, not reloaded', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'instance')
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
win.webContents.emit('did-fail-load', {}, -3, 'ERR_ABORTED', 'file:///index.html', true)
|
||||
|
||||
assert.equal(win.reloadCalls.length, 0)
|
||||
assert.match(logs[0], /\[renderer:instance\] did-fail-load code=-3 url=file:\/\/\/index\.html/)
|
||||
|
||||
// Sub-frame failures are noise; the primary window never logged them.
|
||||
win.webContents.emit('did-fail-load', {}, -3, 'ERR_ABORTED', 'https://example.com/asset.js', false)
|
||||
assert.equal(logs.length, 1)
|
||||
})
|
||||
|
||||
test('console-message at error level is logged, lower levels ignored', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'secondary')
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
|
||||
// Modern shape: (event, messageDetails).
|
||||
win.webContents.emit('console-message', {}, { level: 3, message: 'boom', sourceUrl: 'file:///app.js', lineNumber: 42 })
|
||||
// Deprecated positional shape: (event, level, message, line, sourceId).
|
||||
win.webContents.emit('console-message', {}, 3, 'legacy boom', 7, 'file:///legacy.js')
|
||||
win.webContents.emit('console-message', {}, { level: 1, message: 'info', sourceUrl: 'file:///app.js', lineNumber: 1 })
|
||||
|
||||
assert.equal(logs.length, 2)
|
||||
assert.match(logs[0], /\[renderer:secondary console\] boom \(file:\/\/\/app\.js:42\)/)
|
||||
assert.match(logs[1], /\[renderer:secondary console\] legacy boom \(file:\/\/\/legacy\.js:7\)/)
|
||||
})
|
||||
|
||||
test('onCrashLoopSuppressed fires when the budget trips (main sandbox-relaunch hook)', async () => {
|
||||
const win = makeFakeWindow()
|
||||
const suppressed: Array<{ reason?: string; exitCode?: number }> = []
|
||||
|
||||
const { logs, options } = makeOptions(win, 'main', {
|
||||
reloadWindowMs: 60_000,
|
||||
reloadMax: 1,
|
||||
now: () => 1000,
|
||||
callbacks: {
|
||||
log: (message: string) => {
|
||||
logs.push(message)
|
||||
},
|
||||
reload: () => {
|
||||
win.webContents.reload()
|
||||
},
|
||||
onCrashLoopSuppressed: details => {
|
||||
suppressed.push({ reason: details?.reason, exitCode: details?.exitCode })
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
installWindowRendererLifecycle(win, options)
|
||||
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
assert.equal(win.reloadCalls.length, 1)
|
||||
assert.equal(suppressed.length, 0)
|
||||
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
await flushDeferred()
|
||||
|
||||
assert.equal(win.reloadCalls.length, 1)
|
||||
assert.equal(suppressed.length, 1)
|
||||
assert.deepEqual(suppressed[0], { reason: 'crashed', exitCode: 3 })
|
||||
assert.match(logs[logs.length - 1], /suppressing reload/)
|
||||
|
||||
// Expected teardown and non-recoverable reasons never trip the hook.
|
||||
win.setDestroyed(true)
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'killed', exitCode: 1 })
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'launch-failed', exitCode: 7 })
|
||||
assert.equal(suppressed.length, 1)
|
||||
})
|
||||
|
||||
test('dispose removes every listener (no stacking on window recreation)', () => {
|
||||
const win = makeFakeWindow()
|
||||
const { logs, options } = makeOptions(win, 'secondary')
|
||||
|
||||
const dispose = installWindowRendererLifecycle(win, options)
|
||||
const before = win.webContents.listenerCount('render-process-gone')
|
||||
|
||||
dispose()
|
||||
win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 })
|
||||
|
||||
assert.equal(win.reloadCalls.length, 0)
|
||||
assert.equal(logs.length, 0)
|
||||
assert.equal(win.webContents.listenerCount('render-process-gone'), before - 1)
|
||||
})
|
||||
|
||||
test('consoleMessageLog parses both Electron argument shapes', () => {
|
||||
const modern = consoleMessageLog([{}, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 1 }])
|
||||
|
||||
assert.deepEqual(modern, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 1 })
|
||||
|
||||
const legacy = consoleMessageLog([{}, 3, 'm', 7, 's'])
|
||||
|
||||
assert.deepEqual(legacy, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 7 })
|
||||
})
|
||||
|
||||
test('describeRendererLifecycleEvent sanitizes unknown fields', () => {
|
||||
assert.equal(describeRendererLifecycleEvent({ kind: 'secondary', event: 'render-process-gone' }), '[renderer:secondary] render-process-gone reason=? exitCode=?')
|
||||
assert.equal(describeRendererLifecycleEvent({ kind: 'secondary', event: 'render-process-gone', reason: 'crashed', exitCode: undefined }), '[renderer:secondary] render-process-gone reason=crashed exitCode=?')
|
||||
assert.equal(
|
||||
describeRendererLifecycleEvent({ kind: 'main', event: 'render-process-gone', reason: 'killed', exitCode: 1, isDestroyed: true }),
|
||||
'[renderer:main] render-process-gone reason=killed exitCode=1 (expected teardown)'
|
||||
)
|
||||
})
|
||||
344
apps/desktop/electron/window-renderer-lifecycle.ts
Normal file
344
apps/desktop/electron/window-renderer-lifecycle.ts
Normal file
@@ -0,0 +1,344 @@
|
||||
// Per-window renderer lifecycle diagnostics + crash recovery (#81290).
|
||||
//
|
||||
// The desktop app renders one Chromium renderer per window (primary, secondary
|
||||
// session windows, full instance windows, and the small helper overlays), but
|
||||
// renderer-lifecycle listeners used to be attached ONLY to the primary window:
|
||||
// a dead peer renderer produced no log line and no recovery, leaving the user
|
||||
// with a permanently black window and nothing in desktop.log. This module
|
||||
// attaches the same lifecycle wiring to every window, keyed by a `kind` label,
|
||||
// with an injected reload policy so the pure decision logic stays Electron-free
|
||||
// and unit-testable (mirroring windows-sandbox-fallback.ts / session-windows.ts).
|
||||
//
|
||||
// Policy (matches the primary window's previous behavior, generalized):
|
||||
// - `render-process-gone` with reason `crashed`/`oom` → bounded reload (rolling
|
||||
// crash-loop guard shared across ALL windows — one budget per process).
|
||||
// - `render-process-gone` with any other reason (`killed`, `launch-failed`,
|
||||
// `clean-exit`, unknown) → log only. `killed` after an expected close/destroy
|
||||
// is normal teardown, and blindly reloading it would loop windows back up
|
||||
// after the user closed them.
|
||||
// - `unresponsive` → log only (no reload; Chromium usually follows with
|
||||
// render-process-gone, and forcing a reload while the main thread is wedged
|
||||
// can make things worse).
|
||||
// - `did-fail-load` on the MAIN frame → log only (no blind reload: a repeatable
|
||||
// startup failure would boot-loop; the backend startup path already surfaces
|
||||
// the actionable error).
|
||||
// - `console-message` at error level → log renderer console errors, same as the
|
||||
// primary window did.
|
||||
|
||||
export interface RendererLifecycleDetails {
|
||||
reason?: string
|
||||
exitCode?: number | string | undefined
|
||||
isDestroyed?: boolean
|
||||
}
|
||||
|
||||
export interface RendererLifecycleEvent {
|
||||
kind: string
|
||||
event: 'render-process-gone' | 'unresponsive' | 'did-fail-load' | 'console-message'
|
||||
reason?: string
|
||||
exitCode?: number | string | undefined
|
||||
isDestroyed?: boolean
|
||||
/** did-fail-load: only main-frame failures are meaningful (issue point 4). */
|
||||
isMainFrame?: boolean
|
||||
/** did-fail-load: the Chromium error code (e.g. -3 = ERR_ABORTED). */
|
||||
errorCode?: number | string | undefined
|
||||
/** did-fail-load: the URL that failed. */
|
||||
url?: string
|
||||
/** console-message: renderer error text. */
|
||||
message?: string
|
||||
/** console-message: source URL of the console message. */
|
||||
sourceUrl?: string
|
||||
/** console-message: line number of the console message. */
|
||||
lineNumber?: number | string | undefined
|
||||
}
|
||||
|
||||
export interface ReloadPolicyDecision {
|
||||
reload: boolean
|
||||
/** Why reload was refused, when it was. */
|
||||
suppressedReason?: 'crash-loop' | 'expected-teardown' | 'unrecoverable-reason'
|
||||
}
|
||||
|
||||
export interface WindowRendererLifecycleOptions {
|
||||
/** Stable label used in log lines: 'main' | 'secondary' | 'instance' |
|
||||
* 'overlay' | 'quick' | 'wake'. */
|
||||
kind: string
|
||||
callbacks: {
|
||||
log: (message: string) => void
|
||||
/** Omitted → log-only mode (helper windows never reload). */
|
||||
reload?: () => void
|
||||
/** Called when the shared crash-loop budget suppresses a reload — the
|
||||
* primary window uses it for the #38216 Windows sandbox relaunch check. */
|
||||
onCrashLoopSuppressed?: (details?: RendererLifecycleDetails) => void
|
||||
}
|
||||
/** Rolling crash-loop window, ms. Defaults to 60_000 (RENDERER_RELOAD_WINDOW_MS). */
|
||||
reloadWindowMs?: number
|
||||
/** Max reloads per rolling window. Defaults to 3 (RENDERER_RELOAD_MAX). */
|
||||
reloadMax?: number
|
||||
/** Shared per-process reload budget. Omitted → per-window budget (tests). */
|
||||
recentReloadTimesRef?: { current: number[] }
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
/** Minimal structural surface of BrowserWindow / webContents used here. */
|
||||
export interface LifecycleWindowLike {
|
||||
isDestroyed: () => boolean
|
||||
webContents: {
|
||||
on: (event: string, listener: (...args: any[]) => void) => unknown
|
||||
reload?: () => void
|
||||
removeListener?: (event: string, listener: (...args: any[]) => void) => unknown
|
||||
}
|
||||
}
|
||||
|
||||
const DEFAULT_RELOAD_WINDOW_MS = 60_000
|
||||
const DEFAULT_RELOAD_MAX = 3
|
||||
|
||||
const RECOVERABLE_REASONS = new Set(['crashed', 'oom'])
|
||||
|
||||
function safeNow(now: (() => number) | undefined): number {
|
||||
return typeof now === 'function' ? now() : Date.now()
|
||||
}
|
||||
|
||||
function isWithin(timestamp: number, now: number, windowMs: number): boolean {
|
||||
return now - timestamp < windowMs
|
||||
}
|
||||
|
||||
/** Drop reload timestamps outside the rolling window. Mutates + returns. */
|
||||
export function pruneReloadTimes(times: number[], now: number, windowMs: number): number[] {
|
||||
return times.filter(timestamp => isWithin(timestamp, now, windowMs))
|
||||
}
|
||||
|
||||
/** Record a reload attempt timestamp. Mutates + returns. */
|
||||
export function pushReloadTime(times: number[], now: number): number[] {
|
||||
times.push(now)
|
||||
|
||||
return times
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a render-process-gone event should reload its window.
|
||||
*
|
||||
* Reload only for `crashed`/`oom` on a live window, bounded by the shared
|
||||
* rolling crash-loop budget. Anything else — expected teardown (`killed` after
|
||||
* close/destroy), unrecoverable reasons, unknown reasons — is log-only, exactly
|
||||
* like the primary window's previous behavior but now per window kind.
|
||||
*/
|
||||
export function shouldReloadAfterRendererGone(details: {
|
||||
reason?: string
|
||||
isDestroyed?: boolean
|
||||
recentReloadTimes: number[]
|
||||
reloadWindowMs?: number
|
||||
reloadMax?: number
|
||||
now?: () => number
|
||||
}): ReloadPolicyDecision {
|
||||
if (details.isDestroyed) {
|
||||
return { reload: false, suppressedReason: 'expected-teardown' }
|
||||
}
|
||||
|
||||
const reason = String(details.reason || '')
|
||||
|
||||
if (!RECOVERABLE_REASONS.has(reason)) {
|
||||
return { reload: false, suppressedReason: 'unrecoverable-reason' }
|
||||
}
|
||||
|
||||
const windowMs = details.reloadWindowMs ?? DEFAULT_RELOAD_WINDOW_MS
|
||||
const max = details.reloadMax ?? DEFAULT_RELOAD_MAX
|
||||
const now = safeNow(details.now)
|
||||
const recent = pruneReloadTimes(details.recentReloadTimes, now, windowMs)
|
||||
|
||||
if (recent.length >= max) {
|
||||
return { reload: false, suppressedReason: 'crash-loop' }
|
||||
}
|
||||
|
||||
return { reload: true }
|
||||
}
|
||||
|
||||
/**
|
||||
* One log line per renderer lifecycle event, e.g.
|
||||
* [renderer:secondary] render-process-gone reason=crashed exitCode=3
|
||||
* Sanitizes unknown fields and annotates expected teardown so a support bundle
|
||||
* reads as a story, not a pile of question marks.
|
||||
*/
|
||||
export function describeRendererLifecycleEvent(event: RendererLifecycleEvent): string {
|
||||
const kind = String(event.kind || '?')
|
||||
|
||||
if (event.event === 'console-message') {
|
||||
const src = String(event.sourceUrl || '?')
|
||||
const line = event.lineNumber === undefined ? '?' : String(event.lineNumber)
|
||||
|
||||
return `[renderer:${kind} console] ${String(event.message || '(empty)')} (${src}:${line})`
|
||||
}
|
||||
|
||||
if (event.event === 'unresponsive') {
|
||||
return `[renderer:${kind}] webContents became unresponsive`
|
||||
}
|
||||
|
||||
if (event.event === 'did-fail-load') {
|
||||
const code = event.errorCode === undefined ? '?' : String(event.errorCode)
|
||||
const url = String(event.url || '?')
|
||||
|
||||
return `[renderer:${kind}] did-fail-load code=${code} url=${url}`
|
||||
}
|
||||
|
||||
const reason = String(event.reason || '?')
|
||||
const exitCode = event.exitCode === undefined ? '?' : String(event.exitCode)
|
||||
const teardown = event.isDestroyed && reason === 'killed' ? ' (expected teardown)' : ''
|
||||
|
||||
return `[renderer:${kind}] render-process-gone reason=${reason} exitCode=${exitCode}${teardown}`
|
||||
}
|
||||
|
||||
// Electron ≥36 passes (event, messageDetails); older/deprecated shape is
|
||||
// (event, level, message, line, sourceId). Handle both, matching the primary
|
||||
// window's previous console-message handling.
|
||||
export function consoleMessageLog(args: readonly unknown[]): {
|
||||
level: number
|
||||
message: string
|
||||
sourceUrl: string
|
||||
lineNumber: number | string | undefined
|
||||
} {
|
||||
const details = args[1] && typeof args[1] === 'object' ? (args[1] as Record<string, unknown>) : null
|
||||
const level = details ? Number(details.level ?? 0) : Number(args[1] ?? 0)
|
||||
const modernLine = details?.lineNumber
|
||||
|
||||
return {
|
||||
level,
|
||||
message: details ? String(details.message ?? '') : String(args[2] ?? ''),
|
||||
sourceUrl: details ? String(details.sourceUrl ?? '') : String(args[4] ?? ''),
|
||||
lineNumber: details
|
||||
? typeof modernLine === 'number'
|
||||
? modernLine
|
||||
: modernLine === undefined
|
||||
? undefined
|
||||
: String(modernLine)
|
||||
: typeof args[3] === 'number'
|
||||
? args[3]
|
||||
: String(args[3] ?? '')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach renderer lifecycle listeners to a window. Returns a dispose() that
|
||||
* removes every listener (window recreation must not stack handlers).
|
||||
*
|
||||
* `reload` is never invoked synchronously inside the event handler: Electron
|
||||
* warns about re-entrant webContents calls, and the primary window's previous
|
||||
* implementation deferred via setImmediate for the same reason.
|
||||
*/
|
||||
export function installWindowRendererLifecycle(
|
||||
win: LifecycleWindowLike,
|
||||
options: WindowRendererLifecycleOptions
|
||||
): () => void {
|
||||
const kind = options.kind
|
||||
const { log, reload, onCrashLoopSuppressed } = options.callbacks
|
||||
const reloadWindowMs = options.reloadWindowMs ?? DEFAULT_RELOAD_WINDOW_MS
|
||||
const reloadMax = options.reloadMax ?? DEFAULT_RELOAD_MAX
|
||||
const now = options.now
|
||||
const budgetRef = options.recentReloadTimesRef ?? { current: [] }
|
||||
const contents = win.webContents
|
||||
|
||||
const onRendererGone = (_event: unknown, details?: RendererLifecycleDetails) => {
|
||||
const destroyed = win.isDestroyed()
|
||||
|
||||
log(describeRendererLifecycleEvent({ kind, event: 'render-process-gone', ...details, isDestroyed: destroyed }))
|
||||
|
||||
const nowMs = safeNow(now)
|
||||
const recent = pruneReloadTimes(budgetRef.current, nowMs, reloadWindowMs)
|
||||
|
||||
budgetRef.current.length = 0
|
||||
budgetRef.current.push(...recent)
|
||||
|
||||
const decision = shouldReloadAfterRendererGone({
|
||||
reason: details?.reason,
|
||||
isDestroyed: destroyed,
|
||||
recentReloadTimes: budgetRef.current,
|
||||
reloadWindowMs,
|
||||
reloadMax,
|
||||
now: () => nowMs
|
||||
})
|
||||
|
||||
if (!decision.reload) {
|
||||
if (decision.suppressedReason === 'crash-loop') {
|
||||
log(
|
||||
`[renderer:${kind}] suppressing reload: ${budgetRef.current.length} crashes within ${reloadWindowMs}ms (likely a crash loop)`
|
||||
)
|
||||
onCrashLoopSuppressed?.(details)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if (typeof reload !== 'function') {
|
||||
return
|
||||
}
|
||||
|
||||
pushReloadTime(budgetRef.current, nowMs)
|
||||
|
||||
// Deferred: never reload from inside the event handler (see above).
|
||||
setImmediate(() => {
|
||||
if (win.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
reload()
|
||||
} catch (error) {
|
||||
log(`[renderer:${kind}] reload after crash failed: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const onUnresponsive = () => {
|
||||
log(describeRendererLifecycleEvent({ kind, event: 'unresponsive' }))
|
||||
}
|
||||
|
||||
const onDidFailLoad = (_event: unknown, errorCode: unknown, _errorDescription: unknown, validatedURL: unknown, isMainFrame?: unknown) => {
|
||||
if (isMainFrame === true) {
|
||||
log(
|
||||
describeRendererLifecycleEvent({
|
||||
kind,
|
||||
event: 'did-fail-load',
|
||||
errorCode: typeof errorCode === 'number' ? errorCode : String(errorCode ?? ''),
|
||||
url: String(validatedURL ?? '')
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const onConsoleMessage = (...args: unknown[]) => {
|
||||
const parsed = consoleMessageLog(args)
|
||||
|
||||
if (parsed.level !== 3) {
|
||||
return
|
||||
}
|
||||
|
||||
log(
|
||||
describeRendererLifecycleEvent({
|
||||
kind,
|
||||
event: 'console-message',
|
||||
message: parsed.message,
|
||||
sourceUrl: parsed.sourceUrl,
|
||||
lineNumber: parsed.lineNumber
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
contents.on('render-process-gone', onRendererGone)
|
||||
contents.on('unresponsive', onUnresponsive)
|
||||
contents.on('did-fail-load', onDidFailLoad)
|
||||
contents.on('console-message', onConsoleMessage)
|
||||
|
||||
let disposed = false
|
||||
|
||||
return () => {
|
||||
if (disposed) {
|
||||
return
|
||||
}
|
||||
|
||||
disposed = true
|
||||
|
||||
if (typeof contents.removeListener === 'function') {
|
||||
contents.removeListener('render-process-gone', onRendererGone)
|
||||
contents.removeListener('unresponsive', onUnresponsive)
|
||||
contents.removeListener('did-fail-load', onDidFailLoad)
|
||||
contents.removeListener('console-message', onConsoleMessage)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user