diff --git a/.gitignore b/.gitignore index 283c8a9164..5bbe562f4c 100644 --- a/.gitignore +++ b/.gitignore @@ -197,3 +197,6 @@ native/fts5_cjk/*.so # interrupted; consumed by launch-time recovery. Never commit it (was tracked # by accident via 3a69e34702, removed in the #72002 salvage). .lazy-refresh-incomplete + +# OMC working artifacts +.omc/ diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index daa3bf8fd0..41ce18158f 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -220,6 +220,7 @@ import { fetchMarketplaceThemes, searchMarketplaceThemes } from './vscode-market import { createWakeIndicatorWindowController } from './wake-indicator-window' import { readWindowBelow } from './window-below' import { createWindowRevealController } from './window-reveal' +import { installWindowRendererLifecycle } from './window-renderer-lifecycle' import { bindGeometryPersistence, computeWindowOptions, @@ -1087,13 +1088,15 @@ const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDL // killing one to honor the soft cap would abort a running agent. const POOL_KEEPALIVE_FRESH_MS = 90_000 let poolIdleReaper = null -// Auto-reload budget for renderer crashes. A deterministic startup crash would -// otherwise loop forever (reload → crash → reload), pinning CPU and spamming -// logs. Allow a few reloads per rolling window, then stop and leave the dead -// window so the user can read the error / quit. +// Auto-reload budget for renderer crashes, shared by EVERY window (primary, +// secondary session, instance) so a crash loop anywhere is suppressed after +// the same budget instead of reloading per-window forever. A deterministic +// startup crash would otherwise loop forever (reload → crash → reload), +// pinning CPU and spamming logs. Allow a few reloads per rolling window, then +// stop and leave the dead window so the user can read the error / quit. const RENDERER_RELOAD_WINDOW_MS = 60_000 const RENDERER_RELOAD_MAX = 3 -let rendererReloadTimes = [] +const rendererReloadTimesRef: { current: number[] } = { current: [] } // Latched bootstrap failure: when the first-launch install fails, we hold // onto the error so subsequent startHermes() calls (e.g. the renderer's // ensureGatewayOpen retrying after the WS won't open) return the same error @@ -8919,6 +8922,23 @@ function spawnSecondaryWindow({ sessionId, watch }: { sessionId?: string; watch? wireCommonWindowHandlers(win, zoomWiringForWindowKind('chat')) attachRendererConsoleCapture(win, 'session-window', rememberLog) + // Renderer lifecycle diagnostics + recovery (#81290): a dead session-window + // renderer used to log nothing and stay black; now it logs with its window + // kind and reloads under the shared crash-loop budget, exactly like the + // primary window, without touching any other window. + installWindowRendererLifecycle(win, { + kind: 'secondary', + callbacks: { + log: rememberLog, + reload: () => { + win.webContents.reload() + } + }, + reloadWindowMs: RENDERER_RELOAD_WINDOW_MS, + reloadMax: RENDERER_RELOAD_MAX, + recentReloadTimesRef: rendererReloadTimesRef + }) + loadWindowUrl( win, buildSessionWindowUrl(sessionId, { @@ -8999,6 +9019,22 @@ function createInstanceWindow() { streamThrottle.register(win) wireCommonWindowHandlers(win, zoomWiringForWindowKind('chat')) + // Renderer lifecycle diagnostics + recovery (#81290), same policy as the + // primary and session windows: a crashed instance renderer logs with its + // window kind and reloads under the shared crash-loop budget. + installWindowRendererLifecycle(win, { + kind: 'instance', + callbacks: { + log: rememberLog, + reload: () => { + win.webContents.reload() + } + }, + reloadWindowMs: RENDERER_RELOAD_WINDOW_MS, + reloadMax: RENDERER_RELOAD_MAX, + recentReloadTimesRef: rendererReloadTimesRef + }) + win.on('closed', () => { instanceWindows.delete(win) }) @@ -9015,6 +9051,7 @@ const wakeIndicatorController = createWakeIndicatorWindowController({ devServer: DEV_SERVER, isMac: IS_MAC, loadWindowUrl, + log: rememberLog, preloadPath: PRELOAD_PATH, rendererIndex: resolveRendererIndex, wireWindow: window => wireCommonWindowHandlers(window, zoomWiringForWindowKind('wakeIndicator')) @@ -9110,6 +9147,10 @@ function spawnPetOverlayWindow(bounds) { wireWindowReveal(win, { show: () => win.showInactive() }) + // Log-only renderer lifecycle (#81290): a dead overlay must never resurrect + // itself over the app, but its loss belongs in desktop.log. + installWindowRendererLifecycle(win, { kind: 'overlay', callbacks: { log: rememberLog } }) + win.on('closed', () => { if (petOverlayWindow === win) { petOverlayWindow = null @@ -9670,6 +9711,10 @@ function spawnQuickEntryWindow() { // its own OS window and a zoomed composer would overflow it. wireCommonWindowHandlers(win, zoomWiringForWindowKind('quickEntry')) + // Log-only renderer lifecycle (#81290): a dead quick-entry window must never + // resurrect itself over the app, but its loss belongs in desktop.log. + installWindowRendererLifecycle(win, { kind: 'quick', callbacks: { log: rememberLog } }) + // Hide on blur. The window must never hold the user's focus captive — losing // focus is the cheapest, least surprising dismiss (matches Spotlight). win.on('blur', () => { @@ -9908,71 +9953,60 @@ function createWindow() { streamThrottle.register(mainWindow) wireCommonWindowHandlers(mainWindow, zoomWiringForWindowKind('chat')) - mainWindow.webContents.on('render-process-gone', (_event, details) => { - rememberLog(`[renderer] render-process-gone reason=${details?.reason} exitCode=${details?.exitCode}`) - - if (details?.reason === 'crashed' || details?.reason === 'oom') { - const now = Date.now() - rendererReloadTimes = rendererReloadTimes.filter(t => now - t < RENDERER_RELOAD_WINDOW_MS) - - if (rendererReloadTimes.length >= RENDERER_RELOAD_MAX) { - rememberLog( - `[renderer] suppressing reload: ${rendererReloadTimes.length} crashes within ${RENDERER_RELOAD_WINDOW_MS}ms (likely a crash loop)` - ) - + // Per-window renderer lifecycle diagnostics + recovery (#81290). The reload + // policy (crashed/oom → bounded reload via the shared rolling budget, then + // the #38216 Windows sandbox relaunch check on suppression) is the same + // policy this window used before it moved into the shared helper, so a + // crashed peer renderer now logs and recovers exactly like the primary one. + installWindowRendererLifecycle(mainWindow, { + kind: 'main', + callbacks: { + log: rememberLog, + reload: () => { + mainWindow.webContents.reload() + }, + onCrashLoopSuppressed: details => { // #38216 renderer flavor (same recovery as #56726, credit @Sahil-SS9): // a deterministic Windows renderer crash loop with the sandbox // breakpoint signature gets one --no-sandbox relaunch instead of a // dead window. Gated on the exit code so unrelated crash loops don't // silently drop the sandbox. if ( - shouldRelaunchForRendererSandboxCrashLoop({ + !shouldRelaunchForRendererSandboxCrashLoop({ reason: details?.reason, exitCode: details?.exitCode, alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env), relaunchAttempted: windowsNoSandboxRelaunchAttempted }) ) { - windowsNoSandboxRelaunchAttempted = true - windowsSandboxFallbackActive = true - windowsSandboxFallbackSticky = true - windowsSandboxFallbackReason = 'renderer-crash-loop' - - try { - writeSandboxMarker(app.getPath('userData'), fallbackMarker('renderer-crash-loop', app.getVersion())) - } catch { - void 0 - } - - rememberLog('[renderer] Windows sandbox crash loop detected; relaunching once with --no-sandbox (#38216)') - - try { - app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) - app.exit(0) - } catch (err) { - rememberLog(`[renderer] --no-sandbox relaunch failed: ${err?.message || err}`) - } - } - - return - } - - rendererReloadTimes.push(now) - setImmediate(() => { - if (!mainWindow || mainWindow.isDestroyed()) { return } - try { - mainWindow.webContents.reload() - } catch (err) { - rememberLog(`[renderer] reload after crash failed: ${err?.message || err}`) - } - }) - } - }) + windowsNoSandboxRelaunchAttempted = true + windowsSandboxFallbackActive = true + windowsSandboxFallbackSticky = true + windowsSandboxFallbackReason = 'renderer-crash-loop' - mainWindow.webContents.on('unresponsive', () => rememberLog('[renderer] webContents became unresponsive')) + try { + writeSandboxMarker(app.getPath('userData'), fallbackMarker('renderer-crash-loop', app.getVersion())) + } catch { + void 0 + } + + rememberLog('[renderer] Windows sandbox crash loop detected; relaunching once with --no-sandbox (#38216)') + + try { + app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) + app.exit(0) + } catch (err) { + rememberLog(`[renderer] --no-sandbox relaunch failed: ${err?.message || err}`) + } + } + }, + reloadWindowMs: RENDERER_RELOAD_WINDOW_MS, + reloadMax: RENDERER_RELOAD_MAX, + recentReloadTimesRef: rendererReloadTimesRef + }) // Electron always passes the event first. The canonical (Electron 36+) shape // is (event, messageDetails); the deprecated positional shape is diff --git a/apps/desktop/electron/wake-indicator-window.ts b/apps/desktop/electron/wake-indicator-window.ts index 7ef676a43f..5ad12260ca 100644 --- a/apps/desktop/electron/wake-indicator-window.ts +++ b/apps/desktop/electron/wake-indicator-window.ts @@ -9,11 +9,13 @@ import { type WakeIndicatorState, wakeIndicatorWindowBounds } from './wake-indicator' +import { installWindowRendererLifecycle } from './window-renderer-lifecycle' interface WakeIndicatorWindowOptions { devServer?: string isMac: boolean loadWindowUrl: (window: BrowserWindow, url: string, label: string) => void + log: (message: string) => void preloadPath: string rendererIndex: () => string wireWindow: (window: BrowserWindow) => void @@ -23,6 +25,7 @@ export function createWakeIndicatorWindowController({ devServer, isMac, loadWindowUrl, + log, preloadPath, rendererIndex, wireWindow @@ -100,6 +103,10 @@ export function createWakeIndicatorWindowController({ wireWindow(next) + // Log-only renderer lifecycle (#81290): the wake cue is ambient and + // macOS-only; its loss belongs in desktop.log, never resurrected. + installWindowRendererLifecycle(next, { kind: 'wake', callbacks: { log } }) + next.webContents.on('did-finish-load', sendState) next.once('ready-to-show', () => { if (!next.isDestroyed() && state !== 'hidden') { diff --git a/apps/desktop/electron/wake-indicator.test.ts b/apps/desktop/electron/wake-indicator.test.ts index 433a9ca235..1aa22913ff 100644 --- a/apps/desktop/electron/wake-indicator.test.ts +++ b/apps/desktop/electron/wake-indicator.test.ts @@ -124,6 +124,7 @@ describe('wake indicator window controller', () => { const controller = createWakeIndicatorWindowController({ isMac: true, loadWindowUrl: vi.fn(), + log: () => {}, preloadPath: '/tmp/preload.cjs', rendererIndex: () => '/tmp/index.html', wireWindow: vi.fn() diff --git a/apps/desktop/electron/window-renderer-lifecycle.test.ts b/apps/desktop/electron/window-renderer-lifecycle.test.ts new file mode 100644 index 0000000000..1b5620b69f --- /dev/null +++ b/apps/desktop/electron/window-renderer-lifecycle.test.ts @@ -0,0 +1,391 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { + consoleMessageLog, + describeRendererLifecycleEvent, + installWindowRendererLifecycle, + pruneReloadTimes, + pushReloadTime, + shouldReloadAfterRendererGone +} from './window-renderer-lifecycle' + +// Fake Electron surface — real listener wiring, no Electron import. Mirrors +// how the rest of electron/*.test.ts exercises Electron-free modules. +function makeFakeWindow(overrides: { destroyed?: boolean } = {}) { + const listeners = new Map void)[]>() + const reloadCalls: number[] = [] + let destroyed = overrides.destroyed ?? false + + const win = { + isDestroyed: () => destroyed, + setDestroyed: (value: boolean) => { + destroyed = value + }, + webContents: { + on: (event: string, listener: (...args: any[]) => void) => { + const list = listeners.get(event) ?? [] + + list.push(listener) + listeners.set(event, list) + }, + removeListener: (event: string, listener: (...args: any[]) => void) => { + const list = listeners.get(event) ?? [] + + listeners.set( + event, + list.filter(candidate => candidate !== listener) + ) + }, + emit: (event: string, ...args: unknown[]) => { + for (const listener of listeners.get(event) ?? []) { + listener(...args) + } + }, + reload: () => { + reloadCalls.push(1) + }, + listenerCount: (event: string) => (listeners.get(event) ?? []).length + }, + reloadCalls + } + + return win +} + +function makeOptions(win: ReturnType, kind = 'secondary', extra: Record = {}) { + const logs: string[] = [] + + const options = { + kind, + callbacks: { + log: (message: string) => { + logs.push(message) + }, + reload: () => { + win.webContents.reload() + } + }, + ...extra + } + + return { logs, options } +} + +// The reload fires on setImmediate (never from inside the event handler), so +// tests flush the deferred queue before asserting. +function flushDeferred(): Promise { + return new Promise(resolve => setImmediate(resolve)) +} + +test('pruneReloadTimes drops timestamps outside the rolling window', () => { + const now = 100_000 + + assert.deepEqual(pruneReloadTimes([100_000, 90_000, 39_999], now, 60_000), [100_000, 90_000]) + assert.deepEqual(pruneReloadTimes([], now, 60_000), []) +}) + +test('pushReloadTime records the timestamp', () => { + const times: number[] = [] + + assert.deepEqual(pushReloadTime(times, 42), [42]) +}) + +test('shouldReloadAfterRendererGone reloads crashed/oom on a live window', () => { + assert.deepEqual( + shouldReloadAfterRendererGone({ reason: 'crashed', isDestroyed: false, recentReloadTimes: [] }), + { reload: true } + ) + assert.deepEqual( + shouldReloadAfterRendererGone({ reason: 'oom', isDestroyed: false, recentReloadTimes: [] }), + { reload: true } + ) +}) + +test('shouldReloadAfterRendererGone never reloads expected teardown or unknown reasons', () => { + // A window the user closed reports reason 'killed' — reloading would pop it + // back up after close. + assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'killed', isDestroyed: true, recentReloadTimes: [] }), { + reload: false, + suppressedReason: 'expected-teardown' + }) + // Killed on a live window is a process-initiated loss (e.g. OS reclaim); + // the primary window never reloaded it, so peers don't either. + assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'killed', isDestroyed: false, recentReloadTimes: [] }), { + reload: false, + suppressedReason: 'unrecoverable-reason' + }) + assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'launch-failed', isDestroyed: false, recentReloadTimes: [] }), { + reload: false, + suppressedReason: 'unrecoverable-reason' + }) + assert.deepEqual(shouldReloadAfterRendererGone({ reason: 'unknown-reason', isDestroyed: false, recentReloadTimes: [] }), { + reload: false, + suppressedReason: 'unrecoverable-reason' + }) + assert.deepEqual(shouldReloadAfterRendererGone({ reason: undefined, isDestroyed: false, recentReloadTimes: [] }), { + reload: false, + suppressedReason: 'unrecoverable-reason' + }) +}) + +test('shouldReloadAfterRendererGone suppresses past the shared crash-loop budget', () => { + const recentReloadTimes = [100, 50, 10] + + assert.deepEqual( + shouldReloadAfterRendererGone({ + reason: 'crashed', + isDestroyed: false, + recentReloadTimes, + reloadWindowMs: 60_000, + reloadMax: 3, + now: () => 200 + }), + { reload: false, suppressedReason: 'crash-loop' } + ) + + // An expired budget entry frees a reload slot. + const stale = [100, 50, 10] + + assert.deepEqual( + shouldReloadAfterRendererGone({ + reason: 'crashed', + isDestroyed: false, + recentReloadTimes: stale, + reloadWindowMs: 60_000, + reloadMax: 3, + now: () => 100_000 + }), + { reload: true } + ) +}) + +test('installWindowRendererLifecycle logs and reloads a crashed secondary window', async () => { + const win = makeFakeWindow() + + const { logs, options } = makeOptions(win, 'secondary', { + callbacks: { + log: (message: string) => { + logs.push(message) + }, + reload: () => { + win.webContents.reload() + } + } + }) + + installWindowRendererLifecycle(win, options) + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + + assert.equal(win.reloadCalls.length, 1) + assert.match(logs[0], /\[renderer:secondary\] render-process-gone reason=crashed exitCode=3/) +}) + +test('installWindowRendererLifecycle logs expected teardown without reloading', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'secondary') + + installWindowRendererLifecycle(win, options) + win.setDestroyed(true) + win.webContents.emit('render-process-gone', {}, { reason: 'killed', exitCode: 1 }) + + assert.equal(win.reloadCalls.length, 0) + assert.match(logs[0], /render-process-gone reason=killed exitCode=1 \(expected teardown\)/) +}) + +test('installWindowRendererLifecycle suppresses a peer crash loop after the budget', async () => { + const win = makeFakeWindow() + + const { logs, options } = makeOptions(win, 'instance', { + reloadWindowMs: 60_000, + reloadMax: 3, + now: () => 1000 + }) + + installWindowRendererLifecycle(win, options) + + for (let index = 0; index < 3; index += 1) { + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + } + + await flushDeferred() + assert.equal(win.reloadCalls.length, 3) + + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + + assert.equal(win.reloadCalls.length, 3) + assert.match(logs[logs.length - 1], /suppressing reload: 3 crashes within 60000ms/) +}) + +test('windows share one crash-loop budget via recentReloadTimesRef', async () => { + const shared = { current: [] as number[] } + const main = makeFakeWindow() + const secondary = makeFakeWindow() + + const { logs: mainLogs, options: mainOptions } = makeOptions(main, 'main', { + reloadWindowMs: 60_000, + reloadMax: 3, + now: () => 1000, + recentReloadTimesRef: shared + }) + + const { logs: secondaryLogs, options: secondaryOptions } = makeOptions(secondary, 'secondary', { + reloadWindowMs: 60_000, + reloadMax: 3, + now: () => 1000, + recentReloadTimesRef: shared + }) + + installWindowRendererLifecycle(main, mainOptions) + installWindowRendererLifecycle(secondary, secondaryOptions) + + for (let index = 0; index < 2; index += 1) { + main.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + } + + // The secondary window's crash spends the last budget slot. + secondary.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + assert.equal(secondary.reloadCalls.length, 1) + + // A fourth crash anywhere is suppressed. + main.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + assert.equal(main.reloadCalls.length, 2) + assert.match(mainLogs[mainLogs.length - 1], /suppressing reload/) + assert.equal(secondaryLogs.length, 1) +}) + +test('log-only mode never reloads', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'overlay') + + installWindowRendererLifecycle(win, options) + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + + assert.equal(win.reloadCalls.length, 0) + assert.match(logs[0], /\[renderer:overlay\] render-process-gone reason=crashed exitCode=3/) +}) + +test('unresponsive is logged, never reloaded', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'secondary') + + installWindowRendererLifecycle(win, options) + win.webContents.emit('unresponsive') + + assert.equal(win.reloadCalls.length, 0) + assert.equal(logs[0], '[renderer:secondary] webContents became unresponsive') +}) + +test('did-fail-load on the main frame is logged, not reloaded', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'instance') + + installWindowRendererLifecycle(win, options) + win.webContents.emit('did-fail-load', {}, -3, 'ERR_ABORTED', 'file:///index.html', true) + + assert.equal(win.reloadCalls.length, 0) + assert.match(logs[0], /\[renderer:instance\] did-fail-load code=-3 url=file:\/\/\/index\.html/) + + // Sub-frame failures are noise; the primary window never logged them. + win.webContents.emit('did-fail-load', {}, -3, 'ERR_ABORTED', 'https://example.com/asset.js', false) + assert.equal(logs.length, 1) +}) + +test('console-message at error level is logged, lower levels ignored', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'secondary') + + installWindowRendererLifecycle(win, options) + + // Modern shape: (event, messageDetails). + win.webContents.emit('console-message', {}, { level: 3, message: 'boom', sourceUrl: 'file:///app.js', lineNumber: 42 }) + // Deprecated positional shape: (event, level, message, line, sourceId). + win.webContents.emit('console-message', {}, 3, 'legacy boom', 7, 'file:///legacy.js') + win.webContents.emit('console-message', {}, { level: 1, message: 'info', sourceUrl: 'file:///app.js', lineNumber: 1 }) + + assert.equal(logs.length, 2) + assert.match(logs[0], /\[renderer:secondary console\] boom \(file:\/\/\/app\.js:42\)/) + assert.match(logs[1], /\[renderer:secondary console\] legacy boom \(file:\/\/\/legacy\.js:7\)/) +}) + +test('onCrashLoopSuppressed fires when the budget trips (main sandbox-relaunch hook)', async () => { + const win = makeFakeWindow() + const suppressed: Array<{ reason?: string; exitCode?: number }> = [] + + const { logs, options } = makeOptions(win, 'main', { + reloadWindowMs: 60_000, + reloadMax: 1, + now: () => 1000, + callbacks: { + log: (message: string) => { + logs.push(message) + }, + reload: () => { + win.webContents.reload() + }, + onCrashLoopSuppressed: details => { + suppressed.push({ reason: details?.reason, exitCode: details?.exitCode }) + } + } + }) + + installWindowRendererLifecycle(win, options) + + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + assert.equal(win.reloadCalls.length, 1) + assert.equal(suppressed.length, 0) + + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + await flushDeferred() + + assert.equal(win.reloadCalls.length, 1) + assert.equal(suppressed.length, 1) + assert.deepEqual(suppressed[0], { reason: 'crashed', exitCode: 3 }) + assert.match(logs[logs.length - 1], /suppressing reload/) + + // Expected teardown and non-recoverable reasons never trip the hook. + win.setDestroyed(true) + win.webContents.emit('render-process-gone', {}, { reason: 'killed', exitCode: 1 }) + win.webContents.emit('render-process-gone', {}, { reason: 'launch-failed', exitCode: 7 }) + assert.equal(suppressed.length, 1) +}) + +test('dispose removes every listener (no stacking on window recreation)', () => { + const win = makeFakeWindow() + const { logs, options } = makeOptions(win, 'secondary') + + const dispose = installWindowRendererLifecycle(win, options) + const before = win.webContents.listenerCount('render-process-gone') + + dispose() + win.webContents.emit('render-process-gone', {}, { reason: 'crashed', exitCode: 3 }) + + assert.equal(win.reloadCalls.length, 0) + assert.equal(logs.length, 0) + assert.equal(win.webContents.listenerCount('render-process-gone'), before - 1) +}) + +test('consoleMessageLog parses both Electron argument shapes', () => { + const modern = consoleMessageLog([{}, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 1 }]) + + assert.deepEqual(modern, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 1 }) + + const legacy = consoleMessageLog([{}, 3, 'm', 7, 's']) + + assert.deepEqual(legacy, { level: 3, message: 'm', sourceUrl: 's', lineNumber: 7 }) +}) + +test('describeRendererLifecycleEvent sanitizes unknown fields', () => { + assert.equal(describeRendererLifecycleEvent({ kind: 'secondary', event: 'render-process-gone' }), '[renderer:secondary] render-process-gone reason=? exitCode=?') + assert.equal(describeRendererLifecycleEvent({ kind: 'secondary', event: 'render-process-gone', reason: 'crashed', exitCode: undefined }), '[renderer:secondary] render-process-gone reason=crashed exitCode=?') + assert.equal( + describeRendererLifecycleEvent({ kind: 'main', event: 'render-process-gone', reason: 'killed', exitCode: 1, isDestroyed: true }), + '[renderer:main] render-process-gone reason=killed exitCode=1 (expected teardown)' + ) +}) diff --git a/apps/desktop/electron/window-renderer-lifecycle.ts b/apps/desktop/electron/window-renderer-lifecycle.ts new file mode 100644 index 0000000000..b006637708 --- /dev/null +++ b/apps/desktop/electron/window-renderer-lifecycle.ts @@ -0,0 +1,344 @@ +// Per-window renderer lifecycle diagnostics + crash recovery (#81290). +// +// The desktop app renders one Chromium renderer per window (primary, secondary +// session windows, full instance windows, and the small helper overlays), but +// renderer-lifecycle listeners used to be attached ONLY to the primary window: +// a dead peer renderer produced no log line and no recovery, leaving the user +// with a permanently black window and nothing in desktop.log. This module +// attaches the same lifecycle wiring to every window, keyed by a `kind` label, +// with an injected reload policy so the pure decision logic stays Electron-free +// and unit-testable (mirroring windows-sandbox-fallback.ts / session-windows.ts). +// +// Policy (matches the primary window's previous behavior, generalized): +// - `render-process-gone` with reason `crashed`/`oom` → bounded reload (rolling +// crash-loop guard shared across ALL windows — one budget per process). +// - `render-process-gone` with any other reason (`killed`, `launch-failed`, +// `clean-exit`, unknown) → log only. `killed` after an expected close/destroy +// is normal teardown, and blindly reloading it would loop windows back up +// after the user closed them. +// - `unresponsive` → log only (no reload; Chromium usually follows with +// render-process-gone, and forcing a reload while the main thread is wedged +// can make things worse). +// - `did-fail-load` on the MAIN frame → log only (no blind reload: a repeatable +// startup failure would boot-loop; the backend startup path already surfaces +// the actionable error). +// - `console-message` at error level → log renderer console errors, same as the +// primary window did. + +export interface RendererLifecycleDetails { + reason?: string + exitCode?: number | string | undefined + isDestroyed?: boolean +} + +export interface RendererLifecycleEvent { + kind: string + event: 'render-process-gone' | 'unresponsive' | 'did-fail-load' | 'console-message' + reason?: string + exitCode?: number | string | undefined + isDestroyed?: boolean + /** did-fail-load: only main-frame failures are meaningful (issue point 4). */ + isMainFrame?: boolean + /** did-fail-load: the Chromium error code (e.g. -3 = ERR_ABORTED). */ + errorCode?: number | string | undefined + /** did-fail-load: the URL that failed. */ + url?: string + /** console-message: renderer error text. */ + message?: string + /** console-message: source URL of the console message. */ + sourceUrl?: string + /** console-message: line number of the console message. */ + lineNumber?: number | string | undefined +} + +export interface ReloadPolicyDecision { + reload: boolean + /** Why reload was refused, when it was. */ + suppressedReason?: 'crash-loop' | 'expected-teardown' | 'unrecoverable-reason' +} + +export interface WindowRendererLifecycleOptions { + /** Stable label used in log lines: 'main' | 'secondary' | 'instance' | + * 'overlay' | 'quick' | 'wake'. */ + kind: string + callbacks: { + log: (message: string) => void + /** Omitted → log-only mode (helper windows never reload). */ + reload?: () => void + /** Called when the shared crash-loop budget suppresses a reload — the + * primary window uses it for the #38216 Windows sandbox relaunch check. */ + onCrashLoopSuppressed?: (details?: RendererLifecycleDetails) => void + } + /** Rolling crash-loop window, ms. Defaults to 60_000 (RENDERER_RELOAD_WINDOW_MS). */ + reloadWindowMs?: number + /** Max reloads per rolling window. Defaults to 3 (RENDERER_RELOAD_MAX). */ + reloadMax?: number + /** Shared per-process reload budget. Omitted → per-window budget (tests). */ + recentReloadTimesRef?: { current: number[] } + now?: () => number +} + +/** Minimal structural surface of BrowserWindow / webContents used here. */ +export interface LifecycleWindowLike { + isDestroyed: () => boolean + webContents: { + on: (event: string, listener: (...args: any[]) => void) => unknown + reload?: () => void + removeListener?: (event: string, listener: (...args: any[]) => void) => unknown + } +} + +const DEFAULT_RELOAD_WINDOW_MS = 60_000 +const DEFAULT_RELOAD_MAX = 3 + +const RECOVERABLE_REASONS = new Set(['crashed', 'oom']) + +function safeNow(now: (() => number) | undefined): number { + return typeof now === 'function' ? now() : Date.now() +} + +function isWithin(timestamp: number, now: number, windowMs: number): boolean { + return now - timestamp < windowMs +} + +/** Drop reload timestamps outside the rolling window. Mutates + returns. */ +export function pruneReloadTimes(times: number[], now: number, windowMs: number): number[] { + return times.filter(timestamp => isWithin(timestamp, now, windowMs)) +} + +/** Record a reload attempt timestamp. Mutates + returns. */ +export function pushReloadTime(times: number[], now: number): number[] { + times.push(now) + + return times +} + +/** + * Decide whether a render-process-gone event should reload its window. + * + * Reload only for `crashed`/`oom` on a live window, bounded by the shared + * rolling crash-loop budget. Anything else — expected teardown (`killed` after + * close/destroy), unrecoverable reasons, unknown reasons — is log-only, exactly + * like the primary window's previous behavior but now per window kind. + */ +export function shouldReloadAfterRendererGone(details: { + reason?: string + isDestroyed?: boolean + recentReloadTimes: number[] + reloadWindowMs?: number + reloadMax?: number + now?: () => number +}): ReloadPolicyDecision { + if (details.isDestroyed) { + return { reload: false, suppressedReason: 'expected-teardown' } + } + + const reason = String(details.reason || '') + + if (!RECOVERABLE_REASONS.has(reason)) { + return { reload: false, suppressedReason: 'unrecoverable-reason' } + } + + const windowMs = details.reloadWindowMs ?? DEFAULT_RELOAD_WINDOW_MS + const max = details.reloadMax ?? DEFAULT_RELOAD_MAX + const now = safeNow(details.now) + const recent = pruneReloadTimes(details.recentReloadTimes, now, windowMs) + + if (recent.length >= max) { + return { reload: false, suppressedReason: 'crash-loop' } + } + + return { reload: true } +} + +/** + * One log line per renderer lifecycle event, e.g. + * [renderer:secondary] render-process-gone reason=crashed exitCode=3 + * Sanitizes unknown fields and annotates expected teardown so a support bundle + * reads as a story, not a pile of question marks. + */ +export function describeRendererLifecycleEvent(event: RendererLifecycleEvent): string { + const kind = String(event.kind || '?') + + if (event.event === 'console-message') { + const src = String(event.sourceUrl || '?') + const line = event.lineNumber === undefined ? '?' : String(event.lineNumber) + + return `[renderer:${kind} console] ${String(event.message || '(empty)')} (${src}:${line})` + } + + if (event.event === 'unresponsive') { + return `[renderer:${kind}] webContents became unresponsive` + } + + if (event.event === 'did-fail-load') { + const code = event.errorCode === undefined ? '?' : String(event.errorCode) + const url = String(event.url || '?') + + return `[renderer:${kind}] did-fail-load code=${code} url=${url}` + } + + const reason = String(event.reason || '?') + const exitCode = event.exitCode === undefined ? '?' : String(event.exitCode) + const teardown = event.isDestroyed && reason === 'killed' ? ' (expected teardown)' : '' + + return `[renderer:${kind}] render-process-gone reason=${reason} exitCode=${exitCode}${teardown}` +} + +// Electron ≥36 passes (event, messageDetails); older/deprecated shape is +// (event, level, message, line, sourceId). Handle both, matching the primary +// window's previous console-message handling. +export function consoleMessageLog(args: readonly unknown[]): { + level: number + message: string + sourceUrl: string + lineNumber: number | string | undefined +} { + const details = args[1] && typeof args[1] === 'object' ? (args[1] as Record) : null + const level = details ? Number(details.level ?? 0) : Number(args[1] ?? 0) + const modernLine = details?.lineNumber + + return { + level, + message: details ? String(details.message ?? '') : String(args[2] ?? ''), + sourceUrl: details ? String(details.sourceUrl ?? '') : String(args[4] ?? ''), + lineNumber: details + ? typeof modernLine === 'number' + ? modernLine + : modernLine === undefined + ? undefined + : String(modernLine) + : typeof args[3] === 'number' + ? args[3] + : String(args[3] ?? '') + } +} + +/** + * Attach renderer lifecycle listeners to a window. Returns a dispose() that + * removes every listener (window recreation must not stack handlers). + * + * `reload` is never invoked synchronously inside the event handler: Electron + * warns about re-entrant webContents calls, and the primary window's previous + * implementation deferred via setImmediate for the same reason. + */ +export function installWindowRendererLifecycle( + win: LifecycleWindowLike, + options: WindowRendererLifecycleOptions +): () => void { + const kind = options.kind + const { log, reload, onCrashLoopSuppressed } = options.callbacks + const reloadWindowMs = options.reloadWindowMs ?? DEFAULT_RELOAD_WINDOW_MS + const reloadMax = options.reloadMax ?? DEFAULT_RELOAD_MAX + const now = options.now + const budgetRef = options.recentReloadTimesRef ?? { current: [] } + const contents = win.webContents + + const onRendererGone = (_event: unknown, details?: RendererLifecycleDetails) => { + const destroyed = win.isDestroyed() + + log(describeRendererLifecycleEvent({ kind, event: 'render-process-gone', ...details, isDestroyed: destroyed })) + + const nowMs = safeNow(now) + const recent = pruneReloadTimes(budgetRef.current, nowMs, reloadWindowMs) + + budgetRef.current.length = 0 + budgetRef.current.push(...recent) + + const decision = shouldReloadAfterRendererGone({ + reason: details?.reason, + isDestroyed: destroyed, + recentReloadTimes: budgetRef.current, + reloadWindowMs, + reloadMax, + now: () => nowMs + }) + + if (!decision.reload) { + if (decision.suppressedReason === 'crash-loop') { + log( + `[renderer:${kind}] suppressing reload: ${budgetRef.current.length} crashes within ${reloadWindowMs}ms (likely a crash loop)` + ) + onCrashLoopSuppressed?.(details) + } + + return + } + + if (typeof reload !== 'function') { + return + } + + pushReloadTime(budgetRef.current, nowMs) + + // Deferred: never reload from inside the event handler (see above). + setImmediate(() => { + if (win.isDestroyed()) { + return + } + + try { + reload() + } catch (error) { + log(`[renderer:${kind}] reload after crash failed: ${error instanceof Error ? error.message : String(error)}`) + } + }) + } + + const onUnresponsive = () => { + log(describeRendererLifecycleEvent({ kind, event: 'unresponsive' })) + } + + const onDidFailLoad = (_event: unknown, errorCode: unknown, _errorDescription: unknown, validatedURL: unknown, isMainFrame?: unknown) => { + if (isMainFrame === true) { + log( + describeRendererLifecycleEvent({ + kind, + event: 'did-fail-load', + errorCode: typeof errorCode === 'number' ? errorCode : String(errorCode ?? ''), + url: String(validatedURL ?? '') + }) + ) + } + } + + const onConsoleMessage = (...args: unknown[]) => { + const parsed = consoleMessageLog(args) + + if (parsed.level !== 3) { + return + } + + log( + describeRendererLifecycleEvent({ + kind, + event: 'console-message', + message: parsed.message, + sourceUrl: parsed.sourceUrl, + lineNumber: parsed.lineNumber + }) + ) + } + + contents.on('render-process-gone', onRendererGone) + contents.on('unresponsive', onUnresponsive) + contents.on('did-fail-load', onDidFailLoad) + contents.on('console-message', onConsoleMessage) + + let disposed = false + + return () => { + if (disposed) { + return + } + + disposed = true + + if (typeof contents.removeListener === 'function') { + contents.removeListener('render-process-gone', onRendererGone) + contents.removeListener('unresponsive', onUnresponsive) + contents.removeListener('did-fail-load', onDidFailLoad) + contents.removeListener('console-message', onConsoleMessage) + } + } +}