From 5106ffa2cb72c4a8347dc6efbdc235ff31346e48 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Mon, 28 Sep 2026 21:36:24 -0500 Subject: [PATCH] fix(desktop): coalesce the Linux GPU-child recovery ladders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One GPU-child death now produces exactly one bounded relaunch instead of two racing ones: the #121954 sandbox signature (SIGTERM/143) is tried first — that host matrix shows --disable-gpu still crashes — and any other GPU failure reason falls to the #124843 software ladder, including the relapse after a --no-sandbox boot died again. Also: HERMES_DESKTOP_DISABLE_GPU=1 now appends --disable-gpu so the GPU process is fully spawn-blocked, not just compositing (#97616 — a GPU child still spawned and died error_code=1002 with compositing-only switches), and a post-reveal check catches the #124843 silent-retry manifestation (window reveals, no gpu-process child, sub-zygote spinning at ~350% CPU) that the reactive ladder never sees, engaging the sticky software fallback for the next launch. --- apps/desktop/electron/linux-gpu-fallback.ts | 105 ++++++++++ apps/desktop/electron/main.ts | 208 ++++++++++++++------ 2 files changed, 250 insertions(+), 63 deletions(-) diff --git a/apps/desktop/electron/linux-gpu-fallback.ts b/apps/desktop/electron/linux-gpu-fallback.ts index 8b95ef9bf6..ba3a8c2db6 100644 --- a/apps/desktop/electron/linux-gpu-fallback.ts +++ b/apps/desktop/electron/linux-gpu-fallback.ts @@ -35,6 +35,13 @@ export const LINUX_GPU_FALLBACK_MARKER_FILENAME = 'linux-gpu-fallback.json' /** Consecutive mid-boot aborts required before falling back to software. */ export const BOOT_ABORTS_BEFORE_LINUX_GPU_FALLBACK = 2 +/** + * Seconds after the main window reveals before the #124843 silent-retry check + * looks for a GPU child: long enough that a healthy host's GPU child exists, + * short enough that a sub-zygote spinning at ~350% CPU gets bounded fast. + */ +export const LINUX_GPU_SILENT_RETRY_GRACE_S = 30 + /** `child-process-gone` reasons that witness a broken GPU child. */ const GPU_FAILURE_REASONS = new Set(['crashed', 'launch-failure']) @@ -289,3 +296,101 @@ export function shouldRelaunchForLinuxGpuCrash(options: { return GPU_FAILURE_REASONS.has(String(options.details?.reason || '').toLowerCase()) } + +/** + * Exit status Chromium uses to SIGTERM its GPU process when it never became + * usable (#121954): the sandbox-blocked GPU child dies pre-main on an + * FD-ownership violation and the browser prints "GPU process isn't usable. + * Goodbye." before SIGTERMing it. Node reports signal deaths as `null` + * exitCode + `signalName: 'SIGTERM'`; Electron serializes that as 143. + */ +export const GPU_CHILD_SANDBOX_SIGTERM_EXIT = 143 + +export type LinuxGpuDeathPath = 'no-sandbox' | 'disable-gpu' | null + +/** + * Coalesced recovery decision for one Linux GPU-child death (#86073, #124843, + * #121954). One death produces exactly one bounded relaunch: + * + * - The sandbox signature (SIGTERM/143, #121954) is tried FIRST: that host + * matrix shows `--disable-gpu` still crashes, so software fallback would + * waste the one shot. `alreadyNoSandbox` covers a sticky/relaunched + * `--no-sandbox` boot, `sandboxRelaunchAttempted` the one-shot guard. + * - Any other GPU failure reason (crashed / launch-failure, #124843) falls to + * the software ladder — including the RELAPSE after a `--no-sandbox` + * relaunch died again: that second death proves the sandbox was not the + * (only) problem, so the next bounded step is `--disable-gpu`. + * - null when the death matches neither ladder or both are spent/already on. + * + * Pure so tests never boot Electron; the caller owns the relaunch flags. + */ +export function linuxGpuChildDeathPath(options: { + platform?: NodeJS.Platform | string + details?: { type?: string; reason?: string; exitCode?: number | null; signalName?: string | null } | null + alreadyNoSandbox?: boolean + alreadySoftware?: boolean + sandboxRelaunchAttempted?: boolean + softwareRelaunchAttempted?: boolean +}): LinuxGpuDeathPath { + if ((options.platform ?? process.platform) !== 'linux') { + return null + } + + const type = String(options.details?.type || '').toLowerCase() + + if (type !== 'gpu') { + return null + } + + const reason = String(options.details?.reason || '').toLowerCase() + const sandboxSignature = + options.details?.exitCode === GPU_CHILD_SANDBOX_SIGTERM_EXIT && + String(options.details?.signalName || '').toUpperCase() === 'SIGTERM' + + if (sandboxSignature && !options.alreadyNoSandbox && !options.sandboxRelaunchAttempted) { + return 'no-sandbox' + } + + if (GPU_FAILURE_REASONS.has(reason) && !options.alreadySoftware && !options.softwareRelaunchAttempted) { + return 'disable-gpu' + } + + return null +} + +/** + * #97616: `HERMES_DESKTOP_DISABLE_GPU=1` must fully prevent the GPU process, + * not just compositing. `disableHardwareAcceleration()` alone provably still + * spawns a GPU child (which then fails error_code=1002 on AMD/Mesa). The + * remote-display detection reasons keep their long-standing compositing-only + * behavior; only the explicit env override gains the spawn-blocking switch. + */ +export function disableGpuSwitchNeededForReason(remoteDisplayReason: string | null): boolean { + return typeof remoteDisplayReason === 'string' && remoteDisplayReason.startsWith('override') +} + +/** + * #124843 silent-retry manifestation: the app boots "successfully" (window + * reveals, marker flips to ok) while a sub-zygote retries GPU init forever — + * no `child-process-gone` event ever fires. N seconds after the main window + * reveals, the absence of a `--type=gpu-process` child on a boot that was + * supposed to have one means the retry loop is running: engage the sticky + * software fallback. A GPU child present, a non-Linux host, software already + * on, or the check run before the grace window elapsed all decline. + */ +export function shouldEngageSilentGpuRetryFallback(options: { + platform?: NodeJS.Platform | string + gpuChildPresent?: boolean + graceElapsed?: boolean + alreadySoftware?: boolean +}): boolean { + if ((options.platform ?? process.platform) !== 'linux') { + return false + } + + if (options.alreadySoftware || !options.graceElapsed || options.gpuChildPresent) { + return false + } + + return true +} diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 33b8cca271..5f6a75d084 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -670,10 +670,13 @@ import { } from './windows-stack-cookie-fallback' import { decideLinuxGpuLaunch, + disableGpuSwitchNeededForReason, + LINUX_GPU_SILENT_RETRY_GRACE_S, + linuxGpuChildDeathPath, linuxGpuFallbackMarker, linuxGpuMarkerAfterSuccessfulBoot, readLinuxGpuMarker, - shouldRelaunchForLinuxGpuCrash, + shouldEngageSilentGpuRetryFallback, writeLinuxGpuMarker } from './linux-gpu-fallback' import { readWindowsUserEnvVar } from './windows-user-env' @@ -732,6 +735,13 @@ if (REMOTE_DISPLAY_REASON) { // Belt-and-suspenders for X11/VNC, where the Viz compositor can still glitch // with only --disable-gpu: force compositing onto the CPU too. app.commandLine.appendSwitch('disable-gpu-compositing') + // #97616: disableHardwareAcceleration() alone does NOT stop a GPU child + // from spawning (it then dies error_code=1002 on AMD/Mesa). For the explicit + // HERMES_DESKTOP_DISABLE_GPU override, fully spawn-block it. Remote-display + // detections keep their long-standing compositing-only behavior. + if (disableGpuSwitchNeededForReason(REMOTE_DISPLAY_REASON)) { + app.commandLine.appendSwitch('disable-gpu') + } console.log( `[hermes] remote display detected (${REMOTE_DISPLAY_REASON}); disabling GPU hardware acceleration to prevent flicker` ) @@ -1050,82 +1060,108 @@ if (IS_WINDOWS || process.platform === 'linux') { writeSandboxMarker(windowsUserData, sandboxDecision.nextMarker) - // Catch the first GPU breakpoint death and relaunch before Chromium's - // "GPU process isn't usable" FATAL abort ends the process with no recovery. + // One coalesced Linux GPU-child recovery (#86073, #124843, #121954): the + // sandbox signature is tried first (that host's matrix shows --disable-gpu + // still crashes), then the software ladder — including the relapse after a + // --no-sandbox boot died again. One death, one bounded relaunch; Windows + // keeps its breakpoint-signature fast path unchanged. app.on('child-process-gone', (_event, details) => { - if ( - !shouldRelaunchForGpuSandboxCrash({ - details, - alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env), - relaunchAttempted: windowsNoSandboxRelaunchAttempted - }) - ) { + if (IS_WINDOWS) { + if ( + !shouldRelaunchForGpuSandboxCrash({ + details, + alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env), + relaunchAttempted: windowsNoSandboxRelaunchAttempted + }) + ) { + return + } + + windowsNoSandboxRelaunchAttempted = true + windowsSandboxFallbackActive = true + windowsSandboxFallbackSticky = true + windowsSandboxFallbackReason = 'gpu-breakpoint' + + try { + writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion())) + } catch { + void 0 + } + + console.warn( + `[hermes] GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#38216)` + ) + + try { + app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) + void exitAfterBackendShutdown(0) + } catch (error) { + console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`) + } + return } - windowsNoSandboxRelaunchAttempted = true - windowsSandboxFallbackActive = true - windowsSandboxFallbackSticky = true - windowsSandboxFallbackReason = 'gpu-breakpoint' - - try { - writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion())) - } catch { - void 0 - } - - console.warn( - `[hermes] GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#38216, #121954)` - ) - - try { - app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) - void exitAfterBackendShutdown(0) - } catch (error) { - console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`) - } - }) -} - -// #124843: catch a Linux GPU-child launch failure/crash and relaunch once -// with --disable-gpu before Chromium's init-retry loop burns the machine. -// No-op when software rendering is already on. -if (process.platform === 'linux') { - app.on('child-process-gone', (_event, details) => { const alreadySoftware = LINUX_GPU_SOFTWARE_ACTIVE || linuxGpuFallbackActive || alreadyHasDisableGpu(process.argv, process.env) + const path = linuxGpuChildDeathPath({ + details, + alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env), + alreadySoftware, + sandboxRelaunchAttempted: windowsNoSandboxRelaunchAttempted, + softwareRelaunchAttempted: linuxGpuRelaunchAttempted + }) + + if (path === 'no-sandbox') { + windowsNoSandboxRelaunchAttempted = true + windowsSandboxFallbackActive = true + windowsSandboxFallbackSticky = true + windowsSandboxFallbackReason = 'gpu-breakpoint' + + try { + writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion())) + } catch { + void 0 + } + + console.warn( + `[hermes] Linux GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#121954)` + ) + + try { + app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) + void exitAfterBackendShutdown(0) + } catch (error) { + console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`) + } - if ( - !shouldRelaunchForLinuxGpuCrash({ - details, - alreadySoftware, - relaunchAttempted: linuxGpuRelaunchAttempted - }) - ) { return } - linuxGpuRelaunchAttempted = true - linuxGpuFallbackActive = true - linuxGpuFallbackSticky = true + if (path === 'disable-gpu') { + linuxGpuRelaunchAttempted = true + linuxGpuFallbackActive = true + linuxGpuFallbackSticky = true - const reason = String(details?.reason || '').toLowerCase() === 'launch-failure' ? 'gpu-launch-failure' : 'gpu-crash' + const reason = + String(details?.reason || '').toLowerCase() === 'launch-failure' ? 'gpu-launch-failure' : 'gpu-crash' - try { - writeLinuxGpuMarker(app.getPath('userData'), linuxGpuFallbackMarker(reason, app.getVersion())) - } catch { - void 0 - } + try { + writeLinuxGpuMarker(app.getPath('userData'), linuxGpuFallbackMarker(reason, app.getVersion())) + } catch { + void 0 + } - console.warn( - `[hermes] Linux GPU child gone (reason=${details?.reason}); relaunching once with --disable-gpu (#124843)` - ) + console.warn( + `[hermes] Linux GPU child gone (reason=${details?.reason}); relaunching once with --disable-gpu (#124843)` + ) - try { - app.relaunch({ args: buildDisableGpuRelaunchArgs(process.argv.slice(1)) }) - void exitAfterBackendShutdown(0) - } catch (error) { - console.error(`[hermes] --disable-gpu relaunch failed: ${error?.message || error}`) + try { + app.relaunch({ args: buildDisableGpuRelaunchArgs(process.argv.slice(1)) }) + void exitAfterBackendShutdown(0) + } catch (error) { + console.error(`[hermes] --disable-gpu relaunch failed: ${error?.message || error}`) + } } }) } @@ -15242,6 +15278,52 @@ function createWindow() { } catch (error) { rememberLog(`[gpu] linux marker update after main-window reveal failed: ${error?.message || error}`) } + + // #124843 silent-retry manifestation: the boot "succeeded" (window + // revealed, marker ok) while a sub-zygote retries GPU init forever — + // no child-process-gone event ever fires, so the reactive ladder + // above never engages. After a grace window, a boot that should have + // a GPU child but has none is that retry loop: engage the sticky + // software fallback so the NEXT launch skips it (this boot's switches + // already applied pre-ready and cannot change now). + if (!linuxGpuFallbackSticky) { + const checkSilentGpuRetry = (): void => { + const alreadySoftware = + LINUX_GPU_SOFTWARE_ACTIVE || + linuxGpuFallbackActive || + alreadyHasDisableGpu(process.argv, process.env) || + isHermesDesktopGpuOverrideOff(process.env) + const gpuChildPresent = app + .getAppMetrics() + .some(metric => String(metric?.type || '').toLowerCase() === 'gpu') + + if ( + shouldEngageSilentGpuRetryFallback({ + gpuChildPresent, + graceElapsed: true, + alreadySoftware + }) + ) { + linuxGpuFallbackActive = true + linuxGpuFallbackSticky = true + + try { + writeLinuxGpuMarker( + app.getPath('userData'), + linuxGpuFallbackMarker('gpu-launch-failure', app.getVersion()) + ) + } catch { + void 0 + } + + console.warn( + '[hermes] Linux: no GPU child after window reveal — GPU init is retrying silently; software fallback engaged for the next launch (#124843)' + ) + } + } + + setTimeout(checkSilentGpuRetry, LINUX_GPU_SILENT_RETRY_GRACE_S).unref() + } } } })