hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

fix(desktop): coalesce the Linux GPU-child recovery ladders

Browse Source 
One GPU-child death now produces exactly one bounded relaunch instead of
two racing ones: the #121954 sandbox signature (SIGTERM/143) is tried
first — that host matrix shows --disable-gpu still crashes — and any
other GPU failure reason falls to the #124843 software ladder,
including the relapse after a --no-sandbox boot died again.

Also: HERMES_DESKTOP_DISABLE_GPU=1 now appends --disable-gpu so the GPU
process is fully spawn-blocked, not just compositing (#97616 — a GPU
child still spawned and died error_code=1002 with compositing-only
switches), and a post-reveal check catches the #124843 silent-retry
manifestation (window reveals, no gpu-process child, sub-zygote
spinning at ~350% CPU) that the reactive ladder never sees, engaging the
sticky software fallback for the next launch.
This commit is contained in:
Brooklyn Nicholson
2026-09-28 21:36:24 -05:00
committed by brooklyn!
parent 062a003a09
commit 5106ffa2cb
2 changed files with 250 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
apps/desktop/electron/linux-gpu-fallback.ts
View File

@@ -35,6 +35,13 @@ export const LINUX_GPU_FALLBACK_MARKER_FILENAME = 'linux-gpu-fallback.json'
/** Consecutive mid-boot aborts required before falling back to software. */
export const BOOT_ABORTS_BEFORE_LINUX_GPU_FALLBACK = 2
/**
* Seconds after the main window reveals before the #124843 silent-retry check
* looks for a GPU child: long enough that a healthy host's GPU child exists,
* short enough that a sub-zygote spinning at ~350% CPU gets bounded fast.
*/
export const LINUX_GPU_SILENT_RETRY_GRACE_S = 30
/** `child-process-gone` reasons that witness a broken GPU child. */
const GPU_FAILURE_REASONS = new Set(['crashed', 'launch-failure'])
@@ -289,3 +296,101 @@ export function shouldRelaunchForLinuxGpuCrash(options: {
return GPU_FAILURE_REASONS.has(String(options.details?.reason || '').toLowerCase())
}
/**
* Exit status Chromium uses to SIGTERM its GPU process when it never became
* usable (#121954): the sandbox-blocked GPU child dies pre-main on an
* FD-ownership violation and the browser prints "GPU process isn't usable.
* Goodbye." before SIGTERMing it. Node reports signal deaths as `null`
* exitCode + `signalName: 'SIGTERM'`; Electron serializes that as 143.
*/
export const GPU_CHILD_SANDBOX_SIGTERM_EXIT = 143
export type LinuxGpuDeathPath = 'no-sandbox' | 'disable-gpu' | null
/**
* Coalesced recovery decision for one Linux GPU-child death (#86073, #124843,
* #121954). One death produces exactly one bounded relaunch:
*
* - The sandbox signature (SIGTERM/143, #121954) is tried FIRST: that host
* matrix shows `--disable-gpu` still crashes, so software fallback would
* waste the one shot. `alreadyNoSandbox` covers a sticky/relaunched
* `--no-sandbox` boot, `sandboxRelaunchAttempted` the one-shot guard.
* - Any other GPU failure reason (crashed / launch-failure, #124843) falls to
* the software ladder — including the RELAPSE after a `--no-sandbox`
* relaunch died again: that second death proves the sandbox was not the
* (only) problem, so the next bounded step is `--disable-gpu`.
* - null when the death matches neither ladder or both are spent/already on.
*
* Pure so tests never boot Electron; the caller owns the relaunch flags.
*/
export function linuxGpuChildDeathPath(options: {
platform?: NodeJS.Platform | string
details?: { type?: string; reason?: string; exitCode?: number | null; signalName?: string | null } | null
alreadyNoSandbox?: boolean
alreadySoftware?: boolean
sandboxRelaunchAttempted?: boolean
softwareRelaunchAttempted?: boolean
}): LinuxGpuDeathPath {
if ((options.platform ?? process.platform) !== 'linux') {
return null
}
const type = String(options.details?.type || '').toLowerCase()
if (type !== 'gpu') {
return null
}
const reason = String(options.details?.reason || '').toLowerCase()
const sandboxSignature =
options.details?.exitCode === GPU_CHILD_SANDBOX_SIGTERM_EXIT &&
String(options.details?.signalName || '').toUpperCase() === 'SIGTERM'
if (sandboxSignature && !options.alreadyNoSandbox && !options.sandboxRelaunchAttempted) {
return 'no-sandbox'
}
if (GPU_FAILURE_REASONS.has(reason) && !options.alreadySoftware && !options.softwareRelaunchAttempted) {
return 'disable-gpu'
}
return null
}
/**
* #97616: `HERMES_DESKTOP_DISABLE_GPU=1` must fully prevent the GPU process,
* not just compositing. `disableHardwareAcceleration()` alone provably still
* spawns a GPU child (which then fails error_code=1002 on AMD/Mesa). The
* remote-display detection reasons keep their long-standing compositing-only
* behavior; only the explicit env override gains the spawn-blocking switch.
*/
export function disableGpuSwitchNeededForReason(remoteDisplayReason: string | null): boolean {
return typeof remoteDisplayReason === 'string' && remoteDisplayReason.startsWith('override')
}
/**
* #124843 silent-retry manifestation: the app boots "successfully" (window
* reveals, marker flips to ok) while a sub-zygote retries GPU init forever —
* no `child-process-gone` event ever fires. N seconds after the main window
* reveals, the absence of a `--type=gpu-process` child on a boot that was
* supposed to have one means the retry loop is running: engage the sticky
* software fallback. A GPU child present, a non-Linux host, software already
* on, or the check run before the grace window elapsed all decline.
*/
export function shouldEngageSilentGpuRetryFallback(options: {
platform?: NodeJS.Platform | string
gpuChildPresent?: boolean
graceElapsed?: boolean
alreadySoftware?: boolean
}): boolean {
if ((options.platform ?? process.platform) !== 'linux') {
return false
}
if (options.alreadySoftware || !options.graceElapsed || options.gpuChildPresent) {
return false
}
return true
}

208
apps/desktop/electron/main.ts
View File

@@ -670,10 +670,13 @@ import {
} from './windows-stack-cookie-fallback'
import {
decideLinuxGpuLaunch,
disableGpuSwitchNeededForReason,
LINUX_GPU_SILENT_RETRY_GRACE_S,
linuxGpuChildDeathPath,
linuxGpuFallbackMarker,
linuxGpuMarkerAfterSuccessfulBoot,
readLinuxGpuMarker,
shouldRelaunchForLinuxGpuCrash,
shouldEngageSilentGpuRetryFallback,
writeLinuxGpuMarker
} from './linux-gpu-fallback'
import { readWindowsUserEnvVar } from './windows-user-env'
@@ -732,6 +735,13 @@ if (REMOTE_DISPLAY_REASON) {
// Belt-and-suspenders for X11/VNC, where the Viz compositor can still glitch
// with only --disable-gpu: force compositing onto the CPU too.
app.commandLine.appendSwitch('disable-gpu-compositing')
// #97616: disableHardwareAcceleration() alone does NOT stop a GPU child
// from spawning (it then dies error_code=1002 on AMD/Mesa). For the explicit
// HERMES_DESKTOP_DISABLE_GPU override, fully spawn-block it. Remote-display
// detections keep their long-standing compositing-only behavior.
if (disableGpuSwitchNeededForReason(REMOTE_DISPLAY_REASON)) {
app.commandLine.appendSwitch('disable-gpu')
}
console.log(
`[hermes] remote display detected (${REMOTE_DISPLAY_REASON}); disabling GPU hardware acceleration to prevent flicker`
)
@@ -1050,82 +1060,108 @@ if (IS_WINDOWS || process.platform === 'linux') {
writeSandboxMarker(windowsUserData, sandboxDecision.nextMarker)
// Catch the first GPU breakpoint death and relaunch before Chromium's
// "GPU process isn't usable" FATAL abort ends the process with no recovery.
// One coalesced Linux GPU-child recovery (#86073, #124843, #121954): the
// sandbox signature is tried first (that host's matrix shows --disable-gpu
// still crashes), then the software ladder — including the relapse after a
// --no-sandbox boot died again. One death, one bounded relaunch; Windows
// keeps its breakpoint-signature fast path unchanged.
app.on('child-process-gone', (_event, details) => {
if (
!shouldRelaunchForGpuSandboxCrash({
details,
alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env),
relaunchAttempted: windowsNoSandboxRelaunchAttempted
})
) {
if (IS_WINDOWS) {
if (
!shouldRelaunchForGpuSandboxCrash({
details,
alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env),
relaunchAttempted: windowsNoSandboxRelaunchAttempted
})
) {
return
}
windowsNoSandboxRelaunchAttempted = true
windowsSandboxFallbackActive = true
windowsSandboxFallbackSticky = true
windowsSandboxFallbackReason = 'gpu-breakpoint'
try {
writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion()))
} catch {
void 0
}
console.warn(
`[hermes] GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#38216)`
)
try {
app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) })
void exitAfterBackendShutdown(0)
} catch (error) {
console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`)
}
return
}
windowsNoSandboxRelaunchAttempted = true
windowsSandboxFallbackActive = true
windowsSandboxFallbackSticky = true
windowsSandboxFallbackReason = 'gpu-breakpoint'
try {
writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion()))
} catch {
void 0
}
console.warn(
`[hermes] GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#38216, #121954)`
)
try {
app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) })
void exitAfterBackendShutdown(0)
} catch (error) {
console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`)
}
})
}
// #124843: catch a Linux GPU-child launch failure/crash and relaunch once
// with --disable-gpu before Chromium's init-retry loop burns the machine.
// No-op when software rendering is already on.
if (process.platform === 'linux') {
app.on('child-process-gone', (_event, details) => {
const alreadySoftware =
LINUX_GPU_SOFTWARE_ACTIVE || linuxGpuFallbackActive || alreadyHasDisableGpu(process.argv, process.env)
const path = linuxGpuChildDeathPath({
details,
alreadyNoSandbox: windowsSandboxFallbackActive || alreadyHasNoSandbox(process.argv, process.env),
alreadySoftware,
sandboxRelaunchAttempted: windowsNoSandboxRelaunchAttempted,
softwareRelaunchAttempted: linuxGpuRelaunchAttempted
})
if (path === 'no-sandbox') {
windowsNoSandboxRelaunchAttempted = true
windowsSandboxFallbackActive = true
windowsSandboxFallbackSticky = true
windowsSandboxFallbackReason = 'gpu-breakpoint'
try {
writeSandboxMarker(app.getPath('userData'), fallbackMarker('gpu-breakpoint', app.getVersion()))
} catch {
void 0
}
console.warn(
`[hermes] Linux GPU child died with the sandbox signature (exit=${details?.exitCode}); relaunching once with --no-sandbox (#121954)`
)
try {
app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) })
void exitAfterBackendShutdown(0)
} catch (error) {
console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`)
}
if (
!shouldRelaunchForLinuxGpuCrash({
details,
alreadySoftware,
relaunchAttempted: linuxGpuRelaunchAttempted
})
) {
return
}
linuxGpuRelaunchAttempted = true
linuxGpuFallbackActive = true
linuxGpuFallbackSticky = true
if (path === 'disable-gpu') {
linuxGpuRelaunchAttempted = true
linuxGpuFallbackActive = true
linuxGpuFallbackSticky = true
const reason = String(details?.reason || '').toLowerCase() === 'launch-failure' ? 'gpu-launch-failure' : 'gpu-crash'
const reason =
String(details?.reason || '').toLowerCase() === 'launch-failure' ? 'gpu-launch-failure' : 'gpu-crash'
try {
writeLinuxGpuMarker(app.getPath('userData'), linuxGpuFallbackMarker(reason, app.getVersion()))
} catch {
void 0
}
try {
writeLinuxGpuMarker(app.getPath('userData'), linuxGpuFallbackMarker(reason, app.getVersion()))
} catch {
void 0
}
console.warn(
`[hermes] Linux GPU child gone (reason=${details?.reason}); relaunching once with --disable-gpu (#124843)`
)
console.warn(
`[hermes] Linux GPU child gone (reason=${details?.reason}); relaunching once with --disable-gpu (#124843)`
)
try {
app.relaunch({ args: buildDisableGpuRelaunchArgs(process.argv.slice(1)) })
void exitAfterBackendShutdown(0)
} catch (error) {
console.error(`[hermes] --disable-gpu relaunch failed: ${error?.message || error}`)
try {
app.relaunch({ args: buildDisableGpuRelaunchArgs(process.argv.slice(1)) })
void exitAfterBackendShutdown(0)
} catch (error) {
console.error(`[hermes] --disable-gpu relaunch failed: ${error?.message || error}`)
}
}
})
}
@@ -15242,6 +15278,52 @@ function createWindow() {
} catch (error) {
rememberLog(`[gpu] linux marker update after main-window reveal failed: ${error?.message || error}`)
}
// #124843 silent-retry manifestation: the boot "succeeded" (window
// revealed, marker ok) while a sub-zygote retries GPU init forever —
// no child-process-gone event ever fires, so the reactive ladder
// above never engages. After a grace window, a boot that should have
// a GPU child but has none is that retry loop: engage the sticky
// software fallback so the NEXT launch skips it (this boot's switches
// already applied pre-ready and cannot change now).
if (!linuxGpuFallbackSticky) {
const checkSilentGpuRetry = (): void => {
const alreadySoftware =
LINUX_GPU_SOFTWARE_ACTIVE ||
linuxGpuFallbackActive ||
alreadyHasDisableGpu(process.argv, process.env) ||
isHermesDesktopGpuOverrideOff(process.env)
const gpuChildPresent = app
.getAppMetrics()
.some(metric => String(metric?.type || '').toLowerCase() === 'gpu')
if (
shouldEngageSilentGpuRetryFallback({
gpuChildPresent,
graceElapsed: true,
alreadySoftware
})
) {
linuxGpuFallbackActive = true
linuxGpuFallbackSticky = true
try {
writeLinuxGpuMarker(
app.getPath('userData'),
linuxGpuFallbackMarker('gpu-launch-failure', app.getVersion())
)
} catch {
void 0
}
console.warn(
'[hermes] Linux: no GPU child after window reveal — GPU init is retrying silently; software fallback engaged for the next launch (#124843)'
)
}
}
setTimeout(checkSilentGpuRetry, LINUX_GPU_SILENT_RETRY_GRACE_S).unref()
}
}
}
})