merge: integrate final desktop update handoff tests
This commit is contained in:
ethernet
2
.github/workflows/install-e2e.yml
vendored
2
.github/workflows/install-e2e.yml
vendored
@@ -270,7 +270,7 @@ jobs:
|
||||
--paginate --jq '.jobs[] | {name, conclusion}' > /tmp/e2e-jobs.ndjson
|
||||
gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts?per_page=100" \
|
||||
--paginate --jq '.artifacts[] | {name, id}' > /tmp/e2e-artifacts.ndjson
|
||||
echo 'Legend: ✅ ran green · ❌ ran red · pre-desktop / TODO = why a leg skipped · 📼 opens the leg player (recording + synced logs)'
|
||||
echo 'Legend: ✅ upgrade passed · known [n] = exact historical failure, see footnote · ❌ unexpected failure · pre-desktop / TODO = why a leg skipped · 📼 opens the leg player (recording + synced logs)'
|
||||
echo
|
||||
node scripts/sandbox/generate-e2e-matrix.mjs --format results \
|
||||
--tags '${{ needs.pick-releases.outputs.tags }}' \
|
||||
|
||||
@@ -6,11 +6,14 @@ import { expect, it, vi } from 'vitest'
|
||||
const { observeProcessClose } = createRequire(import.meta.url)('../tests/install/e2e-assets/process-close.cjs')
|
||||
|
||||
it('waits for native close, not exit, and retains a close observed before hand-off', async () => {
|
||||
const child = new EventEmitter()
|
||||
const pipe = { destroy: vi.fn() }
|
||||
const child = Object.assign(new EventEmitter(), { stdio: [null, pipe], exitCode: null, signalCode: null })
|
||||
const waitForClose = observeProcessClose(child)
|
||||
expect(pipe.destroy).not.toHaveBeenCalled()
|
||||
let finished = false
|
||||
const completion = waitForClose().then(() => { finished = true })
|
||||
child.emit('exit', 0)
|
||||
expect(pipe.destroy).toHaveBeenCalledOnce()
|
||||
await Promise.resolve()
|
||||
expect(finished).toBe(false)
|
||||
child.emit('close', 0)
|
||||
@@ -22,7 +25,7 @@ it('waits for native close, not exit, and retains a close observed before hand-o
|
||||
it('fails if the launched process never closes', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const waitForClose = observeProcessClose(new EventEmitter())
|
||||
const waitForClose = observeProcessClose(Object.assign(new EventEmitter(), { stdio: [], exitCode: null, signalCode: null }))
|
||||
const completion = expect(waitForClose(2_000)).rejects.toThrow('Electron process did not close')
|
||||
await vi.advanceTimersByTimeAsync(2_000)
|
||||
await completion
|
||||
|
||||
@@ -39,4 +39,6 @@ Evidence required: an app-update leg from this released commit and those explici
|
||||
|
||||
## Not classified as unfixable
|
||||
|
||||
The July desktop-installer → app-update failure was a driver lifetime bug, not a released-updater exception. The driver treated an expected page closure as failure and could exit before Playwright released its launch process. On Windows, inherited pipes delayed the `close` event even after the launch process exited with code 0. Playwright then ran its tree-kill cleanup. The driver now waits independently of the closing page, releases its pipe handles after process exit, and waits for `close` before it exits. [The real July rerun](https://github.com/ethernet8023/hermes-agent/actions/runs/34075042380/job/101599434616) reached the target commit, cleared the update marker, passed the CLI check, and relaunched the app.
|
||||
|
||||
Onboarding click failures, zoom drift, native permission dialogs, AutoHotkey window waits, stale update markers, autostash conflicts, network failures, and generic timeouts remain actionable or unclassified until diagnosed. They must not inherit a historical label because they occurred on an old release.
|
||||
|
||||
@@ -19,8 +19,6 @@
|
||||
const path = require('node:path')
|
||||
const fs = require('node:fs')
|
||||
|
||||
// Capture Playwright's native process teardown in disposable CI evidence.
|
||||
if (process.env.GITHUB_ACTIONS === 'true') process.env.DEBUG = 'pw:browser'
|
||||
const { _electron } = require('@playwright/test')
|
||||
const { prepareWindowForInput } = require('./window-input.cjs')
|
||||
const { observeProcessClose } = require('./process-close.cjs')
|
||||
@@ -97,8 +95,7 @@ async function main() {
|
||||
timeout: 120_000
|
||||
})
|
||||
const child = app.process()
|
||||
child.on('exit', (code, signal) => log(`launch process exit code=${code} signal=${signal}`))
|
||||
child.on('close', (code, signal) => log(`launch process close code=${code} signal=${signal}`))
|
||||
|
||||
const waitForProcessClose = observeProcessClose(child)
|
||||
log(`launched Electron pid=${child.pid}`)
|
||||
|
||||
@@ -326,12 +323,7 @@ async function main() {
|
||||
|
||||
// A marker appears before Electron exits. Exiting this driver at that point
|
||||
// lets Playwright taskkill the entire tree, including the detached updater.
|
||||
try {
|
||||
await waitForProcessClose()
|
||||
} catch (error) {
|
||||
log(`launch process state: exitCode=${child.exitCode} signalCode=${child.signalCode} killed=${child.killed}`)
|
||||
throw error
|
||||
}
|
||||
await waitForProcessClose()
|
||||
log('Electron process closed — detached updater owns the rest')
|
||||
}
|
||||
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
"""CI-only Python stack snapshots for the opaque staged-updater hand-off.
|
||||
|
||||
No command arguments, environment, or frame locals are recorded. The real
|
||||
updater runs unchanged; stacks identify where its child is blocked.
|
||||
"""
|
||||
import os
|
||||
|
||||
if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("HERMES_E2E_HANDOFF_TRACE"):
|
||||
import faulthandler
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_directory = Path(os.environ["HERMES_E2E_HANDOFF_TRACE"])
|
||||
_directory.mkdir(parents=True, exist_ok=True)
|
||||
_stream = (_directory / f"python-stacks-{os.getpid()}.log").open("a", encoding="utf-8")
|
||||
_stream.write(f"started pid={os.getpid()} parent={os.getppid()} executable={sys.executable}\n")
|
||||
_stream.flush()
|
||||
faulthandler.dump_traceback_later(90, repeat=True, file=_stream)
|
||||
|
||||
# Call boundaries locate an early exit before the first timed stack dump.
|
||||
# Never record arguments, frame locals, or return values.
|
||||
_watched = {
|
||||
"cmd_update", "_cmd_update_impl", "_run_pre_update_backup",
|
||||
"_pause_windows_gateways_for_update", "find_gateway_pids",
|
||||
"_scan_gateway_pids", "_get_service_pids", "_install_hangup_protection",
|
||||
"_finalize_update_output", "load_config", "is_installed",
|
||||
}
|
||||
|
||||
def _trace_calls(frame, event, arg):
|
||||
if event in ("call", "return") and frame.f_code.co_name in _watched:
|
||||
_stream.write(f"{event} {frame.f_code.co_filename}:{frame.f_lineno} {frame.f_code.co_name}\n")
|
||||
_stream.flush()
|
||||
elif event == "c_call" and getattr(arg, "__name__", "") in {"kill", "_exit", "abort"}:
|
||||
_stream.write(f"c_call {frame.f_code.co_filename}:{frame.f_lineno} {arg.__name__}\n")
|
||||
_stream.flush()
|
||||
|
||||
sys.setprofile(_trace_calls)
|
||||
@@ -1,160 +0,0 @@
|
||||
# CI-only diagnostic sampler for the July staged-updater handoff stall.
|
||||
#
|
||||
# Runs on the GitHub Actions Windows runner only; never on a user workstation.
|
||||
# Captures a bounded, secret-free snapshot of the update handoff state:
|
||||
# - processes whose executable or command line references the staged
|
||||
# hermes-setup.exe (or anything under the e2e hermes-home), plus their
|
||||
# full descendant tree: pid, ppid, exe path, sanitized command line,
|
||||
# CPU seconds, working set
|
||||
# - the update-in-progress marker file (pid + timestamp, non-secret)
|
||||
# - git HEAD + `git status --porcelain` file NAMES only (no diffs)
|
||||
# - update log filenames/sizes (no contents)
|
||||
#
|
||||
# Everything prints to stdout so the parent job log carries the snapshot.
|
||||
# Usage: powershell -File july-handoff-diagnostics.ps1 -WorkRoot <dir> [-Label handoff|timeout|finally]
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$WorkRoot,
|
||||
[string]$Label = "sample"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "SilentlyContinue"
|
||||
if ($env:GITHUB_ACTIONS -ne "true") { throw "handoff diagnostics are restricted to disposable CI runners" }
|
||||
|
||||
function Write-Section([string]$Name) {
|
||||
Write-Output ""
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] $Name ==="
|
||||
}
|
||||
|
||||
if (-not (Test-Path $WorkRoot)) {
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] WorkRoot not found: $WorkRoot ==="
|
||||
exit 0
|
||||
}
|
||||
$WorkRoot = (Resolve-Path $WorkRoot).Path
|
||||
|
||||
# Flags whose VALUE is redacted from command lines. Names only are kept.
|
||||
$SensitiveFlags = @("--token", "--key", "--api-key", "--password", "--secret", "-t", "--auth")
|
||||
|
||||
function Format-Cmdline([string]$ExePath, [string]$Cmdline) {
|
||||
# Tokenize on whitespace, redact the value that follows a sensitive flag,
|
||||
# and redact anything that looks like an embedded secret assignment.
|
||||
if ([string]::IsNullOrWhiteSpace($Cmdline)) { return "<no cmdline>" }
|
||||
$parts = @($Cmdline -split '\s+')
|
||||
$out = New-Object System.Collections.Generic.List[string]
|
||||
for ($i = 0; $i -lt $parts.Count; $i++) {
|
||||
$p = $parts[$i]
|
||||
if ($SensitiveFlags -contains $p.ToLower()) {
|
||||
$out.Add($p)
|
||||
if ($i + 1 -lt $parts.Count) { $out.Add("<redacted>"); $i++ }
|
||||
}
|
||||
elseif ($p -match '(?i)(token|secret|password|api[_-]?key)\s*=') {
|
||||
$out.Add(($p -replace '=.*$', '=<redacted>'))
|
||||
}
|
||||
else { $out.Add($p) }
|
||||
}
|
||||
return ($out -join " ")
|
||||
}
|
||||
|
||||
Write-Section "meta"
|
||||
Write-Output ("utc={0} workroot={1}" -f (Get-Date).ToUniversalTime().ToString("o"), $WorkRoot)
|
||||
|
||||
Write-Section "processes"
|
||||
$procs = @(Get-CimInstance Win32_Process -ErrorAction Continue)
|
||||
Write-Output "CIM process count=$($procs.Count)"
|
||||
# The updater PID is authoritative even when its argv uses unnormalized paths.
|
||||
$ownerPath = Join-Path $WorkRoot "hermes-home\.hermes-update-in-progress"
|
||||
$ownerPid = if (Test-Path $ownerPath) { (Get-Content $ownerPath -First 1).Trim() } else { "" }
|
||||
Get-Process -ErrorAction Continue | Where-Object { $_.Id -eq $ownerPid -or $_.Path -like '*hermes-desktop-gui-e2e*' } |
|
||||
Select-Object Id, ProcessName, Path, CPU, WorkingSet64 | Format-List | Out-String | Write-Output
|
||||
$rootPids = @{}
|
||||
foreach ($p in $procs) {
|
||||
$exe = [string]$p.ExecutablePath
|
||||
$cmd = [string]$p.CommandLine
|
||||
if ($exe) { $exe = [System.IO.Path]::GetFullPath($exe) }
|
||||
$ref = ($exe -like "$WorkRoot\*") -or ($p.ProcessId -eq $ownerPid)
|
||||
if ($ref) { $rootPids[[uint32]$p.ProcessId] = $true }
|
||||
}
|
||||
# Expand descendants transitively (both directions of interest: children of
|
||||
# the staged updater and children of its hermes update child).
|
||||
$changed = $true
|
||||
while ($changed) {
|
||||
$changed = $false
|
||||
foreach ($p in $procs) {
|
||||
$pp = [uint32]$p.ParentProcessId
|
||||
$cp = [uint32]$p.ProcessId
|
||||
if (-not $rootPids.ContainsKey($cp) -and $rootPids.ContainsKey($pp)) {
|
||||
$rootPids[$cp] = $true
|
||||
$changed = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($rootPids.Count -eq 0) {
|
||||
Write-Output "no hermes/staged-updater processes alive"
|
||||
}
|
||||
foreach ($p in $procs | Sort-Object ProcessId) {
|
||||
$cp = [uint32]$p.ProcessId
|
||||
if (-not $rootPids.ContainsKey($cp)) { continue }
|
||||
$cpu = "-"
|
||||
$ws = "-"
|
||||
try {
|
||||
$raw = Get-Process -Id $cp -ErrorAction SilentlyContinue
|
||||
if ($raw) {
|
||||
$cpu = [math]::Round($raw.TotalProcessorTime.TotalSeconds, 1)
|
||||
$ws = [math]::Round($raw.WorkingSet64 / 1MB, 1)
|
||||
}
|
||||
} catch {}
|
||||
$marker = ""
|
||||
if ($rootPids.ContainsKey([uint32]$p.ParentProcessId)) { $marker = "child-of=$($p.ParentProcessId)" }
|
||||
elseif ([uint32]$p.ParentProcessId -ne 0) { $marker = "root(parent=$($p.ParentProcessId))" }
|
||||
Write-Output ("pid={0} {1} cpu_s={2} ws_mb={3} exe={4}" -f $cp, $marker, $cpu, $ws, $p.ExecutablePath)
|
||||
Write-Output (" cmd: {0}" -f (Format-Cmdline $p.ExecutablePath $p.CommandLine))
|
||||
}
|
||||
|
||||
Write-Section "update-in-progress-marker"
|
||||
$markerPath = Join-Path $WorkRoot "hermes-home\hermes-agent\.hermes-update-in-progress"
|
||||
if (-not (Test-Path $markerPath)) {
|
||||
# Common alternate layout: marker lives directly under hermes-home.
|
||||
$alt = Join-Path $WorkRoot "hermes-home\.hermes-update-in-progress"
|
||||
if (Test-Path $alt) { $markerPath = $alt } else { $markerPath = $null }
|
||||
}
|
||||
if ($markerPath -and (Test-Path $markerPath)) {
|
||||
$fi = Get-Item $markerPath
|
||||
Write-Output ("marker={0} size={1} mtime={2}" -f $fi.FullName, $fi.Length, $fi.LastWriteTimeUtc.ToString("o"))
|
||||
# Contents are "pid\nstarted_at" — non-secret by contract.
|
||||
Write-Output ("marker-contents: {0}" -f ((Get-Content $markerPath -Raw) -replace "`r?`n", " / ").Trim())
|
||||
} else {
|
||||
Write-Output "no update-in-progress marker found"
|
||||
}
|
||||
|
||||
Write-Section "git"
|
||||
$repo = Join-Path $WorkRoot "hermes-home\hermes-agent"
|
||||
if (Test-Path (Join-Path $repo ".git")) {
|
||||
$head = & git -C $repo rev-parse HEAD 2>$null
|
||||
$branch = & git -C $repo rev-parse --abbrev-ref HEAD 2>$null
|
||||
Write-Output ("head={0} branch={1}" -f $head, $branch)
|
||||
# Names only: no diff content, no remote URLs, no stash payloads.
|
||||
$st = & git -C $repo status --porcelain 2>$null
|
||||
if ($st) { $st | ForEach-Object { Write-Output ("status: {0}" -f $_) } }
|
||||
else { Write-Output "status: clean" }
|
||||
$last = & git -C $repo log -1 --format="%h %ad %s" --date=short 2>$null
|
||||
Write-Output ("last-commit: {0}" -f $last)
|
||||
} else {
|
||||
Write-Output "no .git under $repo"
|
||||
}
|
||||
|
||||
Write-Section "logs"
|
||||
foreach ($dir in @(
|
||||
(Join-Path $WorkRoot "hermes-home\hermes-agent\logs"),
|
||||
(Join-Path $WorkRoot "hermes-home\logs"))) {
|
||||
if (Test-Path $dir) {
|
||||
Get-ChildItem $dir -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTimeUtc -Descending |
|
||||
Select-Object -First 15 |
|
||||
ForEach-Object {
|
||||
Write-Output ("{0} size={1} mtime={2}" -f $_.FullName, $_.Length, $_.LastWriteTimeUtc.ToString("o"))
|
||||
}
|
||||
}
|
||||
}
|
||||
Write-Output ""
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] done ==="
|
||||
exit 0
|
||||
@@ -6,6 +6,13 @@ function observeProcessClose(child) {
|
||||
closed = true
|
||||
resolve()
|
||||
}))
|
||||
// Windows descendants can inherit pipe handles and postpone 'close' after
|
||||
// the launch process exits. Release our handles, never kill descendants.
|
||||
const releasePipes = () => {
|
||||
for (const stream of child.stdio) stream?.destroy()
|
||||
}
|
||||
child.once('exit', releasePipes)
|
||||
if (child.exitCode !== null || child.signalCode !== null) releasePipes()
|
||||
return async function waitForClose(timeoutMs = 120_000) {
|
||||
if (closed) return
|
||||
let timer
|
||||
|
||||
@@ -806,12 +806,7 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) {
|
||||
$updateLog = Join-Path $HermesHome "logs\update.log"
|
||||
$updateLogPos = 0
|
||||
$deadline = (Get-Date).AddMinutes(35)
|
||||
$nextDiagnostic = Get-Date
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ($env:GITHUB_ACTIONS -eq "true" -and (Get-Date) -ge $nextDiagnostic) {
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "waiting"
|
||||
$nextDiagnostic = (Get-Date).AddMinutes(2)
|
||||
}
|
||||
if (Test-Path -LiteralPath $resultPath) { break }
|
||||
$head = ""
|
||||
try { $head = Get-InstalledHead } catch {}
|
||||
@@ -885,9 +880,7 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) {
|
||||
Write-Host "::endgroup::"
|
||||
Copy-Item $handoffLog (Join-Path $proof "desktop-update-handoff.log") -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($env:GITHUB_ACTIONS -eq "true") {
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "before-teardown"
|
||||
}
|
||||
|
||||
# Quit the relaunched app so job teardown is clean.
|
||||
Stop-HermesAppProcesses "post-update"
|
||||
}
|
||||
@@ -1018,14 +1011,6 @@ function Invoke-PhaseUpdate {
|
||||
}
|
||||
|
||||
function Invoke-CheckedPhaseUpdate {
|
||||
# Trace old Python stacks on CI without replacing updater behavior.
|
||||
$state = Read-State
|
||||
if ($env:GITHUB_ACTIONS -eq "true" -and $state.old -eq "7c1a029553d87c43ecff8a3821336bc95872213b" -and $InstallMethod -eq "desktop-installer@latest" -and $Route -eq "open-app-update") {
|
||||
$traceDir = Join-Path $AssetsDir "handoff-trace"
|
||||
$env:PYTHONPATH = if ($env:PYTHONPATH) { "$traceDir;$env:PYTHONPATH" } else { $traceDir }
|
||||
$env:HERMES_E2E_HANDOFF_TRACE = Join-Path $WorkRoot "proof\handoff-stacks"
|
||||
$env:PYTHONUNBUFFERED = "1"
|
||||
}
|
||||
Remove-Item -LiteralPath (Join-Path $WorkRoot "known-failure.json") -Force -ErrorAction SilentlyContinue
|
||||
# Only evidence produced by this update attempt can match an exception.
|
||||
foreach ($oldLog in @((Join-Path $WorkRoot "logs\update.log"), (Join-Path $HermesHome "logs\desktop.log"))) {
|
||||
|
||||
Reference in New Issue
Block a user