From 3859e5c5b2dd8536a012472a40a848c1560a2793 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 27 Sep 2026 00:19:54 -0700 Subject: [PATCH] test(e2e/windows): a stuck install.ps1 fails with its process table and Python stacks install.ps1 now runs under the same watchdog as `hermes update` (40 min), and the watchdog adds a py-spy stack for every Python process of the leg before it stops them. The workflow installs py-spy beside pywinpty. --- .github/workflows/install-e2e-windows-run.yml | 6 +++- tests/install/windows-e2e.ps1 | 35 ++++++++++++++----- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/.github/workflows/install-e2e-windows-run.yml b/.github/workflows/install-e2e-windows-run.yml index 200fd55710..022ebc4ab1 100644 --- a/.github/workflows/install-e2e-windows-run.yml +++ b/.github/workflows/install-e2e-windows-run.yml @@ -217,15 +217,19 @@ jobs: # version pyproject.toml requires. Install it with uv (the toolchain python ships # without pip) out of HERMES_RUNTIME_DIR, which is the tools directory itself and # is not on PATH for workflow steps; the import check proves it landed. + # py-spy lands beside it: the driver's hang watchdog dumps the stack of + # every Python process of a stuck install or update before stopping it. - name: Install the pseudoconsole runner's dependency shell: pwsh run: | $uvDir = Get-ChildItem $env:HERMES_RUNTIME_DIR -Filter 'uv-*' -Directory | Select-Object -First 1 if (-not $uvDir) { throw "uv not found under $env:HERMES_RUNTIME_DIR" } - & (Join-Path $uvDir.FullName 'uv.exe') pip install --system --python $env:HERMES_PYTHON --quiet "pywinpty==3.0.5" + & (Join-Path $uvDir.FullName 'uv.exe') pip install --system --python $env:HERMES_PYTHON --quiet "pywinpty==3.0.5" "py-spy==0.4.2" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python -c "import winpty; print('pty-run dependency ready')" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & (Join-Path (Split-Path $env:HERMES_PYTHON) 'py-spy.exe') --version + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } # One recording mechanism on every OS: the composite action verifies # ffmpeg from the PM toolchain (packages: ffmpeg above), starts the diff --git a/tests/install/windows-e2e.ps1 b/tests/install/windows-e2e.ps1 index a301b21550..f5f717fe0b 100644 --- a/tests/install/windows-e2e.ps1 +++ b/tests/install/windows-e2e.ps1 @@ -482,6 +482,8 @@ function Invoke-RefInstaller { # directory, so give it one that is not a project. $runDir = Join-Path $WorkRoot "install-cwd" New-Item -ItemType Directory -Path $runDir -Force | Out-Null + $hangLog = Join-Path $WorkRoot "logs\install-$Label-hang-processes.txt" + $watchdog = Start-HangWatchdog -Minutes $InstallDeadlineMinutes -EvidencePath $hangLog $prevEap = $ErrorActionPreference; $ErrorActionPreference = "Continue" Push-Location $runDir try { @@ -489,9 +491,14 @@ function Invoke-RefInstaller { $installExit = $LASTEXITCODE } finally { Pop-Location + Stop-HangWatchdog $watchdog } $ErrorActionPreference = $prevEap Write-LogGroup "install.ps1 ($Label) transcript" $log + if (Test-Path -LiteralPath $hangLog) { + Write-LogGroup "install.ps1 ($Label) hang evidence (process table, Python stacks)" $hangLog + throw "E2E ASSERTION FAILED: install.ps1 ($Label) was still running after $InstallDeadlineMinutes minutes; the process table and stacks above show where" + } Assert-True ($installExit -eq 0) "install.ps1 ($Label) exited 0" } @@ -527,17 +534,19 @@ function Invoke-HermesUpdate { } Write-LogGroup "hermes update transcript" $log if (Test-Path -LiteralPath $hangLog) { - Write-LogGroup "hermes update hang evidence (process table)" $hangLog + Write-LogGroup "hermes update hang evidence (process table, Python stacks)" $hangLog throw "E2E ASSERTION FAILED: hermes update was still running after $UpdateDeadlineMinutes minutes (its output pipe never closed); the process table above shows which process held it" } Assert-True ($updateExit -eq 0) "hermes update exited $updateExit (expected 0)" } -# `hermes update` normally finishes in under 25 minutes. Past this deadline -# the watchdog records every process (pid, parent, start time, command line) -# and stops this leg's processes, so a hang fails with evidence -- whether -# the updater itself is stuck or a detached child still holds its output -# pipe -- instead of being cancelled blind at the job cap. +# install.ps1 (with -IncludeDesktop) and `hermes update` normally finish in +# under 25 minutes. Past these deadlines the watchdog records every process +# (pid, parent, start time, command line) plus a py-spy stack of each Python +# process, then stops this leg's processes, so a hang fails with evidence -- +# whether the product itself is stuck or a detached child still holds its +# output pipe -- instead of being cancelled blind at the job cap. +$InstallDeadlineMinutes = 40 $UpdateDeadlineMinutes = 45 function Start-HangWatchdog([int]$Minutes, [string]$EvidencePath) { @@ -545,8 +554,10 @@ function Start-HangWatchdog([int]$Minutes, [string]$EvidencePath) { $exclude = @() if ($script:ChatMock) { $exclude += $script:ChatMock.Id } $homes = @($HermesHome, [System.IO.Path]::GetFullPath($HermesHome)) | Select-Object -Unique - Start-Job -ArgumentList $PID, $Minutes, $EvidencePath, $homes, $exclude -ScriptBlock { - param($driverPid, $minutes, $out, $homes, $exclude) + # py-spy is installed next to the driver's Python by the workflow. + $pyspy = Join-Path (Split-Path $DriverPython) 'py-spy.exe' + Start-Job -ArgumentList $PID, $Minutes, $EvidencePath, $homes, $exclude, $pyspy -ScriptBlock { + param($driverPid, $minutes, $out, $homes, $exclude, $pyspy) Start-Sleep -Seconds ($minutes * 60) $all = @(Get-CimInstance Win32_Process) $row = { param($p) '{0,7} <- {1,7} {2:yyyy-MM-ddTHH:mm:ss} {3}' -f $p.ProcessId, $p.ParentProcessId, $p.CreationDate, $(if ($p.CommandLine) { $p.CommandLine } else { $p.Name }) } @@ -567,6 +578,14 @@ function Start-HangWatchdog([int]$Minutes, [string]$EvidencePath) { $lines += @($leg | ForEach-Object { & $row $_ }) $lines += @('', '== every process, oldest first ==') $lines += @($all | Sort-Object CreationDate | ForEach-Object { & $row $_ }) + if (Test-Path -LiteralPath $pyspy) { + foreach ($p in (@($tree) + @($leg) | Where-Object { $_.Name -match '^(python|pythonw|hermes)' } | Sort-Object ProcessId -Unique)) { + $lines += @('', "== py-spy dump --pid $($p.ProcessId) ($($p.Name)) ==") + $lines += @(& $pyspy dump --pid $p.ProcessId --nonblocking 2>&1 | ForEach-Object { "$_" }) + } + } else { + $lines += @('', "(no Python stacks: $pyspy is missing)") + } Set-Content -LiteralPath $out -Value $lines -Encoding UTF8 foreach ($victim in (@($tree) + @($leg))) { Stop-Process -Id $victim.ProcessId -Force -ErrorAction SilentlyContinue