diff --git a/.github/workflows/install-e2e-windows-run.yml b/.github/workflows/install-e2e-windows-run.yml index b6bf850680..b2157da8b6 100644 --- a/.github/workflows/install-e2e-windows-run.yml +++ b/.github/workflows/install-e2e-windows-run.yml @@ -137,9 +137,19 @@ jobs: run: powershell -NoProfile -ExecutionPolicy Bypass -File tests\install\windows-e2e.ps1 -Phase install -InstallMethod "${{ inputs.install-method }}" -Route "${{ inputs.update-method }}" -InstallRef "${{ inputs.install-ref }}" -SetupExeUrl ${{ inputs.setup-exe-url }} - name: Update ${{ inputs.install-ref }} -> HEAD (${{ inputs.update-method }}) + id: update shell: powershell run: powershell -NoProfile -ExecutionPolicy Bypass -File tests\install\windows-e2e.ps1 -Phase update -InstallMethod "${{ inputs.install-method }}" -Route "${{ inputs.update-method }}" -InstallRef "${{ inputs.install-ref }}" -SetupExeUrl ${{ inputs.setup-exe-url }} + - name: Upload known-failure receipt + if: steps.update.outputs.known_failure != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: install-e2e-known-${{ steps.update.outputs.known_failure }}--${{ inputs.leg-id }} + path: ${{ env.HERMES_E2E_WORKROOT }}\known-failure.json + if-no-files-found: error + retention-days: 14 + - name: Stop screen recording if: always() uses: ./.github/actions/e2e-screen-record diff --git a/.github/workflows/install-e2e.yml b/.github/workflows/install-e2e.yml index 26154a4bef..a7a21fd511 100644 --- a/.github/workflows/install-e2e.yml +++ b/.github/workflows/install-e2e.yml @@ -57,6 +57,11 @@ on: required: false type: string default: '3' + install-ref: + description: 'Optional exact release tag for a focused reproduction; overrides tag-count.' + required: false + type: string + default: '' schedule: # Every 12 hours, off the hour to avoid the top-of-hour runner crunch. - cron: '20 7,19 * * *' @@ -102,10 +107,17 @@ jobs: # applies to each per-OS matrix separately; at 10 tags the largest # is windows at 180 (first over the cap at 15 tags = 270). TAG_COUNT: ${{ inputs.tag-count || 2 }} + INSTALL_REF: ${{ inputs.install-ref }} run: | set -euo pipefail [[ "$TAG_COUNT" =~ ^(10|[1-9])$ ]] || { echo "tag-count must be 1-10, got: $TAG_COUNT" >&2; exit 1; } - tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")" + if [ -n "$INSTALL_REF" ]; then + [[ "$INSTALL_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || { echo 'install-ref must be an exact release tag' >&2; exit 1; } + git rev-parse --verify "refs/tags/$INSTALL_REF^{commit}" >/dev/null + tags="$(jq -cn --arg ref "$INSTALL_REF" '[$ref]')" + else + tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")" + fi echo "Testing updates from: $tags" # Annotate each tag with what its own tree supports, so run # workflows can natively skip surfaces the starting version does @@ -241,7 +253,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs + sparse-checkout: | + scripts/sandbox/generate-e2e-matrix.mjs + tests/install/e2e-assets/known-failures.json sparse-checkout-cone-mode: false - env: GH_TOKEN: ${{ github.token }} diff --git a/scripts/sandbox/generate-e2e-matrix.mjs b/scripts/sandbox/generate-e2e-matrix.mjs index 01b306de3b..2db29ac3be 100644 --- a/scripts/sandbox/generate-e2e-matrix.mjs +++ b/scripts/sandbox/generate-e2e-matrix.mjs @@ -294,6 +294,9 @@ export function renderMarkdownPlan(envs, tags) { * @returns {string} */ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = new Map()) { + const knownRules = JSON.parse(fs.readFileSync(new URL('../../tests/install/e2e-assets/known-failures.json', import.meta.url), 'utf8')); + /** @type {Map} */ + const footnotes = new Map(); const LEG = /^(linux|windows|macos): (\S+) -> (\S+) \(([^)]+) -> HEAD\) \//; /** @type {Map} */ const desktopByTag = new Map(tagAnnotations.map((t) => [t.ref, t.desktop])); @@ -315,7 +318,7 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = // run workflow may have one inner job per driver arm; exactly one runs // and the others natively skip), so cells merge by significance: a real // outcome always beats a skip, and a bad outcome beats a good one. - const RANK = ['skip', 'TODO', 'pre-desktop', '✅', 'running', 'cancelled', '❌']; + const RANK = ['skip', 'TODO', 'pre-desktop', '✅', 'known', 'running', 'cancelled', '❌']; const SKIPS = ['skip', 'TODO', 'pre-desktop']; // Rendered success/failure cells carry artifact links after the glyph; // rank by the leading token or every such cell would rank as unknown (-1) @@ -351,7 +354,14 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = ? ` [📼](${runBase}/artifacts/${playerId}#zip=${encodeURIComponent(`${runBase}/artifacts/${logsId}`)}) [⬇️](${runBase}/artifacts/${logsId})` : ''; switch (job.conclusion) { - case 'success': return `✅${reel}`; + case 'success': { + // Only the classifier's uploaded receipt turns a successful job into + // a known-failure cell. Tag membership alone never suppresses a red. + const rule = knownRules.find((/** @type {any} */ r) => artifactById.has(`install-e2e-known-${r.id}--${legId2}`)); + if (!rule) return `✅${reel}`; + if (!footnotes.has(rule.id)) footnotes.set(rule.id, { number: footnotes.size + 1, rule }); + return `known [^${footnotes.get(rule.id)?.number}]${reel}`; + } case 'failure': return `❌${reel}`; case 'skipped': return skipLabel(m[2], m[3], tag); case 'cancelled': return 'cancelled'; @@ -369,10 +379,11 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = const passed = cells.filter((c) => c.startsWith('✅')).length; const failed = cells.filter((c) => c.startsWith('❌')).length; const skipped = cells.filter((c) => SKIPS.includes(c)).length; + const known = cells.filter((c) => c.startsWith('known ')).length; const lines = [ '### Install & Update E2E results', '', - `${passed} passed, ${failed} failed, ${skipped} skipped (TODO = declared, no driver arm yet; pre-desktop = the starting release predates apps/desktop), ${cells.length} legs total`, + `${passed} passed, ${failed} failed, ${known} known failures, ${skipped} skipped (TODO = declared, no driver arm yet; pre-desktop = the starting release predates apps/desktop), ${cells.length} legs total`, '', `| combination | ${tags.join(' | ')} |`, `|---|${tags.map(() => '---').join('|')}|`, @@ -381,6 +392,9 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = lines.push(`| \`${combo}\` | ${tags.map((t) => byTag.get(t) || '-').join(' | ')} |`); } lines.push(''); + for (const { number, rule } of footnotes.values()) { + lines.push(`[^${number}]: **${rule.title}.** ${rule.explanation} [Evidence](${rule.evidence}).`); + } return lines.join('\n'); } diff --git a/tests-js/install-known-failures.test.ts b/tests-js/install-known-failures.test.ts new file mode 100644 index 0000000000..4120f332eb --- /dev/null +++ b/tests-js/install-known-failures.test.ts @@ -0,0 +1,82 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import os from 'node:os' +import path from 'node:path' + +import { describe, expect, it } from 'vitest' + +const { matchKnownFailure, rules } = createRequire(import.meta.url)('../tests/install/e2e-assets/known-failures.cjs') +const classifier = path.resolve(import.meta.dirname, '../tests/install/e2e-assets/known-failures.cjs') +const lockedLog = [ + 'error: failed to remove file `C:/install/venv/Lib/site-packages/../../Scripts/hermes.exe`: Access is denied. (os error 5)', + 'File "C:/install/venv/Scripts/hermes.exe/__main__.py", line 10, in ', + "subprocess.CalledProcessError: Command '['uv', 'pip', 'install', '-e', '.', '--quiet']' returned non-zero exit status 2.", +].join('\n') +const base = { + platform: 'windows', phase: 'update', commit: 'a370ab8391ca5f8de7ebbc449f05cb0df36ade7c', + installMethod: 'installer-script', updateMethod: 'hermes-update', + error: 'E2E ASSERTION FAILED: hermes update exited 1 (expected 0)', logs: { update: lockedLog }, +} + +describe('known install failures', () => { + it('recognizes the released launcher self-lock, not generic access denied', () => { + expect(matchKnownFailure(base)?.id).toBe('windows-launcher-self-lock') + expect(matchKnownFailure({ ...base, logs: { update: lockedLog.replaceAll('hermes.exe', 'other.exe') } })).toBeNull() + expect(matchKnownFailure({ ...base, logs: { update: lockedLog.replace('(os error 5)', '(os error 32)') } })).toBeNull() + }) + + it.each([ + { platform: 'linux' }, { phase: 'install' }, { commit: 'v2026.3.12' }, + { commit: 'f'.repeat(40) }, { installMethod: 'desktop-installer@latest' }, + { updateMethod: 'installer-script' }, { error: 'E2E ASSERTION FAILED: update marker cleaned up' }, + { logs: {} }, + ])('rejects a different case or missing evidence: %j', change => { + expect(matchKnownFailure({ ...base, ...change })).toBeNull() + }) + + it('matches manual-only app updates only for the three proven July script cases', () => { + const sample = { + ...base, commit: '7c1a029553d87c43ecff8a3821336bc95872213b', + updateMethod: 'hermes-desktop-app-update', + error: 'E2E ASSERTION FAILED: app driven via captured hermes desktop spec; update completed', + logs: { desktop: '[hermes] [updates] no staged updater; surfacing manual `hermes update` for CLI install at C:/install\n[hermes] [updates] manual: hermes update\n' }, + } + expect(matchKnownFailure(sample)?.id).toBe('windows-july-manual-app-update') + expect(matchKnownFailure({ ...sample, installMethod: 'desktop-installer@latest' })).toBeNull() + expect(matchKnownFailure({ ...sample, error: 'onboarding timed out' })).toBeNull() + expect(matchKnownFailure({ ...sample, logs: { desktop: '[updates] manual: hermes update' } })).toBeNull() + }) + + it('CLI writes a receipt and exits zero only on a confirmed match', () => { + const root = mkdtempSync(path.join(os.tmpdir(), 'known-install-')) + try { + mkdirSync(path.join(root, 'logs')) + writeFileSync(path.join(root, 'shas.json'), '\uFEFF' + JSON.stringify({ old: base.commit, current: 'f'.repeat(40), old_ref: 'v2026.3.12' })) + writeFileSync(path.join(root, 'logs/update.log'), lockedLog) + const args = [classifier, root, base.installMethod, base.updateMethod, base.error] + expect(spawnSync(process.execPath, args).status).toBe(0) + expect(JSON.parse(readFileSync(path.join(root, 'known-failure.json'), 'utf8')).id).toBe('windows-launcher-self-lock') + writeFileSync(path.join(root, 'logs/update.log'), 'an unrelated failure') + expect(spawnSync(process.execPath, args).status).toBe(1) + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) +}) + +it('renders known receipts as footnotes, without suppressing a red job', async () => { + const modulePath = '../scripts/sandbox/generate-e2e-matrix.mjs' + const { renderMarkdownResults, legId } = await import(modulePath) + const name = 'windows: installer-script -> hermes-update (v2026.3.12 -> HEAD)' + const artifacts = new Map([[`install-e2e-known-${rules[0].id}--${legId(name)}`, 42]]) + const known = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'success' }], [], artifacts) + expect(known).toContain('0 passed, 0 failed, 1 known failures') + expect(known).toContain('known [^1]') + expect(known).toContain(`[^1]: **${rules[0].title}.**`) + const failed = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'failure' }], [], artifacts) + expect(failed).toContain('0 passed, 1 failed, 0 known failures') + expect(failed).not.toContain('known [^1]') + const passed = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'success' }]) + expect(passed).toContain('1 passed, 0 failed, 0 known failures') +}) diff --git a/tests/install/KNOWN_FAILURES.md b/tests/install/KNOWN_FAILURES.md index 0afcd39768..bb11181f36 100644 --- a/tests/install/KNOWN_FAILURES.md +++ b/tests/install/KNOWN_FAILURES.md @@ -1,6 +1,8 @@ # Confirmed historical upgrade limitations -These failures cannot be repaired by changing the update target: the failing code is already loaded from the starting release. This is an evidence register, not a skip list. The workflow still runs these legs and preserves their failing exit codes. A later failure must match the recorded cause before it receives this classification; the tag alone is not enough. +These failures cannot be repaired by changing the update target: the failing code is already loaded from the starting release. The workflow still executes each original update path. Only a match on the exact starting commit, method pair, failed assertion, and fresh log signatures produces a non-red known-failure receipt. Other errors still fail. The results table shows matched cases as `known [n]`, with the explanation and evidence in a footnote at the bottom. These cases are counted separately from passed upgrades. + +The machine-readable rules in `e2e-assets/known-failures.json` own the matcher and report footnote text. This document explains their historical evidence. Logs are rotated before each attempt so an earlier failure cannot classify a later one. ## Windows launcher self-lock diff --git a/tests/install/README.md b/tests/install/README.md index 2e22c4c989..cc4a41c730 100644 --- a/tests/install/README.md +++ b/tests/install/README.md @@ -63,7 +63,7 @@ A grey leg is normal. There are two causes: - The method pair is declared but cannot run: either no OS entry point exists for it (open-app-update after a plain script install registers nothing to open), or no driver arm exists yet. The gate in the run workflow lists the pairs that run. - The starting release predates the surface under test. Example: a release without `apps/desktop` has no window to launch. The tag annotation `tag_has_desktop` from the primary workflow marks these releases. -The result chart on the run summary shows each leg as passed, failed, or skipped. [Confirmed historical upgrade limitations](KNOWN_FAILURES.md) records failures that cannot be fixed in the update target, with exact release commits and CI evidence. These are not blanket skips: the original paths still run and failures remain visible. +The result chart on the run summary shows each leg as passed, failed, or skipped. [Confirmed historical upgrade limitations](KNOWN_FAILURES.md) records failures that cannot be fixed in the update target, with exact release commits and CI evidence. These are not blanket skips: the original paths still run. Exact signature matches are non-red, counted separately as known failures, and linked to footnotes at the bottom of the result chart. An unrelated error on the same tag still fails. ## Triggers and cost diff --git a/tests/install/e2e-assets/handoff-trace/sitecustomize.py b/tests/install/e2e-assets/handoff-trace/sitecustomize.py new file mode 100644 index 0000000000..87d9ffd808 --- /dev/null +++ b/tests/install/e2e-assets/handoff-trace/sitecustomize.py @@ -0,0 +1,15 @@ +"""CI-only Python stack snapshots for the opaque staged-updater hand-off. + +No command arguments, environment, or frame locals are recorded. The real +updater runs unchanged; stacks identify where its child is blocked. +""" +import os + +if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("HERMES_E2E_HANDOFF_TRACE"): + import faulthandler + from pathlib import Path + + _directory = Path(os.environ["HERMES_E2E_HANDOFF_TRACE"]) + _directory.mkdir(parents=True, exist_ok=True) + _stream = (_directory / f"python-stacks-{os.getpid()}.log").open("a", encoding="utf-8") + faulthandler.dump_traceback_later(90, repeat=True, file=_stream) diff --git a/tests/install/e2e-assets/july-handoff-diagnostics.ps1 b/tests/install/e2e-assets/july-handoff-diagnostics.ps1 new file mode 100644 index 0000000000..dc64fd9b65 --- /dev/null +++ b/tests/install/e2e-assets/july-handoff-diagnostics.ps1 @@ -0,0 +1,156 @@ +# CI-only diagnostic sampler for the July staged-updater handoff stall. +# +# Runs on the GitHub Actions Windows runner only; never on a user workstation. +# Captures a bounded, secret-free snapshot of the update handoff state: +# - processes whose executable or command line references the staged +# hermes-setup.exe (or anything under the e2e hermes-home), plus their +# full descendant tree: pid, ppid, exe path, sanitized command line, +# CPU seconds, working set +# - the update-in-progress marker file (pid + timestamp, non-secret) +# - git HEAD + `git status --porcelain` file NAMES only (no diffs) +# - update log filenames/sizes (no contents) +# +# Everything prints to stdout so the parent job log carries the snapshot. +# Usage: powershell -File july-handoff-diagnostics.ps1 -WorkRoot [-Label handoff|timeout|finally] + +param( + [Parameter(Mandatory = $true)][string]$WorkRoot, + [string]$Label = "sample" +) + +$ErrorActionPreference = "SilentlyContinue" +if ($env:GITHUB_ACTIONS -ne "true") { throw "handoff diagnostics are restricted to disposable CI runners" } + +function Write-Section([string]$Name) { + Write-Output "" + Write-Output "=== july-handoff-diagnostics [$Label] $Name ===" +} + +if (-not (Test-Path $WorkRoot)) { + Write-Output "=== july-handoff-diagnostics [$Label] WorkRoot not found: $WorkRoot ===" + exit 0 +} +$WorkRoot = (Resolve-Path $WorkRoot).Path + +# Flags whose VALUE is redacted from command lines. Names only are kept. +$SensitiveFlags = @("--token", "--key", "--api-key", "--password", "--secret", "-t", "--auth") + +function Format-Cmdline([string]$ExePath, [string]$Cmdline) { + # Tokenize on whitespace, redact the value that follows a sensitive flag, + # and redact anything that looks like an embedded secret assignment. + if ([string]::IsNullOrWhiteSpace($Cmdline)) { return "" } + $parts = @($Cmdline -split '\s+') + $out = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $parts.Count; $i++) { + $p = $parts[$i] + if ($SensitiveFlags -contains $p.ToLower()) { + $out.Add($p) + if ($i + 1 -lt $parts.Count) { $out.Add(""); $i++ } + } + elseif ($p -match '(?i)(token|secret|password|api[_-]?key)\s*=') { + $out.Add(($p -replace '=.*$', '=')) + } + else { $out.Add($p) } + } + return ($out -join " ") +} + +Write-Section "meta" +Write-Output ("utc={0} workroot={1}" -f (Get-Date).ToUniversalTime().ToString("o"), $WorkRoot) + +Write-Section "processes" +$procs = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue) +if ($procs.Count -eq 0) { + Write-Output "Get-CimInstance returned nothing" +} +$rootPids = @{} +foreach ($p in $procs) { + $exe = [string]$p.ExecutablePath + $cmd = [string]$p.CommandLine + $ref = ($exe -like "$WorkRoot\*") -or ($cmd -like "*$WorkRoot\*") + if ($ref) { $rootPids[[uint32]$p.ProcessId] = $true } +} +# Expand descendants transitively (both directions of interest: children of +# the staged updater and children of its hermes update child). +$changed = $true +while ($changed) { + $changed = $false + foreach ($p in $procs) { + $pp = [uint32]$p.ParentProcessId + $cp = [uint32]$p.ProcessId + if (-not $rootPids.ContainsKey($cp) -and $rootPids.ContainsKey($pp)) { + $rootPids[$cp] = $true + $changed = $true + } + } +} +if ($rootPids.Count -eq 0) { + Write-Output "no hermes/staged-updater processes alive" +} +foreach ($p in $procs | Sort-Object ProcessId) { + $cp = [uint32]$p.ProcessId + if (-not $rootPids.ContainsKey($cp)) { continue } + $cpu = "-" + $ws = "-" + try { + $raw = Get-Process -Id $cp -ErrorAction SilentlyContinue + if ($raw) { + $cpu = [math]::Round($raw.TotalProcessorTime.TotalSeconds, 1) + $ws = [math]::Round($raw.WorkingSet64 / 1MB, 1) + } + } catch {} + $marker = "" + if ($rootPids.ContainsKey([uint32]$p.ParentProcessId)) { $marker = "child-of=$($p.ParentProcessId)" } + elseif ([uint32]$p.ParentProcessId -ne 0) { $marker = "root(parent=$($p.ParentProcessId))" } + Write-Output ("pid={0} {1} cpu_s={2} ws_mb={3} exe={4}" -f $cp, $marker, $cpu, $ws, $p.ExecutablePath) + Write-Output (" cmd: {0}" -f (Format-Cmdline $p.ExecutablePath $p.CommandLine)) +} + +Write-Section "update-in-progress-marker" +$markerPath = Join-Path $WorkRoot "hermes-home\hermes-agent\.hermes-update-in-progress" +if (-not (Test-Path $markerPath)) { + # Common alternate layout: marker lives directly under hermes-home. + $alt = Join-Path $WorkRoot "hermes-home\.hermes-update-in-progress" + if (Test-Path $alt) { $markerPath = $alt } else { $markerPath = $null } +} +if ($markerPath -and (Test-Path $markerPath)) { + $fi = Get-Item $markerPath + Write-Output ("marker={0} size={1} mtime={2}" -f $fi.FullName, $fi.Length, $fi.LastWriteTimeUtc.ToString("o")) + # Contents are "pid\nstarted_at" — non-secret by contract. + Write-Output ("marker-contents: {0}" -f ((Get-Content $markerPath -Raw) -replace "`r?`n", " / ").Trim()) +} else { + Write-Output "no update-in-progress marker found" +} + +Write-Section "git" +$repo = Join-Path $WorkRoot "hermes-home\hermes-agent" +if (Test-Path (Join-Path $repo ".git")) { + $head = & git -C $repo rev-parse HEAD 2>$null + $branch = & git -C $repo rev-parse --abbrev-ref HEAD 2>$null + Write-Output ("head={0} branch={1}" -f $head, $branch) + # Names only: no diff content, no remote URLs, no stash payloads. + $st = & git -C $repo status --porcelain 2>$null + if ($st) { $st | ForEach-Object { Write-Output ("status: {0}" -f $_) } } + else { Write-Output "status: clean" } + $last = & git -C $repo log -1 --format="%h %ad %s" --date=short 2>$null + Write-Output ("last-commit: {0}" -f $last) +} else { + Write-Output "no .git under $repo" +} + +Write-Section "logs" +foreach ($dir in @( + (Join-Path $WorkRoot "hermes-home\hermes-agent\logs"), + (Join-Path $WorkRoot "hermes-home\logs"))) { + if (Test-Path $dir) { + Get-ChildItem $dir -File -ErrorAction SilentlyContinue | + Sort-Object LastWriteTimeUtc -Descending | + Select-Object -First 15 | + ForEach-Object { + Write-Output ("{0} size={1} mtime={2}" -f $_.FullName, $_.Length, $_.LastWriteTimeUtc.ToString("o")) + } + } +} +Write-Output "" +Write-Output "=== july-handoff-diagnostics [$Label] done ===" +exit 0 diff --git a/tests/install/e2e-assets/known-failures.cjs b/tests/install/e2e-assets/known-failures.cjs new file mode 100644 index 0000000000..d350a77af2 --- /dev/null +++ b/tests/install/e2e-assets/known-failures.cjs @@ -0,0 +1,54 @@ +const fs = require('node:fs') +const path = require('node:path') +const rules = require('./known-failures.json') + +function matchKnownFailure({ platform, phase, commit, installMethod, updateMethod, error, logs }) { + if (platform !== 'windows' || phase !== 'update' || !/^[0-9a-f]{40}$/.test(commit || '')) return null + return rules.find(rule => + rule.commits.includes(commit) && + rule.cases.some(([install, update]) => install === installMethod && update === updateMethod) && + rule.errors.some(pattern => new RegExp(pattern).test(error || '')) && + rule.signatures.every(pattern => new RegExp(pattern, 'i').test(logs[rule.log] || '')), + ) || null +} + +function readOptional(file) { + try { return fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '') } catch (error) { + if (error.code === 'ENOENT') return '' + throw error + } +} + +function classifyWorkRoot(root, installMethod, updateMethod, error) { + const state = JSON.parse(fs.readFileSync(path.join(root, 'shas.json'), 'utf8').replace(/^\uFEFF/, '')) + const rule = matchKnownFailure({ + platform: 'windows', phase: 'update', commit: state.old, installMethod, updateMethod, error, + logs: { + update: readOptional(path.join(root, 'logs', 'update.log')), + desktop: readOptional(path.join(root, 'hermes-home', 'logs', 'desktop.log')), + }, + }) + if (!rule) return null + return { + id: rule.id, title: rule.title, explanation: rule.explanation, evidence: rule.evidence, + commit: state.old, target: state.current, installRef: state.old_ref, + installMethod, updateMethod, error, + } +} + +module.exports = { matchKnownFailure, classifyWorkRoot, rules } + +if (require.main === module) { + const [root, install, update, error] = process.argv.slice(2) + try { + const receipt = classifyWorkRoot(root, install, update, error) + if (!receipt) process.exitCode = 1 + else { + fs.writeFileSync(path.join(root, 'known-failure.json'), JSON.stringify(receipt, null, 2) + '\n') + console.log(JSON.stringify(receipt)) + } + } catch (error) { + console.error(`known-failure classification failed: ${error.message}`) + process.exitCode = 2 + } +} diff --git a/tests/install/e2e-assets/known-failures.json b/tests/install/e2e-assets/known-failures.json new file mode 100644 index 0000000000..eaf226778a --- /dev/null +++ b/tests/install/e2e-assets/known-failures.json @@ -0,0 +1,24 @@ +[ + { + "id": "windows-launcher-self-lock", + "title": "Released Windows updater locks its own console launcher", + "commits": ["a370ab8391ca5f8de7ebbc449f05cb0df36ade7c", "86960cdbb0148145890e2ee90b4e157fa899f6e1"], + "cases": [["installer-script", "hermes-update"]], + "errors": ["^E2E ASSERTION FAILED: hermes update exited [1-9][0-9]* \\(expected 0\\)$"], + "log": "update", + "signatures": ["failed to remove file[^\\r\\n]*Scripts[/\\\\]hermes\\.exe[^\\r\\n]*Access is denied\\. \\(os error 5\\)", "hermes\\.exe[/\\\\]__main__\\.py", "CalledProcessError[^\\r\\n]*pip[^\\r\\n]*install[^\\r\\n]*returned non-zero exit status 2"], + "explanation": "The March/April Windows updater is already running from hermes.exe when uv tries to replace it. Windows refuses the locked launcher. The update target cannot change that loaded code; re-running the installer is a separate recovery route.", + "evidence": "https://github.com/ethernet8023/hermes-agent/actions/runs/34055462305/job/101546487700" + }, + { + "id": "windows-july-manual-app-update", + "title": "July script installs offer a manual update instead of an app hand-off", + "commits": ["7c1a029553d87c43ecff8a3821336bc95872213b"], + "cases": [["installer-script", "hermes-desktop-app-update"], ["installer-script+desktop", "hermes-desktop-app-update"], ["installer-script+desktop", "open-app-update"]], + "errors": ["^E2E ASSERTION FAILED: app driven via captured hermes desktop spec; update completed$", "^E2E ASSERTION FAILED: GUI driver clicked Update now and the app quit for hand-off$"], + "log": "desktop", + "signatures": ["\\[updates\\] no staged updater; surfacing manual `hermes update` for CLI install at", "\\[updates\\] manual: hermes update(?:\\r?\\n|$)"], + "explanation": "The July Windows app has no staged updater after a script install. Its Update button explicitly returns manual: hermes update without starting a hand-off. Desktop-installer installs are not covered by this exception.", + "evidence": "https://github.com/ethernet8023/hermes-agent/actions/runs/34055462305/job/101546487667" + } +] diff --git a/tests/install/windows-e2e.ps1 b/tests/install/windows-e2e.ps1 index 14390233fb..5c45b874de 100644 --- a/tests/install/windows-e2e.ps1 +++ b/tests/install/windows-e2e.ps1 @@ -797,7 +797,12 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) { $updateLog = Join-Path $HermesHome "logs\update.log" $updateLogPos = 0 $deadline = (Get-Date).AddMinutes(35) + $nextDiagnostic = Get-Date while ((Get-Date) -lt $deadline) { + if ($env:GITHUB_ACTIONS -eq "true" -and (Get-Date) -ge $nextDiagnostic) { + & powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "waiting" + $nextDiagnostic = (Get-Date).AddMinutes(2) + } if (Test-Path -LiteralPath $resultPath) { break } $head = "" try { $head = Get-InstalledHead } catch {} @@ -871,6 +876,9 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) { Write-Host "::endgroup::" Copy-Item $handoffLog (Join-Path $proof "desktop-update-handoff.log") -Force -ErrorAction SilentlyContinue } + if ($env:GITHUB_ACTIONS -eq "true") { + & powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "before-teardown" + } # Quit the relaunched app so job teardown is clean. Stop-HermesAppProcesses "post-update" } @@ -965,6 +973,40 @@ function Invoke-PhaseUpdate { Test-HermesRuns "post-update" } +function Invoke-CheckedPhaseUpdate { + # Trace old Python stacks on CI without replacing updater behavior. + $state = Read-State + if ($env:GITHUB_ACTIONS -eq "true" -and $state.old -eq "7c1a029553d87c43ecff8a3821336bc95872213b" -and $InstallMethod -eq "desktop-installer@latest" -and $Route -eq "open-app-update") { + $traceDir = Join-Path $AssetsDir "handoff-trace" + $env:PYTHONPATH = if ($env:PYTHONPATH) { "$traceDir;$env:PYTHONPATH" } else { $traceDir } + $env:HERMES_E2E_HANDOFF_TRACE = Join-Path $WorkRoot "proof\handoff-stacks" + $env:PYTHONUNBUFFERED = "1" + } + Remove-Item -LiteralPath (Join-Path $WorkRoot "known-failure.json") -Force -ErrorAction SilentlyContinue + # Only evidence produced by this update attempt can match an exception. + foreach ($oldLog in @((Join-Path $WorkRoot "logs\update.log"), (Join-Path $HermesHome "logs\desktop.log"))) { + if (Test-Path -LiteralPath $oldLog) { Move-Item -LiteralPath $oldLog -Destination "$oldLog.before-update" -Force } + } + try { + Invoke-PhaseUpdate + } catch { + $failure = $_ + $node = Get-ManagedNode + $classification = & $node (Join-Path $AssetsDir "known-failures.cjs") $WorkRoot $InstallMethod $Route $failure.Exception.Message + $classificationExit = $LASTEXITCODE + if ($classificationExit -ne 0) { throw $failure } + $receipt = ($classification | Out-String) | ConvertFrom-Json + Write-Host "KNOWN FAILURE [$($receipt.id)]: $($receipt.title)" + Write-Host " $($receipt.explanation)" + if ($env:GITHUB_OUTPUT) { + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "known_failure=$($receipt.id)" -Encoding UTF8 + } + if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding UTF8 -Value "Known historical failure: $($receipt.title). See the result chart footnote and uploaded known-failure.json." + } + } +} + # ---------------------------------------------------------------------------- # Dispatch # ---------------------------------------------------------------------------- @@ -982,11 +1024,11 @@ Set-GitRedirect switch ($Phase) { "stage" { Invoke-PhaseStage } "install" { Invoke-PhaseInstall } - "update" { Invoke-PhaseUpdate } + "update" { Invoke-CheckedPhaseUpdate } "all" { Invoke-PhaseStage Invoke-PhaseInstall - Invoke-PhaseUpdate + Invoke-CheckedPhaseUpdate } }