From 5afba1e8d94e1b0ca80016d488ec4f3d25ff729a Mon Sep 17 00:00:00 2001
From: teknium1 <127238744+teknium1@users.noreply.github.com>
Date: Sat, 26 Sep 2026 17:51:42 -0700
Subject: [PATCH] ci(install-e2e): keep one install-e2e-red issue in step with
 the scheduled matrix

A red scheduled run blocked nothing and told nobody. install-e2e-red.yml
runs after every scheduled "Install & Update E2E" run: red opens one issue
labelled install-e2e-red (or rewrites the open one's body in place) with the
red legs grouped by failure class and linked; the first green run closes
it. No per-run comment, never a second issue.

It is its own workflow_run workflow because install-e2e.yml is also called
by stable-release.yml with read-only permissions, and a nested job asking
for issues: write would fail that call at startup. workflow_dispatch with a
dry-run default previews the change for any run id.
---
 .github/workflows/install-e2e-red.yml     |  59 +++++++++++
 tests-js/install-e2e-red-tracker.test.mjs |  25 +++++
 tests/install/README.md                   |   5 +-
 tests/install/e2e-assets/red-tracker.mjs  | 118 ++++++++++++++++++++++
 4 files changed, 205 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/install-e2e-red.yml
 create mode 100644 tests-js/install-e2e-red-tracker.test.mjs
 create mode 100644 tests/install/e2e-assets/red-tracker.mjs

diff --git a/.github/workflows/install-e2e-red.yml b/.github/workflows/install-e2e-red.yml
new file mode 100644
index 0000000000..1a79cd0382
--- /dev/null
+++ b/.github/workflows/install-e2e-red.yml
@@ -0,0 +1,59 @@
+name: Install & Update E2E red tracker
+
+# Someone hears about a red scheduled matrix. After every scheduled
+# "Install & Update E2E" run, tests/install/e2e-assets/red-tracker.mjs keeps
+# ONE issue labelled install-e2e-red in step with it: a red run opens it (or
+# rewrites the open one's body in place with the current red legs grouped by
+# failure class), a green run closes it. No comment per red run, no second
+# issue.
+#
+# A separate workflow on purpose: install-e2e.yml is also a reusable workflow
+# called by stable-release.yml with contents/actions read only, and a nested
+# job asking for issues: write would fail that call at startup.
+#
+# Dispatch it by hand to preview (dry-run, the default) or replay a run.
+
+on:
+  workflow_run:
+    workflows: ['Install & Update E2E']
+    types: [completed]
+  workflow_dispatch:
+    inputs:
+      run-id:
+        description: 'Install & Update E2E run id to report on.'
+        required: true
+        type: string
+      dry-run:
+        description: 'Print the planned issue change instead of making it.'
+        required: false
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: install-e2e-red-tracker
+  cancel-in-progress: false
+
+jobs:
+  track:
+    name: Update the install-e2e-red tracker
+    if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.event == 'schedule'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+      issues: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          sparse-checkout: tests/install/e2e-assets/red-tracker.mjs
+          sparse-checkout-cone-mode: false
+      - name: Open, update or close the tracker
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_ID: ${{ inputs.run-id || github.event.workflow_run.id }}
+          DRY_RUN: ${{ inputs.dry-run == true && '--dry-run' || '' }}
+        run: node tests/install/e2e-assets/red-tracker.mjs --run-id "$RUN_ID" $DRY_RUN
diff --git a/tests-js/install-e2e-red-tracker.test.mjs b/tests-js/install-e2e-red-tracker.test.mjs
new file mode 100644
index 0000000000..e09ab8287c
--- /dev/null
+++ b/tests-js/install-e2e-red-tracker.test.mjs
@@ -0,0 +1,25 @@
+import { expect, test } from 'vitest'
+import { planTracker } from '../tests/install/e2e-assets/red-tracker.mjs'
+
+const run = (conclusion) => ({ conclusion, html_url: 'https://example.test/runs/1', head_sha: 'abcdef0123456789', created_at: '2026-09-26T19:24:13Z' })
+const leg = (name, conclusion, step) => ({ name, conclusion, html_url: `https://example.test/job/${name.length}`, steps: step ? [{ name: 'checkout', conclusion: 'success' }, { name: step, conclusion }] : [] })
+
+test('one tracker: a red run opens it, the next red run rewrites it, a green run closes it', () => {
+  const jobs = [
+    leg('windows: installer-script -> hermes-update (v2026.9.24 -> HEAD) / e2e', 'cancelled', 'Install v2026.9.24 (installer-script)'),
+    leg('windows: installer-script+desktop -> hermes-update (HEAD -> NEXT) / e2e', 'cancelled', 'Install HEAD (installer-script+desktop)'),
+    leg('linux: installer-script -> hermes-update (HEAD -> NEXT) / install & update', 'success'),
+    leg('linux: installer-script -> installer-script (HEAD -> NEXT) / install & update', 'skipped'),
+  ]
+
+  const opened = planTracker(run('cancelled'), jobs, null)
+  expect(opened.action).toBe('open')
+  expect(opened.title).toBe('Install & Update E2E matrix is red (2 legs)')
+  // Both Windows legs hung in the same driver step: one class, not two.
+  expect(opened.body).toContain('**2 × cancelled in step `Install <ref>`**')
+  expect(opened.body).not.toContain('linux:')
+
+  expect(planTracker(run('failure'), jobs, { number: 7 }).action).toBe('update')
+  expect(planTracker(run('success'), [jobs[2]], { number: 7 }).action).toBe('close')
+  expect(planTracker(run('success'), [jobs[2]], null).action).toBe('none')
+})
diff --git a/tests/install/README.md b/tests/install/README.md
index af5f967ea8..d0d261ff2a 100644
--- a/tests/install/README.md
+++ b/tests/install/README.md
@@ -160,9 +160,10 @@ The result chart on the run summary shows each leg as passed, failed, or skipped
 
 ## Triggers and cost
 
-The matrix does not run on pull requests. One leg installs real toolchains and takes more than 10 minutes. The triggers are:
+The full matrix does not run on pull requests. One leg installs real toolchains and takes more than 10 minutes. The triggers are:
 
-- A schedule, every 12 hours. This finds upstream drift.
+- A pull request that touches the install/update surface (the `paths:` list in `install-e2e.yml`). It runs only the `pr` route: `installer-script -> hermes-update` on Linux (newest release -> PR, and PR -> NEXT), Windows (PR -> NEXT) and macOS (newest release -> PR). The four legs run in parallel, so the wall clock is about one leg.
+- A schedule, every 12 hours. This finds upstream drift. When a scheduled run goes red, `install-e2e-red.yml` opens one issue labelled `install-e2e-red`, rewrites its body in place on every later red run, and closes it on the first green one.
 - A matching release tag push.
 - A reusable workflow call from the stable release gate.
 - Manual dispatch. You can select the route and the tag count:
diff --git a/tests/install/e2e-assets/red-tracker.mjs b/tests/install/e2e-assets/red-tracker.mjs
new file mode 100644
index 0000000000..ab9db67d24
--- /dev/null
+++ b/tests/install/e2e-assets/red-tracker.mjs
@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * Keep ONE tracking issue in step with the scheduled install/update matrix.
+ *
+ * Run by .github/workflows/install-e2e-red.yml after every scheduled
+ * "Install & Update E2E" run. A red run (any leg failed, was cancelled, or timed
+ * out) opens the issue labelled `install-e2e-red`, or rewrites the body of the
+ * one already open, so the issue always names the CURRENT red legs and the run
+ * that produced them. A green run closes it with a pointer to that run. Never a
+ * second issue and never a comment per red run: subscribers hear about the open
+ * and the close, and the body is the live state in between.
+ *
+ *   node tests/install/e2e-assets/red-tracker.mjs --run-id <id> [--dry-run]
+ *
+ * Needs GH_TOKEN (issues: write, actions: read) and GITHUB_REPOSITORY.
+ */
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseArgs } from 'node:util';
+
+export const LABEL = 'install-e2e-red';
+const RED = new Set(['failure', 'cancelled', 'timed_out', 'startup_failure']);
+const MAX_ROWS = 60;
+
+/**
+ * Decide what the tracker does for one completed run. Pure, so the policy is
+ * testable without GitHub.
+ *
+ * @param {{conclusion: string | null, html_url: string, head_sha: string, created_at: string}} run
+ * @param {{name: string, conclusion: string | null, html_url: string, steps?: {name: string, conclusion: string | null}[]}[]} jobs
+ * @param {{number: number} | null} openIssue
+ * @returns {{action: 'open' | 'update' | 'close' | 'none', title?: string, body?: string}}
+ */
+export function planTracker(run, jobs, openIssue) {
+  const red = jobs.filter((j) => RED.has(String(j.conclusion)));
+  const runRed = RED.has(String(run.conclusion)) || red.length > 0;
+  if (!runRed) {
+    if (run.conclusion === 'success' && openIssue) {
+      return { action: 'close', body: `The scheduled matrix is green again: ${run.html_url} (\`${run.head_sha.slice(0, 10)}\`). Closing; the next red run reopens a fresh tracker.` };
+    }
+    return { action: 'none' };
+  }
+  // Group by (conclusion, step) so 34 legs cancelled in the same step read as
+  // one failure class, not 34 unrelated reds.
+  /** @type {Map<string, typeof red>} */
+  const classes = new Map();
+  for (const job of red) {
+    const step = (job.steps || []).find((s) => RED.has(String(s.conclusion)))?.name || '(no step reported)';
+    // "Install v2026.9.24 (installer-script)" and "Install HEAD (installer-script+desktop)"
+    // are the same step of the driver; the leg name already carries ref and method.
+    const phase = step.replace(/\bv\d{4}\.\d+\.\d+(?:\.\d+)?\b|\bHEAD\b|\bNEXT\b/g, '<ref>').replace(/\s*\([^)]*\)\s*$/, '');
+    const key = `${job.conclusion} in step \`${phase}\``;
+    if (!classes.has(key)) classes.set(key, []);
+    classes.get(key)?.push(job);
+  }
+  const lines = [
+    `The scheduled **Install & Update E2E** run ${run.html_url} (\`${run.head_sha.slice(0, 10)}\`, ${run.created_at}) finished **${run.conclusion}** with ${red.length} red leg(s).`,
+    '',
+    'This issue is rewritten in place by `.github/workflows/install-e2e-red.yml` after every scheduled run and closed by the first green one. Root-cause each class below; a historical-release limitation that cannot be fixed goes into `tests/install/e2e-assets/known-failures.json`.',
+    '',
+    '### Red legs by failure class',
+  ];
+  let rows = 0;
+  for (const [key, members] of [...classes].sort((a, b) => b[1].length - a[1].length)) {
+    lines.push('', `**${members.length} × ${key}**`, '');
+    for (const job of members) {
+      if (rows >= MAX_ROWS) break;
+      lines.push(`- [${job.name}](${job.html_url})`);
+      rows += 1;
+    }
+  }
+  if (red.length > rows) lines.push('', `…and ${red.length - rows} more; see the run's jobs list.`);
+  const title = `Install & Update E2E matrix is red (${red.length} leg${red.length === 1 ? '' : 's'})`;
+  return { action: openIssue ? 'update' : 'open', title, body: lines.join('\n') };
+}
+
+/** @param {string[]} args @returns {string} */
+function gh(args) {
+  return execFileSync('gh', args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+}
+
+async function main() {
+  const { values } = parseArgs({ options: { 'run-id': { type: 'string' }, 'dry-run': { type: 'boolean', default: false } } });
+  const repo = process.env.GITHUB_REPOSITORY;
+  const runId = values['run-id'];
+  if (!repo || !runId || !/^\d+$/.test(runId)) throw new Error('need GITHUB_REPOSITORY and a numeric --run-id');
+  const run = JSON.parse(gh(['api', `repos/${repo}/actions/runs/${runId}`]));
+  const jobs = gh(['api', '--paginate', `repos/${repo}/actions/runs/${runId}/jobs?per_page=100`,
+    '--jq', '.jobs[] | {name, conclusion, html_url, steps: [.steps[]? | {name, conclusion}]}'])
+    .split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l));
+  const open = JSON.parse(gh(['api', `repos/${repo}/issues?labels=${LABEL}&state=open&per_page=5`]))
+    .filter((/** @type {any} */ i) => !i.pull_request);
+  const openIssue = open.length ? { number: open[0].number } : null;
+  const plan = planTracker(run, jobs, openIssue);
+  console.log(`run ${runId}: conclusion=${run.conclusion} red-legs=${jobs.filter((j) => RED.has(String(j.conclusion))).length} open-tracker=${openIssue ? `#${openIssue.number}` : 'none'} -> ${plan.action}`);
+  if (values['dry-run'] || plan.action === 'none') {
+    if (plan.body) console.log(`\n--- ${plan.title || 'comment'} ---\n${plan.body}`);
+    return;
+  }
+  if (plan.action === 'open') {
+    gh(['label', 'create', LABEL, '--repo', repo, '--force', '--color', 'B60205',
+      '--description', 'The scheduled install/update E2E matrix is red (managed by install-e2e-red.yml)']);
+    const url = gh(['issue', 'create', '--repo', repo, '--label', LABEL, '--title', String(plan.title), '--body', String(plan.body)]);
+    console.log(`opened ${url.trim()}`);
+  } else if (plan.action === 'update' && openIssue) {
+    gh(['api', '-X', 'PATCH', `repos/${repo}/issues/${openIssue.number}`, '-f', `title=${plan.title}`, '-f', `body=${plan.body}`]);
+    console.log(`updated #${openIssue.number} in place`);
+  } else if (plan.action === 'close' && openIssue) {
+    gh(['issue', 'comment', String(openIssue.number), '--repo', repo, '--body', String(plan.body)]);
+    gh(['issue', 'close', String(openIssue.number), '--repo', repo]);
+    console.log(`closed #${openIssue.number}`);
+  }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  await main();
+}