458 lines
20 KiB
JavaScript
458 lines
20 KiB
JavaScript
#!/usr/bin/env node
|
|
// @ts-check
|
|
/**
|
|
* Expand the install/update support matrix into concrete E2E combinations.
|
|
*
|
|
* One source of truth for every {os, install-method, update-method} pair a
|
|
* user could be on. This file only DECLARES and EXPANDS: it knows nothing
|
|
* about which combinations CI can drive. Every combination is dispatched to
|
|
* its OS's run workflow, and THAT workflow natively skips the method pairs
|
|
* its driver cannot run yet -- capability knowledge lives next to each
|
|
* driver (install-e2e-run.yml for linux, install-e2e-macos-run.yml,
|
|
* and install-e2e-windows-run.yml). Correctness here is enforced by the type
|
|
* unions below (checked via `tsc --checkJs`), not by runtime validation --
|
|
* anything the types can't catch is self-evident on the next CI run.
|
|
*
|
|
* Used by .github/workflows/install-e2e.yml, which runs it with the picked
|
|
* release tags (annotated at pick time with what each tag's tree ships):
|
|
*
|
|
* node scripts/sandbox/generate-e2e-matrix.mjs \
|
|
* --tags '[{"ref":"v2026.8.3","desktop":true}]'
|
|
*
|
|
* Prints JSON: { linux: {include:[...]}, windows: {include:[...]},
|
|
* macos: {include:[...]} } -- every entry is {name, install_method,
|
|
* update_method, install_ref, tag_has_desktop} (the tag annotation lets
|
|
* a run workflow natively skip desktop-surface legs from releases that
|
|
* predate the desktop app).
|
|
*/
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { parseArgs } from 'node:util';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
/**
|
|
* The closed method/version vocabulary. Workflows key off these exact
|
|
* strings, so they are types, not conventions.
|
|
*
|
|
* @typedef {'latest'} InstallerVersion
|
|
* The artifact published on the website right now -- Hermes-Setup.exe has
|
|
* no versioned archive yet. Widen this union when one exists.
|
|
* @typedef {'installer-script' | 'installer-script+desktop' | 'desktop-installer' | 'packaged-app'} InstallMethod
|
|
* installer-script is the platform's one-liner (curl | bash on
|
|
* linux/macos, irm | iex on windows); installer-script+desktop is the
|
|
* same one-liner with its desktop stage opted in (--include-desktop /
|
|
* -IncludeDesktop), which also builds the desktop app -- on windows it
|
|
* registers Start Menu / Desktop shortcuts too, on linux/macos it
|
|
* builds into the checkout without registering an OS entry point;
|
|
* packaged-app installs a signed bundle. Its native in-app update pair
|
|
* is declared separately, not crossed with source-checkout update methods.
|
|
* @typedef {InstallMethod | 'hermes-update' | 'open-app-update' | 'hermes-desktop-app-update'} UpdateMethod
|
|
* Every install method doubles as an update method (re-run it over the
|
|
* existing install), plus the updater CLI and the two app-update
|
|
* variants. The variants differ by launch surface: open-app-update
|
|
* starts the app from the OS entry point the install registered
|
|
* (Start Menu / Desktop shortcuts) -- today only the windows desktop
|
|
* installer's stage registers one (install.sh --include-desktop
|
|
* builds the app but registers nothing), so these legs pair with a
|
|
* desktop-installer install; hermes-desktop-app-update starts the app
|
|
* via `hermes desktop`, which every install method provides on every
|
|
* OS that ships the desktop app. Both then update through the app's
|
|
* own Update button.
|
|
* @typedef {'linux' | 'windows' | 'macos'} Os
|
|
*
|
|
* @typedef {{method: InstallMethod, versions?: InstallerVersion[]}} InstallEntry
|
|
* @typedef {{method: UpdateMethod, versions?: InstallerVersion[]}} UpdateEntry
|
|
* `versions` expands the entry into one combination per version
|
|
* ("desktop-installer@latest").
|
|
* @typedef {{install: InstallEntry[], update: UpdateEntry[], secondUpdate?: never[]}} OsSpec
|
|
* secondUpdate (install -> update -> update again) is a real future axis
|
|
* -- the updater that RESULTS from an update must itself update -- typed
|
|
* `never[]` so declaring one is a type error until a leg implements it.
|
|
*
|
|
* @typedef {{ref: string, desktop: boolean}} TagAnnotation
|
|
* A picked release tag plus what its own tree ships (annotated by
|
|
* pick-releases in install-e2e.yml).
|
|
*
|
|
* @typedef {{name: string, leg_id: string, install_method: string, update_method: string,
|
|
* install_ref: string, tag_has_desktop?: boolean}} MatrixEntry
|
|
*/
|
|
|
|
/**
|
|
* Artifact-safe key for one leg: the matrix name with every character
|
|
* outside [A-Za-z0-9._-] collapsed to '-'. Reconstructed by the results
|
|
* renderer from the parsed job name (same formula, same bytes), and used
|
|
* by every run workflow to name its logs/player artifacts, so the report
|
|
* job can map a concluded leg back to its artifacts without GitHub
|
|
* linking jobs to artifacts.
|
|
* @param {string} name
|
|
* @returns {string}
|
|
*/
|
|
export function legId(name) {
|
|
return name.replace(/[^A-Za-z0-9._-]+/g, '-');
|
|
}
|
|
|
|
/** One pinned package transition; source-release sampling does not apply.
|
|
* @param {'windows' | 'macos'} os
|
|
* @param {string} oldTag
|
|
* @returns {{include: MatrixEntry[]}}
|
|
*/
|
|
export function bundledMatrix(os, oldTag) {
|
|
const name = `${os}: packaged-app -> open-app-update (${oldTag} -> HEAD)`;
|
|
return { include: [{ name, leg_id: legId(name), install_method: 'packaged-app',
|
|
update_method: 'open-app-update', install_ref: oldTag, tag_has_desktop: true }] };
|
|
}
|
|
|
|
/** @type {Record<Os, OsSpec>} */
|
|
export const SPEC = {
|
|
windows: {
|
|
install: [
|
|
// irm https://hermes.nousresearch.com/install.ps1 | iex
|
|
{ method: 'installer-script' },
|
|
// The same one-liner with -IncludeDesktop: builds Hermes.exe AND
|
|
// registers Start Menu / Desktop shortcuts, so it is a second real
|
|
// path to a hand-launchable app.
|
|
{ method: 'installer-script+desktop' },
|
|
// Website Hermes-Setup.exe, clicked through the GUI.
|
|
{ method: 'desktop-installer', versions: ['latest'] },
|
|
],
|
|
update: [
|
|
{ method: 'installer-script' },
|
|
{ method: 'installer-script+desktop' },
|
|
// Run the bootstrap exe again over an existing install (--update flow).
|
|
{ method: 'desktop-installer', versions: ['latest'] },
|
|
{ method: 'hermes-update' },
|
|
// Settings -> About -> "Update now", app launched from the installed
|
|
// exe (the entry point the desktop installer created).
|
|
{ method: 'open-app-update' },
|
|
// Same button, app launched via `hermes desktop`.
|
|
{ method: 'hermes-desktop-app-update' },
|
|
],
|
|
},
|
|
macos: {
|
|
install: [
|
|
{ method: 'installer-script' },
|
|
{ method: 'installer-script+desktop' },
|
|
// The published Hermes-Setup.dmg from the website, mounted and run.
|
|
{ method: 'desktop-installer', versions: ['latest'] },
|
|
],
|
|
update: [
|
|
{ method: 'installer-script' },
|
|
{ method: 'installer-script+desktop' },
|
|
{ method: 'hermes-update' },
|
|
// install.sh --include-desktop builds the .app inside the checkout
|
|
// but registers no OS entry point, so open-app-update legs pair
|
|
// with a desktop-installer install (the published dmg).
|
|
{ method: 'open-app-update' },
|
|
{ method: 'hermes-desktop-app-update' },
|
|
],
|
|
},
|
|
linux: {
|
|
install: [
|
|
{ method: 'installer-script' },
|
|
{ method: 'installer-script+desktop' },
|
|
],
|
|
update: [
|
|
{ method: 'installer-script' },
|
|
{ method: 'installer-script+desktop' },
|
|
{ method: 'hermes-update' },
|
|
// No desktop installer and no packaged desktop artifact exist for
|
|
// linux, so there is no open-app-update; `hermes desktop` is always
|
|
// the source-mode path (build apps/desktop from the checkout, launch
|
|
// electron) and is the one app surface a linux install has.
|
|
{ method: 'hermes-desktop-app-update' },
|
|
],
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Expand one method entry into concrete ids ("desktop-installer@latest").
|
|
* @param {InstallEntry | UpdateEntry} entry
|
|
* @returns {string[]}
|
|
*/
|
|
export function expandMethod(entry) {
|
|
if (!entry.versions) return [entry.method];
|
|
return entry.versions.map((v) => `${entry.method}@${v}`);
|
|
}
|
|
|
|
/**
|
|
* Every {os, install, update} combination in SPEC.
|
|
* @param {Record<Os, OsSpec>} spec
|
|
* @returns {{os: Os, install: string, update: string}[]}
|
|
*/
|
|
export function generateEnvironments(spec) {
|
|
/** @type {{os: Os, install: string, update: string}[]} */
|
|
const envs = [];
|
|
for (const [os, osSpec] of /** @type {[Os, OsSpec][]} */ (Object.entries(spec))) {
|
|
for (const install of osSpec.install.flatMap(expandMethod)) {
|
|
for (const update of osSpec.update.flatMap(expandMethod)) {
|
|
envs.push({ os, install, update });
|
|
}
|
|
}
|
|
}
|
|
return envs;
|
|
}
|
|
|
|
/**
|
|
* Split the combinations into one matrix per OS.
|
|
*
|
|
* `tags` (the released versions we test updating FROM) is the OUTER axis:
|
|
* for each tag, for each combination, one dispatch that installs the tag
|
|
* and updates to HEAD. No capability filtering happens here -- every
|
|
* declared combination is dispatched, and the OS's run workflow natively
|
|
* skips what its driver cannot run yet. Entry names carry everything (os,
|
|
* method pair, tag transition) because slash-joined leg names are all the
|
|
* graph renders.
|
|
*
|
|
* @param {{os: Os, install: string, update: string}[]} envs
|
|
* @param {TagAnnotation[]} tags
|
|
* @returns {Record<Os, {include: MatrixEntry[]}>}
|
|
*/
|
|
export function buildMatrices(envs, tags) {
|
|
/** @type {Record<Os, {include: MatrixEntry[]}>} */
|
|
const byOs = { linux: { include: [] }, windows: { include: [] }, macos: { include: [] } };
|
|
for (const env of envs) {
|
|
for (const tag of tags) {
|
|
/** @type {MatrixEntry} */
|
|
const entry = {
|
|
name: `${env.os}: ${env.install} -> ${env.update} (${tag.ref} -> HEAD)`,
|
|
leg_id: legId(`${env.os}: ${env.install} -> ${env.update} (${tag.ref} -> HEAD)`),
|
|
install_method: env.install,
|
|
update_method: env.update,
|
|
install_ref: tag.ref,
|
|
// Every OS declares desktop-surface methods (both app-update
|
|
// variants at minimum), so every leg carries the annotation and
|
|
// its run workflow can natively skip pre-desktop tags.
|
|
tag_has_desktop: tag.desktop,
|
|
};
|
|
byOs[env.os].include.push(entry);
|
|
}
|
|
}
|
|
return byOs;
|
|
}
|
|
|
|
/**
|
|
* Does this method id need the starting tag to ship the desktop app?
|
|
* Shared by the plan chart (pre-desktop cells) and the results chart
|
|
* (labeling WHY a skipped leg skipped).
|
|
* @param {string} m
|
|
* @returns {boolean}
|
|
*/
|
|
export function methodNeedsDesktop(m) {
|
|
return m.startsWith('desktop-installer') || m === 'installer-script+desktop' ||
|
|
m === 'open-app-update' || m === 'hermes-desktop-app-update';
|
|
}
|
|
|
|
/**
|
|
* Render the plan as a markdown cross-table for $GITHUB_STEP_SUMMARY:
|
|
* one row per {os, install -> update} combination, one column per
|
|
* starting tag. Every cell is dispatched; whether it RUNS or greys out
|
|
* is the run workflow's call (capability lives there, not here), so the
|
|
* chart only distinguishes the one thing the plan itself knows:
|
|
* desktop-surface legs from tags that predate the desktop app.
|
|
*
|
|
* @param {{os: Os, install: string, update: string}[]} envs
|
|
* @param {TagAnnotation[]} tags
|
|
* @returns {string}
|
|
*/
|
|
export function renderMarkdownPlan(envs, tags) {
|
|
const lines = [
|
|
'### Install & Update E2E plan',
|
|
'',
|
|
`${envs.length} combinations x ${tags.length} starting tags = ${envs.length * tags.length} legs`,
|
|
'',
|
|
`| combination | ${tags.map((t) => t.ref).join(' | ')} |`,
|
|
`|---|${tags.map(() => '---').join('|')}|`,
|
|
];
|
|
for (const env of envs) {
|
|
const cells = tags.map((tag) => {
|
|
if (
|
|
!tag.desktop &&
|
|
(methodNeedsDesktop(env.install) || methodNeedsDesktop(env.update))
|
|
) {
|
|
return 'pre-desktop';
|
|
}
|
|
return '⏳ ';
|
|
});
|
|
lines.push(`| \`${env.os}: ${env.install} -> ${env.update}\` | ${cells.join(' | ')} |`);
|
|
}
|
|
return lines.join('\n');
|
|
}
|
|
|
|
/**
|
|
* Render the run's OUTCOME as the same cross-table, from the run's own job
|
|
* list (GitHub Actions API): one row per combination, one column per tag,
|
|
* each cell the leg's conclusion. Input is NDJSON {name, conclusion} lines
|
|
* -- what `gh api --paginate --jq '.jobs[] | {name, conclusion}'` emits --
|
|
* and legs are recognized by the exact name shape buildMatrices mints
|
|
* ("os: install -> update (tag -> HEAD) / ..."), so unrelated jobs
|
|
* (pick-releases, the report job itself) fall out naturally.
|
|
*
|
|
* @param {{name: string, conclusion: string | null}[]} jobs
|
|
* @param {TagAnnotation[]} [tagAnnotations] When given (the same --tags the
|
|
* plan got), skipped cells carry their REASON: `pre-desktop` when a
|
|
* desktop-surface method meets a tag that predates apps/desktop, `TODO`
|
|
* when the pair is declared but no driver arm runs it yet.
|
|
* @param {Map<string, number>} [artifactById] Artifact name -> id for this
|
|
* run (the report job's --artifacts). Legs that RAN (success or failure)
|
|
* get TWO links: 📼 to the run's single player artifact (playback.html,
|
|
* archive:false names artifacts after the FILE) with the leg's logs zip
|
|
* as a #zip= HASH param — the artifact URL 307s server-side and strips
|
|
* the query, the hash survives — and ⬇️ to the raw logs zip. Logs
|
|
* artifacts are `install-e2e-logs-<leg_id>` (windows) or
|
|
* `install-e2e-logs-<leg_id>-<sha>` (posix arms append the sha at
|
|
* upload) — matched by prefix. leg_id is rebuilt from the parsed job
|
|
* name with the same formula the generator mints (legId).
|
|
* @returns {string}
|
|
*/
|
|
export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = new Map()) {
|
|
const knownRules = JSON.parse(fs.readFileSync(new URL('../../tests/install/e2e-assets/known-failures.json', import.meta.url), 'utf8'));
|
|
/** @type {Map<string, {number: number, rule: any}>} */
|
|
const footnotes = new Map();
|
|
const LEG = /^(linux|windows|macos): (\S+) -> (\S+) \(([^)]+) -> HEAD\) \//;
|
|
/** @type {Map<string, boolean>} */
|
|
const desktopByTag = new Map(tagAnnotations.map((t) => [t.ref, t.desktop]));
|
|
/**
|
|
* @param {string} install @param {string} update @param {string} tag
|
|
* @returns {string}
|
|
*/
|
|
const skipLabel = (install, update, tag) => {
|
|
if (desktopByTag.size === 0) return 'skip';
|
|
if (
|
|
desktopByTag.get(tag) === false &&
|
|
(methodNeedsDesktop(install) || methodNeedsDesktop(update))
|
|
) {
|
|
return 'pre-desktop';
|
|
}
|
|
return 'TODO';
|
|
};
|
|
// A combination can surface as SEVERAL jobs with the same leg name (a
|
|
// run workflow may have one inner job per driver arm; exactly one runs
|
|
// and the others natively skip), so cells merge by significance: a real
|
|
// outcome always beats a skip, and a bad outcome beats a good one.
|
|
const RANK = ['skip', 'TODO', 'pre-desktop', '✅', 'known', 'running', 'cancelled', '❌'];
|
|
const SKIPS = ['skip', 'TODO', 'pre-desktop'];
|
|
// Rendered success/failure cells carry artifact links after the glyph;
|
|
// rank by the leading token or every such cell would rank as unknown (-1)
|
|
// and lose to any skip already in the map.
|
|
/** @param {string} cell */
|
|
const rankOf = (cell) => RANK.findIndex((t) => cell === t || cell.startsWith(`${t} `));
|
|
/** @type {Map<string, Map<string, string>>} */
|
|
const rows = new Map();
|
|
/** @type {string[]} */
|
|
const tags = [];
|
|
for (const job of jobs) {
|
|
const m = job.name.match(LEG);
|
|
if (!m) continue;
|
|
const combo = `${m[1]}: ${m[2]} -> ${m[3]}`;
|
|
const tag = m[4];
|
|
if (!tags.includes(tag)) tags.push(tag);
|
|
if (!rows.has(combo)) rows.set(combo, new Map());
|
|
const cell = (() => {
|
|
const legId2 = legId(`${m[1]}: ${m[2]} -> ${m[3]} (${m[4]} -> HEAD)`);
|
|
// Logs artifacts: `install-e2e-logs-<leg_id>` (windows) or with a
|
|
// trailing `-<sha>` (posix arms append it at upload). Match by prefix.
|
|
const logsName = [...artifactById.keys()].find((n) => n.startsWith(`install-e2e-logs-${legId2}`));
|
|
// Player artifacts: the single per-run leg-player upload; archive:
|
|
// false names it after the FILE (playback.html), ignoring `name:`.
|
|
const playerName = [...artifactById.keys()].find((n) => n === 'playback.html');
|
|
const playerId = playerName !== undefined ? artifactById.get(playerName) : undefined;
|
|
const logsId = logsName !== undefined ? artifactById.get(logsName) : undefined;
|
|
// Two links per ran leg: the player with the zip as a HASH param
|
|
// (GitHub's artifact URL 307s to /suites/... and strips the query —
|
|
// the hash survives client-side), and the raw zip download.
|
|
const runBase = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
|
|
const reel = (playerId !== undefined && logsId !== undefined)
|
|
? ` [📼](${runBase}/artifacts/${playerId}#zip=${encodeURIComponent(`${runBase}/artifacts/${logsId}`)}) [⬇️](${runBase}/artifacts/${logsId})`
|
|
: '';
|
|
switch (job.conclusion) {
|
|
case 'success': {
|
|
// Only the classifier's uploaded receipt turns a successful job into
|
|
// a known-failure cell. Tag membership alone never suppresses a red.
|
|
const rule = knownRules.find((/** @type {any} */ r) => artifactById.has(`install-e2e-known-${r.id}--${legId2}`));
|
|
if (!rule) return `✅${reel}`;
|
|
if (!footnotes.has(rule.id)) footnotes.set(rule.id, { number: footnotes.size + 1, rule });
|
|
return `known [^${footnotes.get(rule.id)?.number}]${reel}`;
|
|
}
|
|
case 'failure': return `❌${reel}`;
|
|
case 'skipped': return skipLabel(m[2], m[3], tag);
|
|
case 'cancelled': return 'cancelled';
|
|
default: return 'running';
|
|
}
|
|
})();
|
|
const byTag = /** @type {Map<string, string>} */ (rows.get(combo));
|
|
const prev = byTag.get(tag);
|
|
if (prev === undefined || rankOf(cell) > rankOf(prev)) {
|
|
byTag.set(tag, cell);
|
|
}
|
|
}
|
|
if (rows.size === 0) return '### Install & Update E2E results\n\n(no legs found in this run)\n';
|
|
const cells = [...rows.values()].flatMap((r) => [...r.values()]);
|
|
const passed = cells.filter((c) => c.startsWith('✅')).length;
|
|
const failed = cells.filter((c) => c.startsWith('❌')).length;
|
|
const skipped = cells.filter((c) => SKIPS.includes(c)).length;
|
|
const known = cells.filter((c) => c.startsWith('known ')).length;
|
|
const lines = [
|
|
'### Install & Update E2E results',
|
|
'',
|
|
`${passed} passed, ${failed} failed, ${known} known failures, ${skipped} skipped (TODO = declared, no driver arm yet; pre-desktop = the starting release predates apps/desktop), ${cells.length} legs total`,
|
|
'',
|
|
`| combination | ${tags.join(' | ')} |`,
|
|
`|---|${tags.map(() => '---').join('|')}|`,
|
|
];
|
|
for (const [combo, byTag] of rows) {
|
|
lines.push(`| \`${combo}\` | ${tags.map((t) => byTag.get(t) || '-').join(' | ')} |`);
|
|
}
|
|
lines.push('');
|
|
for (const { number, rule } of footnotes.values()) {
|
|
lines.push(`[^${number}]: **${rule.title}.** ${rule.explanation} [Evidence](${rule.evidence}).`);
|
|
}
|
|
return lines.join('\n');
|
|
}
|
|
|
|
/** @returns {Promise<string>} all of stdin */
|
|
function readStdin() {
|
|
return new Promise((resolve) => {
|
|
let data = '';
|
|
process.stdin.on('data', (c) => { data += c; });
|
|
process.stdin.on('end', () => resolve(data));
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
const { values } = parseArgs({
|
|
options: {
|
|
tags: { type: 'string', default: '[]' },
|
|
format: { type: 'string', default: 'json' },
|
|
artifacts: { type: 'string' },
|
|
},
|
|
});
|
|
if (values.format === 'results') {
|
|
const jobs = (await readStdin()).split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l));
|
|
const annotations = /** @type {TagAnnotation[]} */ (JSON.parse(values.tags));
|
|
/** @type {Map<string, number>} */
|
|
const artifactById = new Map();
|
|
if (values.artifacts) {
|
|
for (const line of (await fs.promises.readFile(values.artifacts, 'utf8')).split('\n')) {
|
|
if (!line.trim()) continue;
|
|
const a = JSON.parse(line);
|
|
if (typeof a.name === 'string' && typeof a.id === 'number') artifactById.set(a.name, a.id);
|
|
}
|
|
}
|
|
process.stdout.write(renderMarkdownResults(jobs, annotations, artifactById));
|
|
return;
|
|
}
|
|
const tags = /** @type {TagAnnotation[]} */ (JSON.parse(values.tags));
|
|
const envs = generateEnvironments(SPEC);
|
|
if (values.format === 'markdown') {
|
|
process.stdout.write(renderMarkdownPlan(envs, tags));
|
|
return;
|
|
}
|
|
const matrices = buildMatrices(envs, tags);
|
|
process.stdout.write(`${JSON.stringify(matrices, null, 2)}\n`);
|
|
}
|
|
|
|
if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
|
|
await main();
|
|
}
|