hermes-agent/.github/workflows/install-e2e.yml

name: Install & Update E2E

# Can a user on a released version get to this commit?
#
# The support matrix -- every {os, install-method, update-method} combination
# a user could be on -- lives in scripts/sandbox/generate-e2e-matrix.mjs.
# generate-matrix expands it against the picked release tags into one leg
# per {combination, tag}, split into one matrix job per OS:
#
#   Matrix: linux     the real curl|bash install one-liner, isolated by a
#                     git URL redirect to a local bare clone
#                     (install-e2e-run.yml)
#   Matrix: windows   the real desktop user flow: website Hermes-Setup.exe
#                     clicked by AutoHotkey, update via the app, Playwright
#                     clicking "Update now" (install-e2e-windows-run.yml)
#   Matrix: macos     script installs on the shared OS-agnostic driver,
#                     plus the real desktop user flow: website
#                     Hermes-Setup.dmg mounted and run, updates via the
#                     app under Playwright (install-e2e-macos-run.yml)
#
# Every combination is dispatched to its OS's run workflow; the run
# workflow natively skips (grey) what its driver cannot run yet -- an
# unimplemented method pair, or a starting tag that predates the surface
# under test (pick-releases annotates each tag with what its tree ships,
# e.g. whether the desktop app exists yet). Capability knowledge lives
# next to each driver, never here and never in the generator: declaring a
# method is a spec edit, implementing one is flipping the run workflow's
# gate.
#
# The starting versions are chosen at runtime from the repo's release tags
# (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread
# between. A hardcoded list would stop covering the newest release the day
# after it ships, and would pin an "oldest" that nobody still runs.
#
# Triggers:
#   * every 12 hours, so upstream drift (a new uv, a Node bump, a PyPI change)
#     surfaces on a schedule rather than in someone's review cycle;
#   * manually, where you can pick the route (a preset or specific legs) and how many
#     releases to sample;
#   * pull_request, path-filtered to the install/update surface, runs ONLY the
#     `pr` route (generate-e2e-matrix.mjs PR_SUBSET): installer-script ->
#     hermes-update on linux (newest release -> PR, and PR -> NEXT), windows
#     (PR -> NEXT) and macos (newest release -> PR), sampling just the newest tag.
#     Four legs in parallel keep a PR's wall clock near one leg (~15 min);
#     the full matrix multiplies that, so it stays on the clock and releases.
#     A red scheduled run is reported by install-e2e-red.yml.

on:
  workflow_dispatch:
    inputs:
      route:
        description: 'Which legs to run: all = every OS; both/update/installer = the linux legs; windows-desktop / macos-desktop = that OS; bundled / windows-bundled / macos-bundled = the signed-bundle legs. Anything else selects legs by name (a leg or job name, or a fragment of one).'
        required: false
        type: string
        default: all
      windows-bundle-manifest:
        description: 'HTTPS manifest pinning signed Windows OLD/NEW msixbundles; NEW must match this ref.'
        type: string
        default: ''
      macos-bundle-manifest:
        description: 'HTTPS manifest pinning signed macOS OLD/NEW app ZIPs; NEW must match this ref.'
        type: string
        default: ''
      tag-count:
        description: 'How many release tags to sample (newest, oldest, and a spread between).'
        required: false
        type: string
        default: '3'
      install-ref:
        description: 'Optional exact release tag for a focused reproduction; overrides tag-count.'
        required: false
        type: string
        default: ''
  workflow_call:
    # Reusable entry for the stable-release orchestrator. Same input names as
    # workflow_dispatch; on a release run the caller dispatches on the exact
    # candidate tag, so github.sha is the candidate commit throughout.
    #
    # route: which combinations to run (same choices as dispatch). The
    # bundled routes require the matching bundle manifests, exactly like a
    # manual dispatch — a source (non-bundle) release run passes route=all
    # with empty manifests and gets the source install/update matrix only.
    inputs:
      release:
        description: 'Stable-release candidate run (informational; jobs behave as route=all with empty manifests).'
        required: false
        type: boolean
        default: false
      route:
        description: 'Which legs to run: all = every OS; both/update/installer = the linux legs; windows-desktop / macos-desktop = that OS; bundled / windows-bundled / macos-bundled = the signed-bundle legs. Anything else selects legs by name (a leg or job name, or a fragment of one).'
        required: false
        type: string
        default: all
      windows-bundle-manifest:
        description: 'HTTPS manifest pinning signed Windows OLD/NEW msixbundles; NEW must match this ref.'
        required: false
        type: string
        default: ''
      macos-bundle-manifest:
        description: 'HTTPS manifest pinning signed macOS OLD/NEW app ZIPs; NEW must match this ref.'
        required: false
        type: string
        default: ''
      tag-count:
        description: 'How many release tags to sample (newest, oldest, and a spread between).'
        required: false
        type: string
        default: '3'
      install-ref:
        description: 'Optional exact release tag for a focused reproduction; overrides tag-count.'
        required: false
        type: string
        default: ''
      exclude-ref:
        description: 'Exact release tag to EXCLUDE from the sampled OLD baselines (e.g. the candidate itself — testing an update from it would be a no-change test).'
        required: false
        type: string
        default: ''
  schedule:
    # Every 12 hours, off the hour to avoid the top-of-hour runner crunch.
    - cron: '20 7,19 * * *'
  pull_request:
    # The install/update surface: derived from 60 days of commits whose subject
    # names update/install/pm/bootstrap/relaunch. Keep it to code a real
    # install or `hermes update` executes; hermes_cli/main.py and friends are
    # deliberately absent (they change daily and are covered by e2e-upgrade).
    paths:
      - 'scripts/install*'
      - 'scripts/desktop-update/**'
      - 'setup-hermes.*'
      - 'hermes_bootstrap.py'
      - 'pm/**'
      - 'hermes_cli/update*'
      - 'hermes_cli/*update*'
      - 'hermes_cli/pm*'
      - 'hermes_cli/*install*'
      - 'hermes_cli/_early_recovery.py'
      - 'hermes_cli/_launchers.py'
      - 'hermes_cli/managed_uv.py'
      - 'hermes_cli/venv_sync.py'
      - 'hermes_cli/relaunch.py'
      - 'hermes_cli/gateway.py'
      - 'gateway/status.py'
      - 'apps/bootstrap-installer/**'
      - 'apps/desktop/electron/*update*'
      - 'apps/desktop/electron/updater/**'
      - 'pyproject.toml'
      - 'uv.lock'
      - 'package.json'
      - 'package-lock.json'
      - 'tests/install/**'
      - 'scripts/sandbox/generate-e2e-matrix.mjs'
      - 'scripts/sandbox/pick-release-tags.sh'
      - 'scripts/ci/setup_toolchain.py'
      - '.github/actions/setup-pm/**'
      - '.github/workflows/install-e2e*.yml'

permissions:
  contents: read

# A workflow_call run (stable release) is never cancelled: its group uses
# github.run_id so a parent rerun cannot kill this child mid-flight. The
# scheduled/manual runs keep ref-scoped collapse.
concurrency:
  group: install-e2e-${{ inputs.release == true && github.run_id || github.ref }}
  cancel-in-progress: ${{ inputs.release != true }}

jobs:
  pick-releases:
    name: Pick release tags
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      tags: ${{ steps.pick.outputs.tags }}
    steps:
      # This job only reads tag names and trees, so take the cheap
      # checkout: no blobs (filter), no other files (sparse), but DO fetch
      # tags -- they are the whole input, and the default shallow checkout
      # has none.
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          filter: blob:none
          fetch-tags: true
          sparse-checkout: |
            scripts/sandbox/pick-release-tags.sh
            scripts/releases/pick_tags.py
          sparse-checkout-cone-mode: false
      - id: pick
        env:
          # Inputs never touch shell syntax directly: TAG_COUNT arrives via
          # the environment and is validated decimal-only (bash arithmetic
          # reads a leading zero as octal). GitHub's 256-job cap applies to
          # each per-OS matrix separately; at 10 tags the largest is windows
          # at 180 (first over the cap at 15 tags = 270).
          #
          # EXCLUDE_REF removes one tag from the sampled OLD baselines: on a
          # stable release the candidate tag itself is the newest release
          # tag, and sampling it as an OLD version would produce a no-change
          # update leg — the candidate must never update from itself.
          TAG_COUNT: ${{ github.event_name == 'pull_request' && '1' || inputs.tag-count || 2 }}
          INSTALL_REF: ${{ inputs.install-ref }}
          EXCLUDE_REF: ${{ inputs.exclude-ref }}
          RELEASE_MODE: ${{ inputs.release }}
        run: |
          set -euo pipefail
          [[ "$TAG_COUNT" =~ ^(10|[1-9])$ ]] || { echo "tag-count must be 1-10, got: $TAG_COUNT" >&2; exit 1; }
          if [ -n "$INSTALL_REF" ]; then
            [[ "$INSTALL_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || { echo 'install-ref must be an exact release tag' >&2; exit 1; }
            [ "$INSTALL_REF" != "$EXCLUDE_REF" ] || { echo "Candidate cannot be its own baseline" >&2; exit 1; }
            git rev-parse --verify "refs/tags/$INSTALL_REF^{commit}" >/dev/null
            tags="$(jq -cn --arg ref "$INSTALL_REF" '[$ref]')"
          elif [ "$RELEASE_MODE" = true ]; then
            tags="$(python3 -m scripts.releases.pick_tags --count "$TAG_COUNT" --exclude-ref "$EXCLUDE_REF")"
          else
            tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")"
          fi
          echo "Testing updates from: $tags"
          # Annotate each tag with what its own tree supports, so run
          # workflows can natively skip surfaces the starting version does
          # not have. Today: does the release ship the desktop app
          # (apps/desktop, #20059)? Cheaper here -- the tags are already
          # fetched -- than a probe job per leg.
          enriched="$(for t in $(echo "$tags" | jq -r '.[]'); do
            if git ls-tree -d "$t" apps/desktop | grep -q .; then d=true; else d=false; fi
            echo "{\"ref\":\"$t\",\"desktop\":$d}"
          done | jq -sc .)"
          echo "Annotated: $enriched"
          echo "tags=$enriched" >> "$GITHUB_OUTPUT"

  # Expand the support matrix against the picked tags: one leg per
  # {os, install-method, update-method, tag}, split into a matrix per OS.
  generate-matrix:
    name: Expand combinations
    needs: pick-releases
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      linux: ${{ steps.gen.outputs.linux }}
      windows: ${{ steps.gen.outputs.windows }}
      macos: ${{ steps.gen.outputs.macos }}
      linux-legs: ${{ steps.gen.outputs.linux-legs }}
      windows-legs: ${{ steps.gen.outputs.windows-legs }}
      macos-legs: ${{ steps.gen.outputs.macos-legs }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs
          sparse-checkout-cone-mode: false
      - id: gen
        env:
          # Free text on dispatch, so it reaches the script as data, never as script.
          # Scheduled runs have no inputs: the default route. PRs run the pr subset.
          ROUTE: ${{ github.event_name == 'pull_request' && 'pr' || inputs.route || 'all' }}
        run: |
          set -euo pipefail
          matrices="$(node scripts/sandbox/generate-e2e-matrix.mjs \
            --tags '${{ needs.pick-releases.outputs.tags }}' --route "$ROUTE")"
          echo "$matrices"
          for key in linux windows macos; do
            echo "$key=$(echo "$matrices" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>console.log(JSON.stringify(JSON.parse(d)[process.argv[1]])))' "$key")" >> "$GITHUB_OUTPUT"
            echo "$key-legs=$(echo "$matrices" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>console.log(JSON.parse(d)[process.argv[1]].include.length))' "$key")" >> "$GITHUB_OUTPUT"
          done
          # The plan, human-readable: a combination x starting-tag chart on
          # the run's summary page.
          node scripts/sandbox/generate-e2e-matrix.mjs \
            --tags '${{ needs.pick-releases.outputs.tags }}' \
            --format markdown >> "$GITHUB_STEP_SUMMARY"

  bundle-inputs:
    name: Validate pinned bundle transitions
    # Bundled legs need dispatch-only manifests; a PR never runs them.
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      windows: ${{ steps.inputs.outputs.windows }}
      macos: ${{ steps.inputs.outputs.macos }}
      windows-arch: ${{ steps.inputs.outputs.windows-arch }}
      macos-arch: ${{ steps.inputs.outputs.macos-arch }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - id: inputs
        env:
          BUNDLE_WINDOWS_MANIFEST: ${{ inputs.windows-bundle-manifest }}
          BUNDLE_MACOS_MANIFEST: ${{ inputs.macos-bundle-manifest }}
          BUNDLE_ROUTE: ${{ inputs.route }}
        run: |
          npm ci --ignore-scripts --no-audit --no-fund
          node tests/install/e2e-assets/bundle-plan.mjs

  windows-bundled:
    name: ${{ matrix.name }}
    if: inputs.windows-bundle-manifest != '' && contains(fromJSON('["all", "bundled", "windows-bundled"]'), inputs.route)
    needs: bundle-inputs
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.bundle-inputs.outputs.windows) }}
    uses: ./.github/workflows/install-e2e-windows-run.yml
    with:
      install-method: packaged-app
      update-method: open-app-update
      install-ref: ${{ matrix.install_ref }}
      leg-id: ${{ matrix.leg_id }}
      bundle-manifest-url: ${{ inputs.windows-bundle-manifest }}
      bundle-arch: ${{ needs.bundle-inputs.outputs.windows-arch }}

  macos-bundled:
    name: ${{ matrix.name }}
    if: inputs.macos-bundle-manifest != '' && contains(fromJSON('["all", "bundled", "macos-bundled"]'), inputs.route)
    needs: bundle-inputs
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.bundle-inputs.outputs.macos) }}
    uses: ./.github/workflows/install-e2e-macos-run.yml
    with:
      install-method: packaged-app
      update-method: open-app-update
      install-ref: ${{ matrix.install_ref }}
      leg-id: ${{ matrix.leg_id }}
      bundle-manifest-url: ${{ inputs.macos-bundle-manifest }}
      bundle-arch: ${{ needs.bundle-inputs.outputs.macos-arch }}

  linux:
    name: ${{ matrix.name }}
    # The generator applies the route; an OS it selected no legs for is skipped
    # (GitHub rejects an empty matrix).
    if: needs.generate-matrix.outputs.linux-legs != '0'
    needs: generate-matrix
    strategy:
      # One leg breaking is worth knowing about even if another already
      # failed, so let every leg report.
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-matrix.outputs.linux) }}
    uses: ./.github/workflows/install-e2e-run.yml
    with:
      install-method: ${{ matrix.install_method }}
      update-method: ${{ matrix.update_method }}
      install-ref: ${{ matrix.install_ref }}
      update-ref: ${{ matrix.update_ref }}
      tag-has-desktop: ${{ matrix.tag_has_desktop }}
      leg-id: ${{ matrix.leg_id }}

  windows:
    name: ${{ matrix.name }}
    if: needs.generate-matrix.outputs.windows-legs != '0'
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-matrix.outputs.windows) }}
    uses: ./.github/workflows/install-e2e-windows-run.yml
    with:
      install-method: ${{ matrix.install_method }}
      update-method: ${{ matrix.update_method }}
      install-ref: ${{ matrix.install_ref }}
      update-ref: ${{ matrix.update_ref }}
      tag-has-desktop: ${{ matrix.tag_has_desktop }}
      leg-id: ${{ matrix.leg_id }}

  macos:
    name: ${{ matrix.name }}
    if: needs.generate-matrix.outputs.macos-legs != '0'
    needs: generate-matrix
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.generate-matrix.outputs.macos) }}
    # Two driver arms: the OS-agnostic script driver (shared with linux)
    # and the published-dmg GUI driver; the run workflow routes.
    uses: ./.github/workflows/install-e2e-macos-run.yml
    with:
      install-method: ${{ matrix.install_method }}
      update-method: ${{ matrix.update_method }}
      install-ref: ${{ matrix.install_ref }}
      update-ref: ${{ matrix.update_ref }}
      tag-has-desktop: ${{ matrix.tag_has_desktop }}
      leg-id: ${{ matrix.leg_id }}

  # The leg player: one static HTML for the whole run. Uploaded BEFORE the
  # matrix legs so it exists even when every leg dies; the report job links
  # every ran leg to it with that leg's logs zip as a #zip= hash param
  # (hash survives the artifact URL's server-side redirect, the query does
  # not). archive: false makes GitHub name the artifact after the FILE
  # (playback.html), ignoring the name: input -- harmless, the renderer
  # looks it up by that name.
  leg-player:
    name: Upload leg player
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: tests/install/e2e-assets/playback.html
          sparse-checkout-cone-mode: false
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: install-e2e-player
          path: tests/install/e2e-assets/playback.html
          archive: false
          retention-days: 14
          if-no-files-found: error

  # Strict source-matrix gate (release runs). needs collapses each matrix to
  # one aggregate, so the three source parents are the required check here.
  # Every source leg must succeed — a skip is a failure EXCEPT when it is the
  # native, declared capability skip (a starting tag predating the desktop
  # app reports tag-has-desktop=false and its GUI leg legitimately cannot
  # run); those are honest skips, counted and listed. A no-legs-at-all
  # generation (empty matrix) also fails: a release run must actually test
  # the source install/update paths.
  source-gate:
    name: Source matrix gate
    if: always() && inputs.release == true
    needs: [generate-matrix, linux, windows, macos]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Evaluate source legs
        env:
          LINUX: ${{ needs.linux.result }}
          WINDOWS: ${{ needs.windows.result }}
          MACOS: ${{ needs.macos.result }}
          LINUX_MATRIX: ${{ needs.generate-matrix.outputs.linux }}
          WINDOWS_MATRIX: ${{ needs.generate-matrix.outputs.windows }}
          MACOS_MATRIX: ${{ needs.generate-matrix.outputs.macos }}
        run: |
          set -euo pipefail
          fail=0
          for os_name in LINUX WINDOWS MACOS; do
            result="${!os_name}"
            matrix_var="${os_name}_MATRIX"
            matrix="${!matrix_var}"
            legs=$(printf '%s' "$matrix" | jq 'if type == "object" then ([.include // []] | add // []) else . end | length')
            echo "$os_name: result=$result legs=$legs"
            if [ "$result" != "success" ] || [ "$legs" -eq 0 ]; then
              echo "$os_name requires executed source coverage"
              echo "::error::$os_name source matrix result=$result (legs=$legs)"
              fail=1
            fi
          done
          if [ "$fail" -ne 0 ]; then
            echo "::error::source install/update matrix did not fully succeed; a release cannot proceed on a partial or skipped matrix"
            exit 1
          fi
          echo "Source install/update matrix: all OS aggregates succeeded"

  # The outcome, human-readable: the plan chart again, with each cell
  # replaced by how that leg actually concluded. Per-leg conclusions are
  # NOT reachable through `needs` (a matrix job's result collapses to one
  # aggregate), so the table body comes from the run's own job list; the
  # `needs` results only sequence this job after every leg and provide
  # the per-OS aggregates.
  report:
    name: Result chart
    if: always()
    needs: [leg-player, pick-releases, linux, windows, macos, bundle-inputs, windows-bundled, macos-bundled]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: |
            scripts/sandbox/generate-e2e-matrix.mjs
            tests/install/e2e-assets/known-failures.json
          sparse-checkout-cone-mode: false
      - env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          {
            echo "OS jobs: linux ${{ needs.linux.result }}, windows ${{ needs.windows.result }}, macos ${{ needs.macos.result }}"
            echo
            # The tag annotations let the chart say WHY a cell skipped
            # (pre-desktop vs declared TODO) instead of a flat "skip".
            gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?per_page=100" \
              --paginate --jq '.jobs[] | {name, conclusion}' > /tmp/e2e-jobs.ndjson
            gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts?per_page=100" \
              --paginate --jq '.artifacts[] | {name, id}' > /tmp/e2e-artifacts.ndjson
            echo 'Legend: ✅ upgrade passed · known [n] = exact historical failure, see footnote · ❌ unexpected failure · pre-desktop / TODO = why a leg skipped · 📼 opens the leg player (recording + synced logs)'
            echo
            node scripts/sandbox/generate-e2e-matrix.mjs --format results \
              --tags '${{ needs.pick-releases.outputs.tags }}' \
              --artifacts /tmp/e2e-artifacts.ndjson < /tmp/e2e-jobs.ndjson
          } >> "$GITHUB_STEP_SUMMARY"