From b676997d2d386a1479b7730a05dcdf22d1977e3b Mon Sep 17 00:00:00 2001 From: ethernet Date: Tue, 8 Sep 2026 12:45:15 -0400 Subject: [PATCH] ci: provision locked Python and Node toolchains through PM --- .github/actions/setup-pm/README.md | 75 +++++++ .github/actions/setup-pm/action.yml | 145 ++++++++++++ .github/actions/setup-pm/prune/action.yml | 14 ++ .github/actions/setup-pm/prune/index.mjs | 25 +++ .github/workflows/canary-release.yml | 5 +- .github/workflows/deploy-site.yml | 19 +- .github/workflows/desktop-bundled-release.yml | 166 ++++---------- .github/workflows/docker.yml | 24 +- .github/workflows/docs-site-checks.yml | 19 +- .github/workflows/e2e-desktop.yml | 36 +-- .github/workflows/icons-freshness-check.yml | 14 +- .github/workflows/js-autofix.yml | 11 +- .github/workflows/js-tests.yml | 18 +- .github/workflows/lint.yml | 32 +-- .github/workflows/pm-bundle.yml | 12 +- .github/workflows/pm-toolchain-smoke.yml | 97 ++++++++ .github/workflows/pm-toolchain.yml | 41 ++++ .github/workflows/skills-index.yml | 6 +- .github/workflows/stable-release.yml | 20 +- .github/workflows/termux-verify.yml | 23 +- .github/workflows/tests-os.yml | 36 +-- .github/workflows/tests.yml | 94 +------- .github/workflows/uv-lockfile-check.yml | 9 +- .github/workflows/windows-venv-e2e.yml | 21 +- scripts/ci/setup_toolchain.py | 212 ++++++++++++++++++ scripts/ci/verify_toolchain.py | 70 ++++++ tests-js/setup-pm-post.test.mjs | 33 +++ tests/scripts/test_setup_toolchain.py | 61 +++++ 28 files changed, 921 insertions(+), 417 deletions(-) create mode 100644 .github/actions/setup-pm/README.md create mode 100644 .github/actions/setup-pm/action.yml create mode 100644 .github/actions/setup-pm/prune/action.yml create mode 100644 .github/actions/setup-pm/prune/index.mjs create mode 100644 .github/workflows/pm-toolchain-smoke.yml create mode 100644 .github/workflows/pm-toolchain.yml create mode 100644 scripts/ci/setup_toolchain.py create mode 100644 scripts/ci/verify_toolchain.py create mode 100644 tests-js/setup-pm-post.test.mjs create mode 100644 tests/scripts/test_setup_toolchain.py diff --git a/.github/actions/setup-pm/README.md b/.github/actions/setup-pm/README.md new file mode 100644 index 0000000000..3bedd1ffbd --- /dev/null +++ b/.github/actions/setup-pm/README.md @@ -0,0 +1,75 @@ +# Locked CI toolchains + +Check out this repository, then use `./.github/actions/setup-pm`. The runner's +preinstalled Python bootstraps PM with the standard library only. PM downloads +and verifies the exact native artifacts from `pm/lock.json`. The action does +not resolve a version range, install another setup action, or modify the lock. + +```yaml +- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 +- uses: ./.github/actions/setup-pm + with: + toolchain: all + extras: '["dev"]' +- run: python --version && uv --version && node --version && npm --version +``` + +`toolchain` defaults to `python` (Python and uv). `node` installs Node and npm; +`all` installs both pairs. There are no version overrides. `extras` is a JSON +array because GitHub action inputs are strings. Omit it for tools only; `[]` +installs the core Python dependencies, and `["dev"]` adds the dev extra. PM +checks `uv.lock`, installs the requested dependencies, validates the environment, +and publishes its selection. It does not enable plugins. + +Subsequent steps get `python`, `python3`, `uv`, `uvx`, `node`, `npm` and `npx` +for the selected toolchain on PATH. The pinned npm precedes Node's bundled npm. +On Windows, PM selects the host architecture even if the bootstrap interpreter +runs under x64 emulation. A disposable command environment supplies the missing +`python3.exe` alias without changing the verified interpreter store. + +For Python dependencies, the action exports `HERMES_PYTHON`, `VIRTUAL_ENV` and +`UV_PROJECT_ENVIRONMENT`. Use `scripts/run_tests.sh`; do not activate `.venv`. +The environment belongs to PM under the runner's temporary home, not the +checkout. Tool-only jobs can install small CI-specific package subsets with +`uv pip install --python "$HERMES_PYTHON" package==version`; these writes do not +modify the cached tool store. Native builds still need their system libraries +and compiler, such as OpenSSL for Windows ARM64 cryptography. + +## Caches + +The official, SHA-pinned `actions/cache` transports three independent caches: + +| Cache | Contents | Identity | +| --- | --- | --- | +| Tools | PM store and installed facts | Native target, toolchain and PM lock hash | +| Python | PM's actual uv download/build cache | Native target, OS version, Python version, prune policy and dependency-file hash | +| Node | `npm config get cache` | Runner OS, native architecture and npm dependency-lock hash | + +All restores use the exact primary key, without fallback prefixes, matching +setup-uv and setup-node's npm behavior. Successful jobs save at teardown; +exact hits are not saved again. PM re-verifies restored tools before use. +Dependency caches never contain `node_modules` or virtual environments. +Keep an installed-tree cache in the caller if that job needs one. + +`cache`, `cache-python` and `cache-node` independently disable the store, uv, +and npm caches. Each defaults to `true`; language-specific caches run only for +that toolchain. `python-cache-dependency-glob` defaults to `pyproject.toml` and +`uv.lock`. `node-cache-dependency-path` defaults to `package-lock.json`; use +`website/package-lock.json` for site jobs. Both accept multiline glob strings. +An npm cache without a matching lockfile fails, rather than caching an +unversioned dependency set. + +`prune-python-cache: true` registers `uv cache prune --ci --force` at teardown, +after the caller's installs and before the cache save. It is skipped on an +exact hit. The default is `false`, as in setup-uv v9; migrated v8 callers opt in +to retain their former policy. The small nested JavaScript action exists only +because GitHub composite actions cannot declare their own post step. It uses +Node's standard library and has no bundled dependencies. + +Outputs include `python-version`, `uv-version`, `node-version`, `npm-version`, +`python-path`, `uv-path`, `venv`, `target`, and the three `*-cache-hit` flags. +Use the version outputs in installed-tree cache keys instead of repeating pins. + +`.github/workflows/pm-toolchain.yml` exercises cold setup and a separate warm +runner for Linux, macOS and Windows on both architectures. Its optional cache +suffix keeps that proof isolated from ordinary build caches. diff --git a/.github/actions/setup-pm/action.yml b/.github/actions/setup-pm/action.yml new file mode 100644 index 0000000000..cfd5ddc5b7 --- /dev/null +++ b/.github/actions/setup-pm/action.yml @@ -0,0 +1,145 @@ +name: Set up the locked PM toolchain +description: Install Python/uv and Node/npm from pm/lock.json; cache tools and dependency downloads. +inputs: + toolchain: + description: python (Python and uv), node (Node and npm), or all. + default: python + extras: + description: 'JSON array of Python project extras. Omit for tools only; [] installs core; ["dev"] adds dev.' + default: '' + cache: + description: Cache the verified PM tool store. + default: 'true' + cache-python: + description: Cache uv downloads and built wheels; never cache the dependency environment. + default: 'true' + cache-node: + description: Cache npm downloads; never cache node_modules. + default: 'true' + python-cache-dependency-glob: + description: Dependency files that invalidate the uv cache. + default: | + pyproject.toml + uv.lock + node-cache-dependency-path: + description: npm lockfiles that invalidate the npm download cache; supports multiline globs. + default: package-lock.json + prune-python-cache: + description: Prune uv's cache at job teardown before saving, as setup-uv v8 did. + default: 'false' + cache-suffix: + description: Optional namespace for isolated cache smoke tests. + default: '' +outputs: + python-version: + description: Locked CPython version, without the PBS build suffix. + value: ${{ steps.prepare.outputs.python-version }} + uv-version: + description: Locked uv version. + value: ${{ steps.prepare.outputs.uv-version }} + node-version: + description: Locked Node version. + value: ${{ steps.prepare.outputs.node-version }} + npm-version: + description: Locked npm version. + value: ${{ steps.prepare.outputs.npm-version }} + python-path: + description: Interpreter for subsequent steps, including requested extras. + value: ${{ steps.dependencies.outputs.python-path || steps.install.outputs.python-path }} + uv-path: + description: PM-provisioned uv executable. + value: ${{ steps.install.outputs.uv-path }} + venv: + description: PM-selected dependency environment, if extras were supplied. + value: ${{ steps.dependencies.outputs.venv }} + target: + description: Native PM target, independent of the bootstrap interpreter architecture. + value: ${{ steps.prepare.outputs.target }} + tools-cache-hit: + description: Exact PM store cache hit. + value: ${{ steps.tools-cache.outputs.cache-hit }} + python-cache-hit: + description: Exact uv dependency cache hit. + value: ${{ steps.python-cache.outputs.cache-hit }} + node-cache-hit: + description: Exact npm dependency cache hit. + value: ${{ steps.node-cache.outputs.cache-hit }} +runs: + using: composite + steps: + - name: Read PM pins with the runner bootstrap Python + id: prepare + shell: bash + env: + _PM_ACTION: ${{ github.action_path }} + _PM_TOOLCHAIN: ${{ inputs.toolchain }} + _PM_EXTRAS: ${{ inputs.extras }} + run: | + set -euo pipefail + # Windows provides python, not necessarily python3. The host resolver + # in PM still selects ARM64 when this bootstrap Python runs under x64. + if [ "$RUNNER_OS" = Windows ]; then bootstrap=python; else bootstrap=python3; fi + "$bootstrap" -S "$_PM_ACTION/../../../scripts/ci/setup_toolchain.py" prepare \ + --toolchain "$_PM_TOOLCHAIN" --extras "$_PM_EXTRAS" --home "$RUNNER_TEMP/setup-pm" + + - name: Cache verified PM tools + id: tools-cache + if: inputs.cache == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.prepare.outputs.store }} + key: setup-pm-tools-v1-${{ steps.prepare.outputs.target }}-${{ inputs.toolchain }}-${{ hashFiles('pm/lock.json') }}-${{ inputs.cache-suffix }} + + - name: Install and verify tools through PM + id: install + shell: bash + env: + _PM_ACTION: ${{ github.action_path }} + _PM_BOOTSTRAP: ${{ steps.prepare.outputs.bootstrap-python }} + _PM_TOOLCHAIN: ${{ inputs.toolchain }} + run: | + "$_PM_BOOTSTRAP" -S "$_PM_ACTION/../../../scripts/ci/setup_toolchain.py" install \ + --toolchain "$_PM_TOOLCHAIN" --home "$HERMES_HOME" + + - name: Cache uv dependency downloads and builds + id: python-cache + if: inputs.toolchain != 'node' && inputs.cache-python == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.install.outputs.uv-cache-path }} + key: setup-pm-uv-v1-${{ steps.prepare.outputs.target }}-${{ steps.prepare.outputs.os-version }}-${{ steps.prepare.outputs.python-version }}-${{ inputs.prune-python-cache }}-${{ hashFiles(inputs.python-cache-dependency-glob) || 'no-dependency-glob' }}-${{ inputs.cache-suffix }} + + # Post steps run in reverse registration order: prune before cache save. + - name: Register uv cache pruning + if: inputs.toolchain != 'node' && inputs.cache-python == 'true' && inputs.prune-python-cache == 'true' && steps.python-cache.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-pm/prune + with: + uv: ${{ steps.install.outputs.uv-path }} + cache: ${{ steps.install.outputs.uv-cache-path }} + + - name: Require an npm dependency lock for caching + if: inputs.toolchain != 'python' && inputs.cache-node == 'true' && hashFiles(inputs.node-cache-dependency-path) == '' + shell: bash + run: | + printf '%s\n' '::error::No npm lock matched node-cache-dependency-path.' + exit 1 + + - name: Cache npm dependency downloads + id: node-cache + if: inputs.toolchain != 'python' && inputs.cache-node == 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.install.outputs.npm-cache-path }} + key: node-cache-${{ runner.os }}-${{ steps.prepare.outputs.arch }}-npm-${{ hashFiles(inputs.node-cache-dependency-path) }}${{ inputs.cache-suffix != '' && format('-{0}', inputs.cache-suffix) || '' }} + + - name: Install the requested Python extras through PM + id: dependencies + if: inputs.extras != '' + shell: bash + env: + _PM_ACTION: ${{ github.action_path }} + _PM_TOOLCHAIN: ${{ inputs.toolchain }} + _PM_EXTRAS: ${{ inputs.extras }} + run: | + "$UV_PYTHON" -S "$_PM_ACTION/../../../scripts/ci/setup_toolchain.py" dependencies \ + --toolchain "$_PM_TOOLCHAIN" --extras "$_PM_EXTRAS" --home "$HERMES_HOME" diff --git a/.github/actions/setup-pm/prune/action.yml b/.github/actions/setup-pm/prune/action.yml new file mode 100644 index 0000000000..1a807290cd --- /dev/null +++ b/.github/actions/setup-pm/prune/action.yml @@ -0,0 +1,14 @@ +name: Prune the PM uv cache at job teardown +description: Register uv pruning before the enclosing cache action saves. +inputs: + uv: + description: Absolute path to the PM-provisioned uv executable. + required: true + cache: + description: Cache directory restored by the enclosing setup action. + required: true +runs: + using: node24 + main: index.mjs + post: index.mjs + post-if: success() diff --git a/.github/actions/setup-pm/prune/index.mjs b/.github/actions/setup-pm/prune/index.mjs new file mode 100644 index 0000000000..e45a13f217 --- /dev/null +++ b/.github/actions/setup-pm/prune/index.mjs @@ -0,0 +1,25 @@ +import { spawnSync } from 'node:child_process' +import { appendFileSync } from 'node:fs' +import { pathToFileURL } from 'node:url' + +// Composite actions cannot register a post step. Called after actions/cache, +// this action's post runs first, pruning the cache just before it is saved. +export function run(env, execute = spawnSync) { + if (!env.STATE_uv) { + for (const [key, value] of Object.entries({ uv: env.INPUT_UV, cache: env.INPUT_CACHE })) { + if (!value || /[\r\n\0]/.test(value)) throw new Error(`invalid ${key}`) + appendFileSync(env.GITHUB_STATE, `${key}=${value}\n`, 'utf8') + } + return + } + const result = execute(env.STATE_uv, ['cache', 'prune', '--ci', '--force'], { + env: { ...env, UV_CACHE_DIR: env.STATE_cache }, + stdio: 'inherit', + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`uv cache prune failed: ${result.status}`) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + run(process.env) +} diff --git a/.github/workflows/canary-release.yml b/.github/workflows/canary-release.yml index d73b11dca2..4f97afef2f 100644 --- a/.github/workflows/canary-release.yml +++ b/.github/workflows/canary-release.yml @@ -76,10 +76,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: python3 scripts/release.py --prune-canaries --publish --remote origin - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - uses: ./.github/actions/setup-pm with: - version: '0.12.3' - enable-cache: false + cache-python: false - name: Prune R2 canary objects env: CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 7f6c4b5ad9..20b16a7516 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -63,24 +63,21 @@ jobs: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. + - name: Set up locked Node and npm + id: node + uses: ./.github/actions/setup-pm with: - node-version: 26 - cache: npm - cache-dependency-path: website/package-lock.json + toolchain: node + node-cache-dependency-path: website/package-lock.json - - name: grab npm 12 - run: | - npm i -g npm@12 - - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.14' + cache-python: true - name: Install PyYAML for skill extraction uses: ./.github/actions/retry with: - command: pip install pyyaml==6.0.2 httpx==0.28.1 + command: uv pip install --python "$HERMES_PYTHON" pyyaml==6.0.2 httpx==0.28.1 - name: Prepare skills index (unified multi-source catalog) env: diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 865d0bb256..3aa20932f2 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -260,19 +260,12 @@ jobs: ref: ${{ needs.validate.outputs.sha }} fetch-tags: true - - name: Resolve toolchain pins from pm/lock.json - id: pins - shell: bash - # The host toolchain that BUILDS the artifact comes from the same - # pin table as the embedded runtimes (pm/lock.json), so gate == pin - # by construction in bundles/desktop.py's toolchain gates. - run: | - python -c ' - import json - pkgs = json.load(open("pm/lock.json"))["packages"] - for tool in ("node", "npm", "uv"): - print(tool + "=" + pkgs[tool]["version"]) - ' >> "$GITHUB_OUTPUT" + - name: Set up the locked build toolchain + id: pm + uses: ./.github/actions/setup-pm + with: + toolchain: all + cache-python: false - name: Resolve toolchain cache key id: toolchain @@ -287,22 +280,6 @@ jobs: console.log(`builder=${eb}`) ' >> "$GITHUB_OUTPUT" - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. - with: - node-version: ${{ steps.pins.outputs.node }} - cache: npm - - - name: Install pinned npm - shell: bash - env: - NPM_PIN: ${{ steps.pins.outputs.npm }} - run: npm --version | grep -qx "$NPM_PIN" || npm i -g "npm@$NPM_PIN" - - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # 9.0.0 - with: - version: ${{ steps.pins.outputs.uv }} - enable-cache: false - - name: Cache verified payload signatures uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -389,7 +366,7 @@ jobs: ui-tui/packages/*/node_modules web/node_modules tests-js/node_modules - key: node-modules-${{ matrix.target.label }}-node${{ steps.pins.outputs.node }}-npm${{ steps.pins.outputs.npm }}-${{ hashFiles('package-lock.json') }} + key: node-modules-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}-${{ hashFiles('package-lock.json') }} # npm ci rm -rf's node_modules before installing, so a restore is # never shipped stale — but a restore-key hit still makes the # reinstall incremental (postinstall outputs like node-pty's @@ -397,7 +374,7 @@ jobs: # The build-bundled install-stamp gate (lock sha + node + npm + # target) is the real guard against stale trees shipping. restore-keys: | - node-modules-${{ matrix.target.label }}-node${{ steps.pins.outputs.node }}-npm${{ steps.pins.outputs.npm }}- + node-modules-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}- - name: Cache node-pty prebuilds (postinstall output, emulated-gyp tax on arm64) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -458,12 +435,12 @@ jobs: AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_TOKEN_CREDENTIALS: prod run: | - uv run --no-project --python 3.14 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled # The Store-submission MSIX is the same bundled payload re-packed # with the Partner Center packaging identity (publish-win32-store # bundles these into the universal Store .msixbundle and submits it; # they also land in the tag archive, never a feed dir). - uv run --no-project --python 3.14 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=store + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=store - name: Verify native signature cache contracts shell: bash @@ -588,19 +565,12 @@ jobs: fetch-tags: true fetch-depth: 0 - - name: Resolve toolchain pins from pm/lock.json - id: pins - shell: bash - # The host toolchain that BUILDS the artifact comes from the same - # pin table as the embedded runtimes (pm/lock.json), so gate == pin - # by construction in bundles/desktop.py's toolchain gates. - run: | - python3 -c ' - import json - pkgs = json.load(open("pm/lock.json"))["packages"] - for tool in ("node", "npm", "uv"): - print(tool + "=" + pkgs[tool]["version"]) - ' >> "$GITHUB_OUTPUT" + - name: Set up the locked build toolchain + id: pm + uses: ./.github/actions/setup-pm + with: + toolchain: all + cache-python: false - name: Resolve toolchain cache key id: toolchain @@ -615,22 +585,6 @@ jobs: console.log(`builder=${eb}`) ' >> "$GITHUB_OUTPUT" - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: ${{ steps.pins.outputs.node }} - cache: npm - - - name: Install pinned npm - shell: bash - env: - NPM_PIN: ${{ steps.pins.outputs.npm }} - run: npm --version | grep -qx "$NPM_PIN" || npm i -g "npm@$NPM_PIN" - - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # 9.0.0 - with: - version: ${{ steps.pins.outputs.uv }} - enable-cache: false - - name: Cache pm store # Tag-dispatched runs (every canary) scope actions/cache under the # dispatch ref, which GitHub mangles to refs/heads/refs/tags/ — @@ -683,7 +637,7 @@ jobs: ui-tui/packages/*/node_modules web/node_modules tests-js/node_modules - key: node-modules-${{ matrix.target.label }}-node${{ steps.pins.outputs.node }}-npm${{ steps.pins.outputs.npm }}-${{ hashFiles('package-lock.json') }} + key: node-modules-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}-${{ hashFiles('package-lock.json') }} # npm ci rm -rf's node_modules before installing, so a restore is # never shipped stale — but a restore-key hit still makes the # reinstall incremental (postinstall outputs like node-pty's @@ -691,7 +645,7 @@ jobs: # The build-bundled install-stamp gate (lock sha + node + npm + # target) is the real guard against stale trees shipping. restore-keys: | - node-modules-${{ matrix.target.label }}-node${{ steps.pins.outputs.node }}-npm${{ steps.pins.outputs.npm }}- + node-modules-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}- - name: Cache node-pty prebuilds (postinstall output) uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -774,7 +728,7 @@ jobs: # every Mach-O) — raise the fd limit and let DEBUG show progress. ulimit -n 16384 2>/dev/null || true echo "file descriptor limit: soft=$(ulimit -Sn) hard=$(ulimit -Hn)" - uv run --no-project --python 3.14 python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled - name: Audit bundle architecture shell: bash @@ -925,16 +879,12 @@ jobs: # minutes-since-the-last-stable from git tags — must see them. fetch-tags: true - - name: Resolve toolchain pins from pm/lock.json - id: pins - shell: bash - run: | - python -c ' - import json - pkgs = json.load(open("pm/lock.json"))["packages"] - for tool in ("node", "npm", "uv"): - print(tool + "=" + pkgs[tool]["version"]) - ' >> "$GITHUB_OUTPUT" + - name: Set up the locked build toolchain + id: pm + uses: ./.github/actions/setup-pm + with: + toolchain: node + cache-node: false - name: Resolve toolchain cache key id: toolchain @@ -1074,16 +1024,12 @@ jobs: ref: ${{ needs.validate.outputs.sha }} fetch-tags: true - - name: Resolve toolchain pins from pm/lock.json - id: pins - shell: bash - run: | - python -c ' - import json - pkgs = json.load(open("pm/lock.json"))["packages"] - for tool in ("node", "npm", "uv"): - print(tool + "=" + pkgs[tool]["version"]) - ' >> "$GITHUB_OUTPUT" + - name: Set up the locked build toolchain + id: pm + uses: ./.github/actions/setup-pm + with: + toolchain: node + cache-node: false - name: Resolve toolchain cache key id: toolchain @@ -1098,10 +1044,6 @@ jobs: console.log(`builder=${eb}`) ' >> "$GITHUB_OUTPUT" - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. - with: - node-version: ${{ steps.pins.outputs.node }} - # makeappx for the Store bundle lives in the same winCodeSign toolset # the win32 legs downloaded — restore the identical eb2 cache. - name: Resolve electron's default download cache path @@ -1222,10 +1164,9 @@ jobs: # Privileged job: pin to the SHA validate admitted, not the tag. ref: ${{ needs.validate.outputs.sha }} - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - uses: ./.github/actions/setup-pm with: - version: '0.12.3' - enable-cache: false + cache-python: false - name: Download both darwin legs' feed ymls uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 @@ -1287,23 +1228,12 @@ jobs: fetch-depth: 0 fetch-tags: true - - name: Resolve toolchain pins from pm/lock.json - id: toolchain_pins - shell: bash - # Same pin table as the desktop legs: the uv that drives the - # wheelhouse resolution is the pinned toolchain uv, not whatever - # happens to be on the runner image. - run: | - python -c ' - import json - pkgs = json.load(open("pm/lock.json"))["packages"] - print("uv=" + pkgs["uv"]["version"]) - ' >> "$GITHUB_OUTPUT" - - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # 9.0.0 + - name: Set up the locked build toolchain + id: pm + uses: ./.github/actions/setup-pm with: - version: ${{ steps.toolchain_pins.outputs.uv }} - enable-cache: false + toolchain: python + cache-python: false - name: Derive the channel from the tag id: channel @@ -1682,9 +1612,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.validate.outputs.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: pattern: stable-candidate-* @@ -1716,9 +1646,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.validate.outputs.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: stable-release-candidates @@ -1744,9 +1674,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.validate.outputs.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: stable-release-candidates @@ -1784,13 +1714,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ needs.validate.outputs.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: '0.12.3' - enable-cache: false + cache-python: false - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: name: stable-release-candidates diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 508ab75ecf..f96584bff4 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -202,27 +202,11 @@ jobs: # (The release path DOES pay that cost — see the save steps below — # because publish must push the exact tested bytes, not a rebuild.) # --------------------------------------------------------------------- - - name: Install uv (for docker tests) - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" - - - name: Set up Python 3.14 (for docker tests) - uses: ./.github/actions/retry - with: - command: uv python install 3.14 - - - name: Install Python dependencies (for docker tests) - # ``dev`` extra pulls in pytest, pytest-asyncio — - # everything tests/docker/ needs. We deliberately avoid ``all`` - # here because the docker tests only drive the container via - # subprocess and don't import hermes_agent's optional deps. - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra dev + extras: '["dev"]' + prune-python-cache: true - name: Run docker integration tests env: diff --git a/.github/workflows/docs-site-checks.yml b/.github/workflows/docs-site-checks.yml index c37f119a82..2bd8d0e044 100644 --- a/.github/workflows/docs-site-checks.yml +++ b/.github/workflows/docs-site-checks.yml @@ -13,15 +13,12 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. + - name: Set up locked Node and npm + id: node + uses: ./.github/actions/setup-pm with: - node-version: 26 - cache: npm - cache-dependency-path: website/package-lock.json - - - name: grab npm 12 - run: | - npm i -g npm@12 + toolchain: node + node-cache-dependency-path: website/package-lock.json - name: Install website dependencies uses: ./.github/actions/retry @@ -29,14 +26,14 @@ jobs: command: npm ci working-directory: website - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: "3.14" + cache-python: true - name: Install ascii-guard uses: ./.github/actions/retry with: - command: python -m pip install ascii-guard==2.3.0 pyyaml==6.0.3 + command: uv pip install --python "$HERMES_PYTHON" ascii-guard==2.3.0 pyyaml==6.0.3 - name: Extract skill metadata for dashboard run: python3 website/scripts/extract-skills.py diff --git a/.github/workflows/e2e-desktop.yml b/.github/workflows/e2e-desktop.yml index dc772cdaac..24ba09d893 100644 --- a/.github/workflows/e2e-desktop.yml +++ b/.github/workflows/e2e-desktop.yml @@ -40,14 +40,11 @@ jobs: xdg-utils libatspi2.0-0 libdrm2 libgbm1 libasound2t64 # ── Node ─────────────────────────────────────────────────────────── - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. + - name: Set up locked Node and npm + id: node + uses: ./.github/actions/setup-pm with: - node-version: 26 - cache: npm - - - name: grab npm 12 - run: | - npm i -g npm@12 + toolchain: node # Full npm ci (not --ignore-scripts): electron's postinstall # downloads the binary we launch, and node-pty's native build is @@ -57,28 +54,11 @@ jobs: command: npm ci # ── Python (for the hermes serve backend) ────────────────────────── - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and backend dependencies + uses: ./.github/actions/setup-pm with: - # Pin the uv version: unpinned, setup-uv resolves "latest" by - # fetching a manifest from raw.githubusercontent.com on EVERY job — - # a transient fetch failure fails the whole job (2026-07-28 slice-5 - # incident). Pinned, the binary downloads directly; no manifest hop. - version: '0.9.28' - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock - - - name: Set up Python 3.14 - uses: ./.github/actions/retry - with: - command: uv python install 3.14 - - - name: Install Python dependencies - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra all --extra dev + extras: '["all", "dev"]' + prune-python-cache: true # ── Build desktop app ───────────────────────────────────────────── # The Playwright step below runs `npm run build` before testing so diff --git a/.github/workflows/icons-freshness-check.yml b/.github/workflows/icons-freshness-check.yml index 195fc7193c..ffdf634c71 100644 --- a/.github/workflows/icons-freshness-check.yml +++ b/.github/workflows/icons-freshness-check.yml @@ -28,17 +28,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock + toolchain: all + cache-python: true + prune-python-cache: true - name: Regenerate every target, then verify structure run: | diff --git a/.github/workflows/js-autofix.yml b/.github/workflows/js-autofix.yml index dd341ba5d6..8db763fd28 100644 --- a/.github/workflows/js-autofix.yml +++ b/.github/workflows/js-autofix.yml @@ -65,14 +65,11 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. + - name: Set up locked Node and npm + id: node + uses: ./.github/actions/setup-pm with: - node-version: 26 - cache: npm - - - name: grab npm 12 - run: | - npm i -g npm@12 + toolchain: node # --ignore-scripts: eslint only needs TS sources + eslint packages. - uses: ./.github/actions/retry diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index c2b707e210..d1067f0960 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -18,19 +18,13 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: actions/setup-node@v7.0.0 # immutable release. safe to pin. + - name: Set up locked Node and npm + id: node + uses: ./.github/actions/setup-pm with: - node-version: 26 - cache: npm + toolchain: node - - name: grab npm 12 - run: | - # No-op once the bundled npm is already 12.x — saves ~5-15s/job and - # keeps the installed major aligned with the npm12 cache-key tag. - npm --version | grep -q '^12\.' || npm i -g npm@12 - - # The ``cache: npm`` option of ``setup-node`` caches only the ~/.npm - # tarball cache. The job then extracts the full workspace node_modules + # setup-pm caches npm downloads, not the installed workspace tree. The job then extracts the full workspace node_modules # again and runs the postinstalls again, which includes the Electron # binary fetch. This caches the installed tree itself, keyed on the # lockfile, and skips ``npm ci`` on an exact hit. There are no @@ -55,7 +49,7 @@ jobs: ui-tui/packages/*/node_modules tests-js/node_modules web/node_modules - key: node-modules-scripts-${{ runner.os }}-node26-npm12-${{ hashFiles('package-lock.json') }} + key: node-modules-scripts-${{ runner.os }}-${{ runner.arch }}-node${{ steps.node.outputs.node-version }}-npm${{ steps.node.outputs.npm-version }}-${{ hashFiles('package-lock.json') }} - uses: ./.github/actions/retry if: steps.node-modules-cache.outputs.cache-hit != 'true' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 05af2e65a5..a25ef164f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -38,13 +38,10 @@ jobs: with: fetch-depth: 0 # need full history for merge-base + worktree - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" + cache-python: true + prune-python-cache: true - name: Install ruff + ty uses: ./.github/actions/retry @@ -132,13 +129,10 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" + cache-python: true + prune-python-cache: true - name: Install ruff uses: ./.github/actions/retry @@ -163,18 +157,10 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" - - - name: Set up Python 3.14 - uses: ./.github/actions/retry - with: - command: uv python install 3.14 + cache-python: true + prune-python-cache: true - name: Run footgun checker run: python scripts/check-windows-footguns.py --all diff --git a/.github/workflows/pm-bundle.yml b/.github/workflows/pm-bundle.yml index 0455482bb2..24940f4986 100644 --- a/.github/workflows/pm-bundle.yml +++ b/.github/workflows/pm-bundle.yml @@ -76,11 +76,10 @@ jobs: ref: ${{ inputs.ref || github.sha }} fetch-tags: true - # The host uv only bootstraps a python to run pm itself; every - # payload tool (uv included) is staged by pm from pm/lock.json. - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # 9.0.0 + # The host and payload toolchains use the same PM pins and installer. + - uses: ./.github/actions/setup-pm with: - enable-cache: false + cache-python: false # One cache for the pm store: keyed on the lockfile, so a pin bump # rotates it. pm verifies every restored entry against the lock @@ -136,13 +135,10 @@ jobs: # Scoped to the cargo target triple so every other sdist keeps MSVC. CC_aarch64_pc_windows_msvc: ${{ matrix.target.label == 'win32-arm64' && 'clang' || '' }} run: | - # The bootstrap interpreter tracks the payload interpreter's - # minor version — one authority (pm/lock.json), no drift. - PYVER=$(python3 -c "import json; v=json.load(open('pm/lock.json'))['packages']['python']['version']; print('.'.join(v.split('+')[0].split('.')[:2]))" 2>/dev/null || python -c "import json; v=json.load(open('pm/lock.json'))['packages']['python']['version']; print('.'.join(v.split('+')[0].split('.')[:2]))") # Archive what actions/checkout actually checked out. On # pull_request events github.sha names a merge commit that a # force-push invalidates mid-run ("not a tree object"). - uv run --no-project --python "$PYVER" python -m pm.cli bundle \ + python -m pm.cli bundle \ --out build/agent-payload \ --ref HEAD diff --git a/.github/workflows/pm-toolchain-smoke.yml b/.github/workflows/pm-toolchain-smoke.yml new file mode 100644 index 0000000000..214a1693a4 --- /dev/null +++ b/.github/workflows/pm-toolchain-smoke.yml @@ -0,0 +1,97 @@ +name: PM toolchain native smoke +on: + workflow_call: + inputs: + warm: + type: boolean + required: true +permissions: + contents: read +jobs: + native: + name: ${{ inputs.warm && 'warm' || 'cold' }} ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + include: + - target: linux-x64 + runner: ubuntu-24.04 + - target: linux-arm64 + runner: ubuntu-24.04-arm + - target: darwin-x64 + runner: macos-15-intel + - target: darwin-arm64 + runner: macos-15 + - target: win32-x64 + runner: windows-2025 + - target: win32-arm64 + runner: windows-11-arm + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # cryptography has no win_arm64 wheel. Its native build uses the + # same OpenSSL prerequisite as the bundled release, not an x64 Python. + - name: Cache native OpenSSL + if: matrix.target == 'win32-arm64' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: C:\vcpkg\installed\arm64-windows-static-md + key: vcpkg-openssl-arm64-windows-static-md-${{ runner.os }} + - name: Install native OpenSSL + if: matrix.target == 'win32-arm64' + shell: bash + run: | + if [ ! -f /c/vcpkg/installed/arm64-windows-static-md/lib/libcrypto.lib ]; then + "$VCPKG_INSTALLATION_ROOT/vcpkg" install openssl:arm64-windows-static-md + fi + printf 'OPENSSL_DIR=C:\\vcpkg\\installed\\arm64-windows-static-md\nOPENSSL_STATIC=1\n' >> "$GITHUB_ENV" + + - name: Set up tools and dev extra from PM + id: pm + uses: ./.github/actions/setup-pm + with: + toolchain: all + extras: '["dev"]' + # Each workflow run proves a genuinely cold save followed by a + # different runner restoring it. No pre-existing cache can mask it. + cache-suffix: smoke-${{ github.run_id }}-${{ github.run_attempt }} + + - name: Verify next-step PATH and exact cache hits + shell: bash + env: + EXPECT_WARM: ${{ inputs.warm }} + TOOLS_HIT: ${{ steps.pm.outputs.tools-cache-hit }} + PYTHON_HIT: ${{ steps.pm.outputs.python-cache-hit }} + NODE_HIT: ${{ steps.pm.outputs.node-cache-hit }} + run: | + python scripts/ci/verify_toolchain.py --extras + python -c 'import os; expected=os.environ["EXPECT_WARM"]=="true"; values={k:os.environ[k]=="true" for k in ("TOOLS_HIT","PYTHON_HIT","NODE_HIT")}; print(values); assert all(v==expected for v in values.values()), values' + + - name: Install the real locked npm workspace + shell: bash + env: + EXPECT_WARM: ${{ inputs.warm }} + run: | + flags=() + if [ "$EXPECT_WARM" = true ]; then flags+=(--offline); fi + npm ci --workspace tests-js --include-workspace-root --include=dev --ignore-scripts --no-audit --no-fund ${flags[@]+"${flags[@]}"} + node node_modules/vitest/vitest.mjs run --root tests-js setup-pm-post.test.mjs + + - name: Run the PM and action contracts + shell: bash + run: | + scripts/run_tests.sh -j 2 tests/scripts/test_setup_toolchain.py tests/pm/test_pm_core.py tests/pm/test_bootstrap_import_closure.py tests/pm/test_uv_cache.py + python scripts/ci/verify_toolchain.py --extras + + - name: Upload native setup proof + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pm-toolchain-${{ matrix.target }}-${{ inputs.warm && 'warm' || 'cold' }} + path: ${{ runner.temp }}/pm-toolchain-proof.json + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/pm-toolchain.yml b/.github/workflows/pm-toolchain.yml new file mode 100644 index 0000000000..e9913f8b10 --- /dev/null +++ b/.github/workflows/pm-toolchain.yml @@ -0,0 +1,41 @@ +name: PM Toolchain +on: + push: + branches: [ci/pm-toolchain] + pull_request: + paths: + - '.github/actions/setup-pm/**' + - '.github/workflows/pm-toolchain*.yml' + - 'scripts/ci/*toolchain.py' + - 'tests/scripts/test_setup_toolchain.py' + - 'tests-js/setup-pm-post.test.mjs' + - 'pm/**' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: pm-toolchain-${{ github.ref }} + cancel-in-progress: true +jobs: + cold: + uses: ./.github/workflows/pm-toolchain-smoke.yml + with: + warm: false + warm: + needs: cold + uses: ./.github/workflows/pm-toolchain-smoke.yml + with: + warm: true + prune: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: ./.github/actions/setup-pm + with: + extras: '["dev"]' + prune-python-cache: true + cache-suffix: smoke-prune-${{ github.run_id }}-${{ github.run_attempt }} + - run: python -c 'import pytest; print(pytest.__version__)' diff --git a/.github/workflows/skills-index.yml b/.github/workflows/skills-index.yml index c15dc8072b..119e195b6d 100644 --- a/.github/workflows/skills-index.yml +++ b/.github/workflows/skills-index.yml @@ -32,14 +32,14 @@ jobs: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: "3.14" + cache-python: true - name: Install dependencies uses: ./.github/actions/retry with: - command: pip install httpx==0.28.1 pyyaml==6.0.2 + command: uv pip install --python "$HERMES_PYTHON" httpx==0.28.1 pyyaml==6.0.2 - name: Build skills index env: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 4eb2831ef5..e9de1e35e1 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -32,9 +32,9 @@ jobs: with: ref: ${{ github.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - id: admit run: python -m scripts.releases.stable admit env: @@ -128,9 +128,9 @@ jobs: with: ref: ${{ github.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - id: plan run: python -m scripts.releases.stable transitions env: @@ -186,9 +186,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - run: python -m scripts.releases.stable gate admit ci docker nix pm-bundle termux-checks windows-live install-e2e candidates transitions windows-packaged macos-packaged env: RELEASE_NEEDS: ${{ toJSON(needs) }} @@ -224,9 +224,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.sha }} - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - run: python -m scripts.releases.stable gate acceptance publish-docker publish-bundles env: RELEASE_NEEDS: ${{ toJSON(needs) }} @@ -266,9 +266,9 @@ jobs: with: ref: ${{ github.sha }} fetch-depth: 0 - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: ./.github/actions/setup-pm with: - python-version: '3.11' + cache-python: false - run: python -m scripts.releases.stable gate admit ci docker acceptance candidates publication promote-docker promote-bundles env: RELEASE_NEEDS: ${{ toJSON(needs) }} diff --git a/.github/workflows/termux-verify.yml b/.github/workflows/termux-verify.yml index ea1cee5298..ef52260173 100644 --- a/.github/workflows/termux-verify.yml +++ b/.github/workflows/termux-verify.yml @@ -39,17 +39,14 @@ jobs: # caller and the head SHA on every other event. ref: ${{ github.sha }} persist-credentials: false - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: '0.12.3' - enable-cache: false - - name: Install the locked test environment + - name: Install the native linker test prerequisite run: | sudo apt-get update sudo apt-get install -y patchelf - uv venv --python 3.14 .venv - uv export --frozen --extra dev --no-emit-project --no-hashes -o "$RUNNER_TEMP/requirements.txt" - uv pip install --python .venv/bin/python -r "$RUNNER_TEMP/requirements.txt" + - uses: ./.github/actions/setup-pm + with: + extras: '["dev"]' + cache-python: false - name: Run the native linker and wheel contracts run: | bash scripts/run_tests.sh -j 2 \ @@ -71,9 +68,10 @@ jobs: # caller and the head SHA on every other event. ref: ${{ github.sha }} persist-credentials: false - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: ./.github/actions/setup-pm with: - node-version: '22' + toolchain: node + cache-node: false - name: Install the locked JS workspace run: npm ci --workspace ui-tui --workspace tests-js --include-workspace-root --include=dev --no-fund --no-audit - name: Check and bundle the TUI @@ -93,10 +91,9 @@ jobs: # caller and the head SHA on every other event. ref: ${{ github.sha }} persist-credentials: false - - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + - uses: ./.github/actions/setup-pm with: - version: '0.12.3' - enable-cache: false + cache-python: false - name: Stage the pinned bionic runtimes run: | bash scripts/termux/build_cpython.sh termux-build/payload diff --git a/.github/workflows/tests-os.yml b/.github/workflows/tests-os.yml index a34bdfc681..0198b8c31c 100644 --- a/.github/workflows/tests-os.yml +++ b/.github/workflows/tests-os.yml @@ -68,35 +68,11 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm with: - # Pinned for the same reason as the Linux lane: unpinned, setup-uv - # resolves "latest" by fetching a manifest on every job and a - # transient fetch failure fails the whole job. - version: "0.9.28" - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock - - - name: Set up Python 3.14 - uses: ./.github/actions/retry - with: - command: uv python install 3.14 - - - name: Install dependencies - # Same extras as the Linux test lane so an OS-marked test can import - # anything its Linux siblings can. ``[all]`` is deliberately - # Windows/macOS-installable (see the policy comment on the extra in - # pyproject.toml — matrix/python-olm was removed from it precisely - # because it could not build here). - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web - - - name: Minimize uv cache - run: uv cache prune --ci + extras: '["all", "dev", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + prune-python-cache: true - name: Run ${{ matrix.marker }} tests # scripts/run_tests.sh — the canonical runner, same as every other @@ -128,7 +104,7 @@ jobs: # Process substitution would hide the helper's exit status, so write # to a file and check it explicitly. - if ! uv run --no-sync python scripts/ci/list_os_marked_tests.py \ + if ! python scripts/ci/list_os_marked_tests.py \ "${{ matrix.marker }}" > "$LIST"; then echo "::error::could not enumerate ${{ matrix.marker }} test files" exit 1 @@ -161,7 +137,7 @@ jobs: # Any non-zero exit propagates red: real test failures, or the # runner's own zero-run guard (every file filtered to empty by # ``-m`` — "must never pass without running its OS's tests"). - SEPARATOR="$(uv run --no-sync python -c 'import os, sys; sys.stdout.write(os.pathsep)')" + SEPARATOR="$(python -c 'import os, sys; sys.stdout.write(os.pathsep)')" FILES="$(tr -d '\r' < "$LIST" | paste -sd "$SEPARATOR" -)" scripts/run_tests.sh --files "$FILES" -- \ ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b518acd6af..bc4deef64d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,51 +41,11 @@ jobs: rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl" rg --version - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm with: - # Pin the uv version: unpinned, setup-uv resolves "latest" by - # fetching a manifest from raw.githubusercontent.com on EVERY job — - # a transient fetch failure fails the whole job (2026-07-28 slice-5 - # incident). Pinned, the binary downloads directly; no manifest hop. - version: "0.9.28" - # Persist uv's download/wheel cache (~/.cache/uv) across runs. - # Keyed on the dependency manifests, so the cache is reused until - # pyproject.toml or uv.lock changes. `uv sync` still runs every - # time, but resolves from the warm cache instead of re-downloading - # and re-building wheels. - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock - - - name: Set up Python 3.14 - uses: ./.github/actions/retry - with: - command: uv python install 3.14 - - - name: Install dependencies - # `uv sync --locked` installs the exact pinned set from uv.lock (and - # fails if the lock is out of sync with pyproject.toml), giving a - # reproducible env. It also creates .venv itself, so no separate - # `uv venv` step is needed. - # - # The trailing extras beyond all/dev are the lazy-install features - # (tools/lazy_deps.py) that tests exercise for real: provider.anthropic, - # stt/tts.mistral, image.fal, terminal.modal, terminal.daytona, - # memory.hindsight, search.parallel. The hermetic test env forbids - # mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in - # tests/conftest.py), so the SDKs those tests need must be in the - # venv up front — resolved from uv.lock like everything else, which - # also honors the exact supply-chain pins these extras carry. - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web - - - name: Minimize uv cache - # Optimized for CI: prunes pre-built wheels that are cheap to - # re-download, keeping the persisted cache small and fast to restore. - run: uv cache prune --ci + extras: '["all", "dev", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + prune-python-cache: true - name: Run tests # Per-file isolation via scripts/run_tests.sh: each test file runs @@ -96,7 +56,6 @@ jobs: # No --files: the runner discovers the suite itself. The discovered # set is identical to the list the removed matrix job used to pass in. run: | - source .venv/bin/activate scripts/run_tests.sh env: # This is the maximum number of test FILES that run together. @@ -146,50 +105,15 @@ jobs: rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl" rg --version - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm with: - # Pin the uv version: unpinned, setup-uv resolves "latest" by - # fetching a manifest from raw.githubusercontent.com on EVERY job — - # a transient fetch failure fails the whole job (2026-07-28 slice-5 - # incident). Pinned, the binary downloads directly; no manifest hop. - version: "0.9.28" - # Persist uv's download/wheel cache (~/.cache/uv) across runs. - # Keyed on the dependency manifests, so the cache is reused until - # pyproject.toml or uv.lock changes. `uv sync` still runs every - # time, but resolves from the warm cache instead of re-downloading - # and re-building wheels. - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock - - - name: Set up Python 3.14 - run: uv python install 3.14 - - - name: Install dependencies - # `uv sync --locked` installs the exact pinned set from uv.lock (and - # fails if the lock is out of sync with pyproject.toml), giving a - # reproducible env. It also creates .venv itself, so no separate - # `uv venv` step is needed. - # - # Same extras as the test job's sync above: the hermetic test env - # forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in - # tests/conftest.py), so lazy-install SDKs exercised by tests must be - # in the venv up front. - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web - - - name: Minimize uv cache - # Optimized for CI: prunes pre-built wheels that are cheap to - # re-download, keeping the persisted cache small and fast to restore. - run: uv cache prune --ci + extras: '["all", "dev", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]' + prune-python-cache: true - name: Run e2e tests run: | - source .venv/bin/activate - python -m pytest tests/e2e/ -v --tb=short + scripts/run_tests.sh tests/e2e/ -v --tb=short env: OPENROUTER_API_KEY: "" OPENAI_API_KEY: "" diff --git a/.github/workflows/uv-lockfile-check.yml b/.github/workflows/uv-lockfile-check.yml index e2c1487aab..a531850375 100644 --- a/.github/workflows/uv-lockfile-check.yml +++ b/.github/workflows/uv-lockfile-check.yml @@ -68,13 +68,10 @@ jobs: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - uses: ./.github/actions/setup-pm with: - # Pinned: unpinned setup-uv fetches a 'latest' manifest from - # raw.githubusercontent.com every job; transient fetch failures - # fail the job (2026-07-28 incident). Keep in sync with tests.yml. - version: "0.9.28" + cache-python: true + prune-python-cache: true # `uv lock --check` re-resolves the project from pyproject.toml and # compares the result to uv.lock, exiting non-zero if they disagree. diff --git a/.github/workflows/windows-venv-e2e.yml b/.github/workflows/windows-venv-e2e.yml index b7a0146916..4643eb0535 100644 --- a/.github/workflows/windows-venv-e2e.yml +++ b/.github/workflows/windows-venv-e2e.yml @@ -48,24 +48,11 @@ jobs: # the exact candidate commit there and the head SHA everywhere else. ref: ${{ github.sha }} - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + - name: Set up locked Python and test dependencies + uses: ./.github/actions/setup-pm with: - version: "0.9.28" - enable-cache: true - cache-dependency-glob: | - pyproject.toml - uv.lock - - - name: Set up Python 3.14 - uses: ./.github/actions/retry - with: - command: uv python install 3.14 - - - name: Install dependencies - uses: ./.github/actions/retry - with: - command: uv sync --locked --python 3.14 --extra dev --extra messaging + extras: '["dev", "messaging"]' + prune-python-cache: true - name: Run venv-holder live E2E # Canonical runner (scripts/run_tests.sh) — never bare pytest: it diff --git a/scripts/ci/setup_toolchain.py b/scripts/ci/setup_toolchain.py new file mode 100644 index 0000000000..3e536a8696 --- /dev/null +++ b/scripts/ci/setup_toolchain.py @@ -0,0 +1,212 @@ +"""GitHub Actions file commands around the real, stdlib-only PM bootstrap.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import platform +import re +import sys + +# The runner invokes this file before the checkout has been installed. +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pm.lock import Lockfile +from pm.paths import lockfile_path +from pm.registry import walk +from pm.store import current_target + + +def packages(toolchain: str) -> list[str]: + roots = {"python": ["python", "uv"], "node": ["npm"], "all": ["python", "uv", "npm"]} + return sorted(package.name for package in walk(roots[toolchain])) + + +def file_commands(destination: str, values: dict) -> None: + """All exports are single-line values, including JSON-encoded arrays.""" + lines = [] + for key, value in values.items(): + text = str(value) + if any(character in key + text for character in "\r\n\0"): + raise ValueError(f"invalid GitHub file command: {key!r}") + lines.append(f"{key}={text}\n") + with open(os.environ[destination], "a", encoding="utf-8") as stream: + stream.writelines(lines) + + +def parse_extras(value: str) -> list[str] | None: + if value == "": + return None + message = "extras must be a JSON array of extra names" + try: + extras = json.loads(value) + except ValueError as exc: + raise argparse.ArgumentTypeError(message) from exc + if not isinstance(extras, list) or any( + not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name) + for name in extras + ): + raise argparse.ArgumentTypeError(message) + return sorted(set(extras)) + + +def prepare(args) -> None: + home = args.home.resolve() + lock = Lockfile(lockfile_path()) + target = current_target() + names = packages(args.toolchain) + values = { + "packages": json.dumps(names), "target": target, "arch": target.split("-")[1], + "store": str(home / "tools"), + } + for name in names: + version = lock.version(name) + if not version or not lock.artifacts(name, target): + raise ValueError(f"{name} has no pinned artifact for {target}") + values[f"{name}-version"] = version.partition("+")[0] if name == "python" else version + # The OS image belongs in the uv cache identity: built wheels can link + # against its system libraries. Unlike npm's cache, these are not just JS. + values["os-version"] = platform.platform() + values["bootstrap-python"] = sys.executable + file_commands("GITHUB_OUTPUT", values) + file_commands("GITHUB_ENV", { + "HERMES_HOME": home, + "HERMES_RUNTIME_DIR": home / "tools", + "PYTHONUTF8": "1", + }) + + +def add_path(directories: list[str]) -> None: + # The runner prepends each line, so write PM's dependent-first PATH in + # reverse. npm must shadow the different npm bundled inside Node. + with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as stream: + for directory in reversed(list(dict.fromkeys(directories))): + if any(character in directory for character in "\r\n\0"): + raise ValueError("invalid PATH directory") + stream.write(directory + "\n") + + +def python3_alias(python: Path) -> None: + if os.name == "nt": + import shutil + + alias = python.with_name("python3.exe") + if not alias.exists(): + shutil.copy2(python, alias) + + +def install(args) -> None: + import subprocess + + from pm.cli import _live_progress + from pm.ensure import ensure, env_for + from pm.lock import Facts + from pm.packages import uv_cache_dir + from pm.paths import facts_path, store_root + from pm.registry import get_package + + names = packages(args.toolchain) + for name in names: + ensure(name, explicit=True, progress=_live_progress(name)) + facts = Facts(facts_path()) + target = current_target() + binaries = { + name: get_package(name).binary(store_root() / facts.get(name)["entry"], target) + for name in names + } + environment = env_for(*names) + path = env_for(*names, base_env={})["PATH"].split(os.pathsep) + exported = {} + outputs = {f"{name}-path": str(binary) for name, binary in binaries.items()} + if "python" in names: + tool_bin = args.home.resolve() / "bin" + # A writable command environment keeps callers' uv pip installs out + # of the verified tool store. On Windows its redirector also supplies + # python3.exe, which PBS does not ship. + commands = args.home.resolve() / "python" / facts.get("python")["entry"] + if not (commands / "pyvenv.cfg").is_file(): + subprocess.run( + [str(binaries["uv"]), "venv", "--relocatable", "--python", str(binaries["python"]), str(commands)], + check=True, env=environment, timeout=120, + ) + python = commands / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + python3_alias(python) + path.insert(0, str(python.parent)) + outputs["python-path"] = str(python) + exported.update({ + "HERMES_PYTHON": python, + "UV_PYTHON": binaries["python"], + "UV_PYTHON_DOWNLOADS": "never", + "UV_CACHE_DIR": uv_cache_dir(), + "UV_TOOL_DIR": args.home.resolve() / "uv-tools", + "UV_TOOL_BIN_DIR": tool_bin, + }) + outputs["uv-cache-path"] = str(uv_cache_dir()) + path.insert(0, str(tool_bin)) + if "npm" in names: + cache = subprocess.check_output( + [str(binaries["npm"]), "config", "get", "cache"], + env=environment, text=True, encoding="utf-8", timeout=60, + ).strip() + outputs["npm-cache-path"] = cache + file_commands("GITHUB_ENV", exported) + file_commands("GITHUB_OUTPUT", outputs) + add_path(path) + print("PM toolchain ready: " + ", ".join(f"{name} {facts.get(name)['version']}" for name in names)) + + +def dependencies(args) -> None: + if args.extras is None: + return + import subprocess + import tomllib + + from hermes_cli.runtime_paths import selected_venv + from pm.ensure import sync_venv, uv + from pm.paths import repo_root + + project = repo_root() + metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8-sig")) + unknown = set(args.extras) - metadata["project"]["optional-dependencies"].keys() + if unknown: + raise ValueError(f"unknown project extras: {sorted(unknown)}") + uv_bin, environment = uv() + environment.pop("UV_NO_CONFIG", None) # keep project indexes and exclude-newer + environment["UV_PYTHON"] = sys.executable + # PM's frozen sync must not turn a stale project lock into a green job. + subprocess.run([uv_bin, "lock", "--check"], cwd=project, env=environment, check=True, timeout=1800) + sync_venv(args.extras, explicit=True, plugin_dirs=[]) + venv = selected_venv(project) + bindir = venv / ("Scripts" if os.name == "nt" else "bin") + python = bindir / ("python.exe" if os.name == "nt" else "python") + python3_alias(python) + file_commands("GITHUB_ENV", { + "HERMES_PYTHON": python, + "VIRTUAL_ENV": venv, + "UV_PROJECT_ENVIRONMENT": venv, + "PYTHONPATH": project, + }) + file_commands("GITHUB_OUTPUT", {"python-path": python, "venv": venv}) + add_path([str(bindir)]) + print(f"PM dependencies ready: {args.extras} in {venv}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + phases = {"prepare": prepare, "install": install, "dependencies": dependencies} + parser.add_argument("phase", choices=list(phases)) + parser.add_argument("--toolchain", choices=["python", "node", "all"], default="python") + parser.add_argument("--home", type=Path, required=True) + parser.add_argument("--extras", type=parse_extras, default="") + args = parser.parse_args() + if args.toolchain == "node" and args.extras is not None: + parser.error("extras require the python or all toolchain") + os.environ["HERMES_HOME"] = str(args.home.resolve()) + os.environ["HERMES_RUNTIME_DIR"] = str(args.home.resolve() / "tools") + phases[args.phase](args) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/verify_toolchain.py b/scripts/ci/verify_toolchain.py new file mode 100644 index 0000000000..9b0d77081b --- /dev/null +++ b/scripts/ci/verify_toolchain.py @@ -0,0 +1,70 @@ +"""Native setup-pm smoke: verify PATH, PM identity and installed extras.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from pm.lock import Facts, Lockfile +from pm.package import machine_matches_binary +from pm.paths import facts_path, lockfile_path, runtime_facts_path, store_root +from pm.registry import get_package +from pm.store import current_target, tree_digest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--extras", action="store_true") + args = parser.parse_args() + lock = Lockfile(lockfile_path()) + facts = Facts(facts_path()) + target = current_target() + rows = {} + for name in ("python", "python3", "uv", "node", "npm", "npx"): + binary = shutil.which(name) + if binary is None: + raise RuntimeError(f"{name} is missing from PATH") + result = subprocess.check_output([binary, "--version"], text=True, encoding="utf-8", timeout=60).strip() + package = {"python3": "python", "npx": "npm"}.get(name, name) + pin = lock.version(package) + expected = pin.partition("+")[0] + actual = result.split()[1] if package in ("python", "uv") else result.removeprefix("v") + if actual != expected: + raise RuntimeError(f"{name} on PATH is {result}, expected {pin}: {binary}") + entry = store_root() / facts.get(package)["entry"] + artifact = get_package(package).binary(entry, target) + if facts.get(package)["artifacts"] != [a["sha256"] for a in lock.artifacts(package, target)]: + raise RuntimeError(f"{name} has the wrong pinned artifact identity") + if machine_matches_binary(artifact, target) is False: + raise RuntimeError(f"{name} is not native {target}") + if package == name and facts.get(package)["digest"] != tree_digest(entry): + raise RuntimeError(f"{name} store was mutated after verification") + rows[name] = {"path": binary, "version": result, "target": target} + if args.extras: + selected = Facts(runtime_facts_path()).get("venv") + if selected["extras"] != ["dev"]: + raise RuntimeError(f"unexpected PM extras: {selected['extras']}") + if Path(sys.prefix).resolve() != Path(selected["environment"]).resolve(): + raise RuntimeError("PATH Python did not select the PM dependency environment") + import pytest + import yaml + + rows["dependencies"] = {"pytest": pytest.__version__, "pyyaml": yaml.__version__} + code = "import sys,pytest; print(sys.prefix); print(pytest.__version__)" + probe = subprocess.check_output([shutil.which("python3"), "-c", code], text=True, encoding="utf-8", timeout=60) + if pytest.__version__ not in probe: + raise RuntimeError("python3 did not inherit the installed dev extra") + print(json.dumps(rows, indent=2)) + destination = Path(os.environ["RUNNER_TEMP"]) / "pm-toolchain-proof.json" + destination.write_text(json.dumps(rows, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tests-js/setup-pm-post.test.mjs b/tests-js/setup-pm-post.test.mjs new file mode 100644 index 0000000000..1437244184 --- /dev/null +++ b/tests-js/setup-pm-post.test.mjs @@ -0,0 +1,33 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' + +import { run } from '../.github/actions/setup-pm/prune/index.mjs' + +const directories = [] +afterEach(() => { + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }) +}) + +it('prunes the saved uv cache at teardown, never during registration', () => { + const directory = mkdtempSync(join(tmpdir(), 'pm-post-')) + directories.push(directory) + const state = join(directory, 'state') + const execute = vi.fn(() => ({ status: 0 })) + const cache = join(directory, 'cache with spaces') + const uv = join(directory, 'locked uv') + run({ GITHUB_STATE: state, INPUT_UV: uv, INPUT_CACHE: cache }, execute) + expect(execute).not.toHaveBeenCalled() + const saved = Object.fromEntries(readFileSync(state, 'utf8').trim().split('\n').map(line => { + const index = line.indexOf('=') + return [`STATE_${line.slice(0, index)}`, line.slice(index + 1)] + })) + run({ ...saved, UV_CACHE_DIR: 'a later unrelated cache' }, execute) + expect(execute).toHaveBeenCalledWith(uv, ['cache', 'prune', '--ci', '--force'], expect.objectContaining({ + env: expect.objectContaining({ UV_CACHE_DIR: cache }), + stdio: 'inherit', + })) + execute.mockReturnValueOnce({ status: 1 }) + expect(() => run(saved, execute)).toThrow('uv cache prune failed') +}) diff --git a/tests/scripts/test_setup_toolchain.py b/tests/scripts/test_setup_toolchain.py new file mode 100644 index 0000000000..b2d2469ce1 --- /dev/null +++ b/tests/scripts/test_setup_toolchain.py @@ -0,0 +1,61 @@ +"""The CI bootstrap reads PM's pins without importing installed dependencies.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from pm.lock import Lockfile +from pm.paths import lockfile_path +from pm.store import current_target + + +@pytest.mark.parametrize("toolchain,names", [ + ("python", {"python", "uv"}), + ("node", {"node", "npm"}), + ("all", {"python", "uv", "node", "npm"}), +]) +def test_stdlib_bootstrap_exports_the_pm_lock(toolchain, names, tmp_path): + root = Path(__file__).resolve().parents[2] + output = tmp_path / "output" + envfile = tmp_path / "environment" + home = tmp_path / "runner state" + env = {**os.environ, "GITHUB_OUTPUT": str(output), "GITHUB_ENV": str(envfile)} + result = subprocess.run( + [sys.executable, "-S", str(root / "scripts/ci/setup_toolchain.py"), + "prepare", "--toolchain", toolchain, "--home", str(home)], + cwd=tmp_path, env=env, capture_output=True, text=True, encoding="utf-8", timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + values = dict(line.split("=", 1) for line in output.read_text(encoding="utf-8-sig").splitlines()) + lock = Lockfile(lockfile_path()) + assert json.loads(values["packages"]) == sorted(names) + assert values["target"] == current_target() + for name in names: + expected = lock.version(name) + assert values[f"{name}-version"] == (expected.partition("+")[0] if name == "python" else expected) + exported = dict(line.split("=", 1) for line in envfile.read_text(encoding="utf-8-sig").splitlines()) + assert Path(exported["HERMES_HOME"]) == home + assert Path(exported["HERMES_RUNTIME_DIR"]).is_relative_to(home) + assert not Path(exported["HERMES_RUNTIME_DIR"]).exists(), "prepare must not provision before cache restore" + + +@pytest.mark.parametrize("extras", ['"dev"', '{}', '[1]', '["dev\\nHERMES_HOME=bad"]', '["--all"]']) +def test_invalid_extras_do_not_export_or_install(extras, tmp_path): + root = Path(__file__).resolve().parents[2] + output = tmp_path / "output" + home = tmp_path / "state" + result = subprocess.run( + [sys.executable, "-S", str(root / "scripts/ci/setup_toolchain.py"), "prepare", + "--home", str(home), "--extras", extras], + cwd=tmp_path, env={**os.environ, "GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True, encoding="utf-8", timeout=30, + ) + assert result.returncode != 0 + assert "extras must be a JSON array of extra names" in result.stderr + assert not output.exists() + assert not home.exists()