From 2efa4ff94f0b2aec37fba1928e96c312eb845206 Mon Sep 17 00:00:00 2001 From: ethernet Date: Sun, 13 Sep 2026 14:28:31 -0400 Subject: [PATCH] refactor(desktop): prepare dependencies before saving build caches Dependency acquisition during packaging left native wheels and packager inputs outside the pre-build cache save. Compose PM and existing providers into a preparation phase, then require builds to consume admitted inputs. Share native preparation with PM Bundle. Keep path-bound environments and signing outputs separate from reusable caches. Use read-only cache tokens for commit builds and preserve the one-command local build path. Verify pinned tools through PM, probe PTYs under the prepared Electron, and supply dmgbuild through a build-only PM package. Resolve bundled tool stores from their payload manifest so relocation preserves discovery. Validation: focused Python and JS tests, checkJs, Ruff, Windows checks, anti-slop, cache relocation, and network-denied Linux AppImage builds. Relocated runtime smoke passed with NixOS host libraries supplied. Native Windows/macOS signing and live GitHub cache behavior remain untested. --- .../actions/desktop-build-cache/action.yml | 96 ++++ .github/workflows/desktop-bundled-release.yml | 443 ++++++------------ .github/workflows/pm-bundle.yml | 81 ++-- .gitignore | 1 + apps/desktop/BUILDING.md | 81 +++- apps/desktop/scripts/before-pack.mjs | 49 +- apps/desktop/scripts/dmgbuild-diagnostics.cjs | 4 +- .../scripts/dmgbuild-diagnostics.test.mjs | 11 + apps/desktop/scripts/prepare-dmgbuild.mjs | 26 + .../desktop/scripts/prepare-dmgbuild.test.mjs | 36 ++ .../scripts/prepare-packaging-tools.mjs | 134 ++++++ apps/desktop/scripts/prepare_dmgbuild.py | 33 ++ apps/desktop/scripts/prepared-native-deps.mjs | 55 +++ .../scripts/prepared-native-deps.test.mjs | 44 ++ apps/desktop/scripts/prepared-packaging.mjs | 132 ++++++ .../scripts/prepared-packaging.test.mjs | 61 +++ .../scripts/prepared-windows-tools.test.mjs | 59 +++ .../desktop/scripts/probe-prepared-native.mjs | 85 ++++ .../scripts/probe-prepared-native.test.mjs | 12 + apps/desktop/scripts/run-electron-builder.mjs | 233 +++++---- .../scripts/run-electron-builder.test.mjs | 68 +++ apps/desktop/scripts/sign-msix.mjs | 82 ++-- apps/desktop/scripts/stage-native-deps.mjs | 15 +- apps/desktop/scripts/utils.mjs | 1 + apps/desktop/scripts/windows-bundle-tools.mjs | 34 +- docs/shared-bundle-builds.md | 143 +++++- hermes_cli/runtime_paths.py | 10 +- pm/__init__.py | 4 +- pm/build_operations.py | 146 ++++++ pm/client.py | 22 +- pm/ensure.py | 5 +- pm/lock.json | 13 + pm/packages.py | 29 ++ pm/worker_operations.py | 2 + scripts/build/README.md | 43 +- scripts/build/icon_environment.py | 15 +- scripts/build/node-deps.mjs | 38 +- scripts/bundles/desktop.py | 143 +++--- scripts/bundles/desktop_inputs.py | 67 +++ scripts/bundles/desktop_prepare.py | 280 +++++++++++ scripts/bundles/desktop_toolchain.py | 193 ++++++++ scripts/bundles/native.py | 150 ++++-- scripts/bundles/native_build.py | 114 +++++ scripts/bundles/native_prepared.py | 141 ++++++ scripts/ci/desktop_build_cache.py | 126 +++++ scripts/ci/desktop_commands.py | 45 ++ tests-js/desktop-builder.test.mjs | 1 + tests-js/node-deps.test.mjs | 19 + tests/hermes_cli/test_runtime_paths.py | 67 +++ tests/pm/test_build_operations.py | 145 ++++++ tests/pm/test_dmgbuild_package.py | 46 ++ tests/scripts/test_bundle_native.py | 127 ++++- .../scripts/test_commit_bundle_entrypoints.py | 94 ++-- tests/scripts/test_desktop_build_cache.py | 297 ++++++++++++ tests/scripts/test_desktop_preparation.py | 159 +++++++ tests/scripts/test_desktop_toolchain.py | 286 +++++++++++ tests/scripts/test_native_build.py | 99 ++++ tests/scripts/test_pm_runtime_bundle.py | 5 +- website/docs/reference/package-management.md | 19 + 59 files changed, 4225 insertions(+), 744 deletions(-) create mode 100644 .github/actions/desktop-build-cache/action.yml create mode 100644 apps/desktop/scripts/prepare-dmgbuild.mjs create mode 100644 apps/desktop/scripts/prepare-dmgbuild.test.mjs create mode 100644 apps/desktop/scripts/prepare-packaging-tools.mjs create mode 100644 apps/desktop/scripts/prepare_dmgbuild.py create mode 100644 apps/desktop/scripts/prepared-native-deps.mjs create mode 100644 apps/desktop/scripts/prepared-native-deps.test.mjs create mode 100644 apps/desktop/scripts/prepared-packaging.mjs create mode 100644 apps/desktop/scripts/prepared-packaging.test.mjs create mode 100644 apps/desktop/scripts/prepared-windows-tools.test.mjs create mode 100644 apps/desktop/scripts/probe-prepared-native.mjs create mode 100644 apps/desktop/scripts/probe-prepared-native.test.mjs create mode 100644 apps/desktop/scripts/run-electron-builder.test.mjs create mode 100644 scripts/bundles/desktop_inputs.py create mode 100644 scripts/bundles/desktop_prepare.py create mode 100644 scripts/bundles/desktop_toolchain.py create mode 100644 scripts/bundles/native_build.py create mode 100644 scripts/bundles/native_prepared.py create mode 100644 scripts/ci/desktop_build_cache.py create mode 100644 scripts/ci/desktop_commands.py create mode 100644 tests/hermes_cli/test_runtime_paths.py create mode 100644 tests/pm/test_dmgbuild_package.py create mode 100644 tests/scripts/test_desktop_build_cache.py create mode 100644 tests/scripts/test_desktop_preparation.py create mode 100644 tests/scripts/test_desktop_toolchain.py create mode 100644 tests/scripts/test_native_build.py diff --git a/.github/actions/desktop-build-cache/action.yml b/.github/actions/desktop-build-cache/action.yml new file mode 100644 index 0000000000..10b99a1bf9 --- /dev/null +++ b/.github/actions/desktop-build-cache/action.yml @@ -0,0 +1,96 @@ +name: Desktop build dependency cache +description: Transport reusable dependency candidates; preparation always performs owner admission. +inputs: + phase: + description: Explicit restore or save, with preparation between the two calls. + required: true + source: + description: Admitted source checkout containing the dependency declarations. + required: true + cache: + description: Dedicated reusable dependency root passed to preparation. + required: true + work: + description: Job-local preparation root; never part of the snapshot. + required: true + producer: + description: Producer namespace (not a cache-write authorization boundary). + required: true + key: + description: For save, the cache-key output from the restore call. + default: '' +outputs: + cache-key: + description: Immutable save key; pass this to the save phase. + value: ${{ inputs.key || steps.describe.outputs.cache-key }} + cache-paths: + description: JSON array of direct provider paths, including not-yet-created destinations. + value: ${{ steps.describe.outputs.cache-paths }} + input-prefix: + description: Preferred dependency-input restore prefix. + value: ${{ steps.describe.outputs.input-prefix }} + restore-prefix: + description: Same-producer, target and host fallback prefix. + value: ${{ steps.describe.outputs.restore-prefix }} + cache-hit: + description: Transport result only; never permission to skip preparation. + value: ${{ steps.restore.outputs.cache-hit }} +runs: + using: composite + steps: + - name: Describe dependency transport with runner Python + id: describe + if: ${{ !cancelled() }} + shell: bash + env: + CACHE_ACTION: ${{ github.action_path }} + CACHE_PHASE: ${{ inputs.phase }} + CACHE_SOURCE: ${{ inputs.source }} + CACHE_ROOT: ${{ inputs.cache }} + CACHE_WORK: ${{ inputs.work }} + CACHE_PRODUCER: ${{ inputs.producer }} + CACHE_KEY: ${{ inputs.key }} + run: | + set -euo pipefail + case "$CACHE_PHASE" in + restore) ;; + save) test -n "$CACHE_KEY" || { printf '%s\n' '::error::Save requires the restore cache-key.'; exit 1; } ;; + *) printf '%s\n' '::error::Cache phase must be restore or save.'; exit 1 ;; + esac + # Description must not create CACHE_WORK: only preparation may claim it. + # Paths go straight to Actions; no job-local manifest is transported. + if [ "$RUNNER_OS" = Windows ]; then bootstrap=python; else bootstrap=python3; fi + "$bootstrap" -S "$CACHE_ACTION/../../../scripts/ci/desktop_build_cache.py" describe \ + --source "$CACHE_SOURCE" --cache "$CACHE_ROOT" --work "$CACHE_WORK" --producer "$CACHE_PRODUCER" + + - name: Restore dependency candidates + id: restore + if: ${{ !cancelled() && inputs.phase == 'restore' && steps.describe.outcome == 'success' }} + continue-on-error: true + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ join(fromJSON(steps.describe.outputs.cache-paths), fromJSON('"\n"')) }} + key: ${{ steps.describe.outputs.cache-key }} + restore-keys: | + ${{ steps.describe.outputs.input-prefix }} + ${{ steps.describe.outputs.restore-prefix }} + enableCrossOsArchive: 'false' + fail-on-cache-miss: 'false' + + # Owners have stopped writing before the caller enters save. No post hook + # or pruning can race cancellation, and signed products never enter here. + # Trusted workflow/token policy, not this key or condition, admits writers. + - name: Save dependency candidates before compilation and signing + id: save + if: ${{ !cancelled() && inputs.phase == 'save' && steps.describe.outcome == 'success' }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ join(fromJSON(steps.describe.outputs.cache-paths), fromJSON('"\n"')) }} + key: ${{ inputs.key }} + enableCrossOsArchive: 'false' + + - name: Report cache service failure without masking preparation + if: ${{ !cancelled() && (steps.restore.outcome == 'failure' || steps.save.outcome == 'failure') }} + shell: bash + run: printf '%s\n' '::warning::Desktop dependency cache unavailable; provider preparation remains authoritative.' \ No newline at end of file diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 9f5bacfc02..e3cb5b5512 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -254,14 +254,17 @@ jobs: sha: ${{ needs.validate.outputs.sha }} secrets: inherit - build-win32: + # Cache permissions are literal job-token scopes, not checkout/save-step policy. + # Share execution via anchors, and keep the original IDs as downstream gates. + build-win32-release: name: bundled ${{ matrix.target.label }} - if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && inputs.build_commit == '' needs: [validate, archive-inputs] runs-on: ${{ matrix.target.runner }} environment: release-signing + cache-mode: write timeout-minutes: 900 - strategy: + strategy: &win32-strategy fail-fast: false matrix: target: @@ -269,20 +272,16 @@ jobs: runner: windows-2025 - label: win32-arm64 runner: windows-11-arm - env: + env: &win32-env HERMES_DESKTOP_VARIANT: bundled HERMES_PAYLOAD_TAG: ${{ inputs.tag }} HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} HERMES_PAYLOAD_VERSION: ${{ needs.validate.outputs.payload-version }} + # Signature outputs are separate from the prepared dependency snapshot. ELECTRON_BUILDER_CACHE: ${{ github.workspace }}/.cache/electron-builder - ELECTRON_CACHE: ${{ github.workspace }}/.cache/electron - electron_config_cache: ${{ github.workspace }}/.cache/electron - CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} - CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} - CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} - steps: + steps: &win32-steps - name: Exclude build write paths from Defender scanning shell: powershell run: | @@ -313,29 +312,38 @@ jobs: ref: ${{ needs.validate.outputs.sha }} fetch-tags: true - - name: Set up the locked build toolchain - id: pm - uses: ./.github/actions/setup-pm + - name: Restore desktop dependency inputs + id: build-cache + uses: ./.github/actions/desktop-build-cache with: - toolchain: all - archive-inputs: true - # The payload tools cache does not include uv's built wheels. - cache-python: true - save-python-cache: false - save-node-cache: false + phase: restore + source: ${{ github.workspace }} + work: ${{ runner.temp }}/desktop-job + cache: ${{ github.workspace }}/.cache/desktop-inputs + producer: desktop - - name: Resolve toolchain cache key - id: toolchain + - name: Prepare the complete desktop build + id: prepare shell: bash + env: + HERMES_BUNDLE_ENV_JSON: ${{ inputs.bundle_env }} run: | - node -e ' - const l = require("./package-lock.json") - const el = l.packages["apps/desktop/node_modules/electron"].version - const eb = l.packages["node_modules/electron-builder"].version - if (!el || !eb) process.exit(1) - console.log(`electron=${el}`) - console.log(`builder=${eb}`) - ' >> "$GITHUB_OUTPUT" + args=(--tag "$HERMES_PAYLOAD_TAG") + if [ -n "$HERMES_BUILD_COMMIT" ]; then args=(--commit "$HERMES_BUILD_COMMIT"); fi + python scripts/bundles/desktop.py "${args[@]}" --variant bundled --prepare-only \ + --work "$RUNNER_TEMP/desktop-job" --cache "$GITHUB_WORKSPACE/.cache/desktop-inputs" + python -m scripts.ci.desktop_commands "$RUNNER_TEMP/desktop-job/prepared.json" + + - name: Save dependency inputs before building + if: ${{ !cancelled() && steps.prepare.outcome == 'success' && inputs.build_commit == '' }} + uses: ./.github/actions/desktop-build-cache + with: + phase: save + source: ${{ github.workspace }} + work: ${{ runner.temp }}/desktop-job + cache: ${{ github.workspace }}/.cache/desktop-inputs + producer: desktop + key: ${{ steps.build-cache.outputs.cache-key }} - name: Cache verified payload signatures uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -347,100 +355,6 @@ jobs: payload-signatures-v1-${{ runner.os }}-${{ matrix.target.label }}- payload-signatures-v1-${{ runner.os }}- - - name: Set up native ARM64 build dependencies - if: matrix.target.label == 'win32-arm64' - uses: ./.github/actions/setup-windows-build-deps - - - name: Cache pm store - # Tag-dispatched runs (every canary) scope actions/cache under the - # dispatch ref, which GitHub mangles to refs/heads/refs/tags/ — - # a different scope per tag, so an exact key can never be restored - # by a later canary. The content key below is stable across tags - # when pm/lock.json + uv.lock are unchanged; the restore-keys prefix - # (which ignores the tag entirely) rescues the previous canary's - # store when the locks DID move. - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: apps/desktop/build/agent-payload/tools - key: pm-store-v2-${{ matrix.target.label }}-${{ hashFiles('pm/lock.json', 'uv.lock') }} - restore-keys: | - pm-store-v2-${{ matrix.target.label }}- - - - name: Retrieve pinned payload inputs from R2 - shell: bash - run: python -m scripts.ci.archive_inputs --target '${{ matrix.target.label }}' --store apps/desktop/build/agent-payload/tools - - - name: Resolve electron's default download cache path - shell: bash - run: | - # @electron/get does NOT honor ELECTRON_CACHE/electron_config_cache: - # the electron-builder build's electron zip download uses the - # default env-paths cache root. It must be in the actions/cache - # path list or every build re-downloads electron (~115MB). - case "$RUNNER_OS" in - Windows) echo "ELECTRON_DEFAULT_CACHE=$LOCALAPPDATA/electron/Cache" >> "$GITHUB_ENV" ;; - macOS) echo "ELECTRON_DEFAULT_CACHE=$HOME/Library/Caches/electron" >> "$GITHUB_ENV" ;; - *) echo "ELECTRON_DEFAULT_CACHE=$HOME/.cache/electron" >> "$GITHUB_ENV" ;; - esac - - - name: Cache electron + electron-builder toolchain - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ github.workspace }}/.cache/electron-builder - ${{ github.workspace }}/.cache/electron - ${{ env.ELECTRON_DEFAULT_CACHE }} - # eb2: bumped from eb- (2026-08-28) — the old eb- caches never - # contained the electron zip (it lives in @electron/get's default - # cache root, which wasn't in the path list), so every build - # re-downloaded electron. The new key forces a fresh save that - # includes the default cache root. - key: eb2-${{ runner.os }}-${{ runner.arch }}-electron-${{ steps.toolchain.outputs.electron }}-builder-${{ steps.toolchain.outputs.builder }} - # An electron/builder bump misses the exact key, but the previous - # dist is still mostly reusable (electron's postinstall skips the - # download when dist/ exists) — restore it and let npm ci top up. - restore-keys: | - eb2-${{ runner.os }}-${{ runner.arch }}- - - - name: Restore prepared desktop dependencies - id: node-modules-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - node_modules - apps/*/node_modules - ui-tui/node_modules - ui-tui/packages/*/node_modules - web/node_modules - tests-js/node_modules - # No partial restore: npm ci deletes both the tree and native outputs. - key: desktop-node-deps-v1-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}-${{ hashFiles('package-lock.json', 'package.json', 'apps/*/package.json', 'ui-tui/package.json', 'ui-tui/packages/*/package.json', 'web/package.json', 'tests-js/package.json', '.npmrc', 'scripts/build/node-deps.mjs') }} - - - name: Prepare desktop dependencies - shell: bash - run: node scripts/build/node-deps.mjs --source . --workspace apps/desktop --workspace ui-tui --workspace web --reuse - - # Save before packaging: a later build/signing failure must not lose this work, - # and builder mutations must not become the next job's dependency input. - - name: Save prepared desktop dependencies - if: steps.node-modules-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - node_modules - apps/*/node_modules - ui-tui/node_modules - ui-tui/packages/*/node_modules - web/node_modules - tests-js/node_modules - key: ${{ steps.node-modules-cache.outputs.cache-primary-key }} - - - name: Save warmed npm downloads - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.pm.outputs.npm-cache-path }} - key: ${{ steps.pm.outputs.node-cache-key }} - - name: Azure login (OIDC) if: vars.AZURE_CLIENT_ID != '' uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 @@ -463,18 +377,6 @@ jobs: ( while sleep 240; do mint || true; done ) & echo "AZURE_FEDERATED_TOKEN_FILE=$file" >> "$GITHUB_ENV" - - name: Pin CMake < 4 for sdist builds - # python-olm (matrix extra) builds libolm from sdist on non-Linux - # targets, and its libolm/CMakeLists.txt requires CMake < 3.5 - # compat (removed in CMake 4, which the darwin + win32 runners - # ship). Pin a CMake 3.x first on PATH for those legs so the sdist - # build configures. Linux uses the manylinux wheel — no build, no - # cmake needed. The pip cmake package ships a binary wheel for - # every non-Linux target (macos universal2, win_amd64, win_arm64). - shell: bash - run: | - python -m scripts.ci.python_packages cmake==3.31.6 - - name: Build and package shell: powershell timeout-minutes: 900 @@ -489,25 +391,13 @@ jobs: AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_TOKEN_CREDENTIALS: prod run: | - if ($env:HERMES_BUILD_COMMIT) { - python scripts/bundles/desktop.py --commit="$env:HERMES_BUILD_COMMIT" --variant=bundled - } else { - python scripts/bundles/desktop.py --tag="$env:HERMES_PAYLOAD_TAG" --variant=bundled - # The official Store identity is stable-only. Canary must never - # replace it, even in a Store flight ring. - if ($env:HERMES_PAYLOAD_TAG -notlike "*-canary.*") { - python scripts/bundles/desktop.py --tag="$env:HERMES_PAYLOAD_TAG" --variant=store - } + python scripts/bundles/desktop.py --prepared "$env:RUNNER_TEMP/desktop-job/prepared.json" --variant bundled + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (-not $env:HERMES_BUILD_COMMIT -and $env:HERMES_PAYLOAD_TAG -notlike "*-canary.*") { + python scripts/bundles/desktop.py --prepared "$env:RUNNER_TEMP/desktop-job/prepared.json" --variant store + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } - - name: Save warmed Python dependencies even after a packaging failure - if: ${{ !cancelled() && steps.pm.outcome == 'success' }} - uses: ./.github/actions/save-pm-cache - with: - python: ${{ steps.pm.outputs.python-path }} - path: ${{ steps.pm.outputs.uv-cache-path }} - key: ${{ steps.pm.outputs.python-cache-key }} - - name: Verify native signature cache contracts shell: bash working-directory: apps/desktop @@ -524,11 +414,14 @@ jobs: if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} TARGET: ${{ matrix.target.label }} RELEASE_PHASE: ${{ inputs.release-phase }} RELEASE_COMMIT: ${{ needs.validate.outputs.sha }} # The tag archive is the handoff, never an update feed. The receipt - # appears only after both normal and Store packages are uploaded. + # appears after the selected packages (including Store on stable) upload. run: | args=(--include '*.msix') if [ "$RELEASE_PHASE" = candidate ]; then @@ -547,6 +440,42 @@ jobs: --name "$TARGET" --root apps/desktop/release "${args[@]}" fi + build-win32-commit: + name: bundled ${{ matrix.target.label }} (commit) + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && inputs.build_commit != '' + needs: [validate, archive-inputs] + runs-on: ${{ matrix.target.runner }} + environment: release-signing + cache-mode: read + timeout-minutes: 900 + strategy: *win32-strategy + env: *win32-env + steps: *win32-steps + + build-win32: + name: bundled win32 result + # always() crosses the deliberately skipped trust branch; failed admission + # still skips this result, just as it skipped the original native builder. + if: >- + always() && inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') + && needs.validate.result == 'success' && needs.archive-inputs.result == 'success' + needs: [validate, archive-inputs, build-win32-release, build-win32-commit] + runs-on: ubuntu-24.04 + permissions: {} + timeout-minutes: 5 + env: + SELECTED_BUILD_SUCCEEDED: >- + ${{ (inputs.build_commit == '' && needs.build-win32-release.result == 'success' && needs.build-win32-commit.result == 'skipped') + || (inputs.build_commit != '' && needs.build-win32-commit.result == 'success' && needs.build-win32-release.result == 'skipped') }} + steps: &native-build-result-steps + - name: Require exactly the selected native build to succeed + shell: bash + run: | + if [ "$SELECTED_BUILD_SUCCEEDED" != "true" ]; then + echo "::error::selected native build did not succeed, or the other trust branch was not skipped" + exit 1 + fi + # ── macOS builders (REAL) ────────────────────────────────────────────────── # Native per-arch darwin builds via scripts/bundles/desktop.py (the # one driver: same `--mac dmg zip` pass a local mac build runs). Each leg @@ -558,14 +487,15 @@ jobs: # merge-multiple download) and publish-darwin-updater merges them into # releases/darwin//-mac.yml — artifacts first, feed # pointer last, whole channel green before anything publishes. - build-darwin: + build-darwin-release: name: bundled ${{ matrix.target.label }} - if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && inputs.build_commit == '' needs: [validate, archive-inputs] runs-on: ${{ matrix.target.runner }} environment: release-signing + cache-mode: write timeout-minutes: 360 - strategy: + strategy: &darwin-strategy fail-fast: false matrix: target: @@ -573,20 +503,14 @@ jobs: runner: macos-15 - label: darwin-x64 runner: macos-15-intel - env: + env: &darwin-env HERMES_DESKTOP_VARIANT: bundled HERMES_PAYLOAD_TAG: ${{ inputs.tag }} HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} HERMES_PAYLOAD_VERSION: ${{ needs.validate.outputs.payload-version }} - ELECTRON_BUILDER_CACHE: ${{ github.workspace }}/.cache/electron-builder - ELECTRON_CACHE: ${{ github.workspace }}/.cache/electron - electron_config_cache: ${{ github.workspace }}/.cache/electron - CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} - CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} - CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} - steps: + steps: &darwin-steps - name: Disable Spotlight indexing for DMG staging # Spotlight indexes the freshly-mounted dmg staging image past # hdiutil's detach retries (per-VM, not a cross-job race). @@ -603,114 +527,38 @@ jobs: fetch-tags: true fetch-depth: 0 - - name: Set up the locked build toolchain - id: pm - uses: ./.github/actions/setup-pm + - name: Restore desktop dependency inputs + id: build-cache + uses: ./.github/actions/desktop-build-cache with: - toolchain: all - archive-inputs: true - # The payload tools cache does not include uv's built wheels. - cache-python: true - save-python-cache: false - save-node-cache: false + phase: restore + source: ${{ github.workspace }} + work: ${{ runner.temp }}/desktop-job + cache: ${{ github.workspace }}/.cache/desktop-inputs + producer: desktop - - name: Resolve toolchain cache key - id: toolchain + - name: Prepare the complete desktop build + id: prepare shell: bash + env: + HERMES_BUNDLE_ENV_JSON: ${{ inputs.bundle_env }} run: | - node -e ' - const l = require("./package-lock.json") - const el = l.packages["apps/desktop/node_modules/electron"].version - const eb = l.packages["node_modules/electron-builder"].version - if (!el || !eb) process.exit(1) - console.log(`electron=${el}`) - console.log(`builder=${eb}`) - ' >> "$GITHUB_OUTPUT" + args=(--tag "$HERMES_PAYLOAD_TAG") + if [ -n "$HERMES_BUILD_COMMIT" ]; then args=(--commit "$HERMES_BUILD_COMMIT"); fi + python3 scripts/bundles/desktop.py "${args[@]}" --variant bundled --prepare-only \ + --work "$RUNNER_TEMP/desktop-job" --cache "$GITHUB_WORKSPACE/.cache/desktop-inputs" + python3 -m scripts.ci.desktop_commands "$RUNNER_TEMP/desktop-job/prepared.json" - - name: Cache pm store - # Tag-dispatched runs (every canary) scope actions/cache under the - # dispatch ref, which GitHub mangles to refs/heads/refs/tags/ — - # a different scope per tag, so an exact key can never be restored - # by a later canary. The content key below is stable across tags - # when pm/lock.json + uv.lock are unchanged; the restore-keys prefix - # (which ignores the tag entirely) rescues the previous canary's - # store when the locks DID move. - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save dependency inputs before building + if: ${{ !cancelled() && steps.prepare.outcome == 'success' && inputs.build_commit == '' }} + uses: ./.github/actions/desktop-build-cache with: - path: apps/desktop/build/agent-payload/tools - key: pm-store-v2-${{ matrix.target.label }}-${{ hashFiles('pm/lock.json', 'uv.lock') }} - restore-keys: | - pm-store-v2-${{ matrix.target.label }}- - - - name: Retrieve pinned payload inputs from R2 - shell: bash - run: python -m scripts.ci.archive_inputs --target '${{ matrix.target.label }}' --store apps/desktop/build/agent-payload/tools - - - name: Resolve electron's default download cache path - shell: bash - run: | - # @electron/get does NOT honor ELECTRON_CACHE/electron_config_cache: - # the electron-builder build's electron zip download uses the - # default env-paths cache root. It must be in the actions/cache - # path list or every build re-downloads electron (~115MB). - case "$RUNNER_OS" in - Windows) echo "ELECTRON_DEFAULT_CACHE=$LOCALAPPDATA/electron/Cache" >> "$GITHUB_ENV" ;; - macOS) echo "ELECTRON_DEFAULT_CACHE=$HOME/Library/Caches/electron" >> "$GITHUB_ENV" ;; - *) echo "ELECTRON_DEFAULT_CACHE=$HOME/.cache/electron" >> "$GITHUB_ENV" ;; - esac - - - name: Cache electron + electron-builder toolchain - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ github.workspace }}/.cache/electron-builder - ${{ github.workspace }}/.cache/electron - ${{ env.ELECTRON_DEFAULT_CACHE }} - key: eb2-${{ runner.os }}-${{ runner.arch }}-electron-${{ steps.toolchain.outputs.electron }}-builder-${{ steps.toolchain.outputs.builder }} - # An electron/builder bump misses the exact key, but the previous - # dist is still mostly reusable (electron's postinstall skips the - # download when dist/ exists) — restore it and let npm ci top up. - restore-keys: | - eb2-${{ runner.os }}-${{ runner.arch }}- - - - name: Restore prepared desktop dependencies - id: node-modules-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - node_modules - apps/*/node_modules - ui-tui/node_modules - ui-tui/packages/*/node_modules - web/node_modules - tests-js/node_modules - # No partial restore: npm ci deletes both the tree and native outputs. - key: desktop-node-deps-v1-${{ matrix.target.label }}-node${{ steps.pm.outputs.node-version }}-npm${{ steps.pm.outputs.npm-version }}-${{ hashFiles('package-lock.json', 'package.json', 'apps/*/package.json', 'ui-tui/package.json', 'ui-tui/packages/*/package.json', 'web/package.json', 'tests-js/package.json', '.npmrc', 'scripts/build/node-deps.mjs') }} - - - name: Prepare desktop dependencies - shell: bash - run: node scripts/build/node-deps.mjs --source . --workspace apps/desktop --workspace ui-tui --workspace web --reuse - - # Save before packaging: a later build/signing failure must not lose this work, - # and builder mutations must not become the next job's dependency input. - - name: Save prepared desktop dependencies - if: steps.node-modules-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - node_modules - apps/*/node_modules - ui-tui/node_modules - ui-tui/packages/*/node_modules - web/node_modules - tests-js/node_modules - key: ${{ steps.node-modules-cache.outputs.cache-primary-key }} - - - name: Save warmed npm downloads - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ steps.pm.outputs.npm-cache-path }} - key: ${{ steps.pm.outputs.node-cache-key }} + phase: save + source: ${{ github.workspace }} + work: ${{ runner.temp }}/desktop-job + cache: ${{ github.workspace }}/.cache/desktop-inputs + producer: desktop + key: ${{ steps.build-cache.outputs.cache-key }} # Commit artifacts are downloadable too. Require signing for them, # candidates, and published tags before building any payload. @@ -749,22 +597,11 @@ jobs: printf '%s\n' "$APPLE_API_KEY_P8" > "$RUNNER_TEMP/apple-api-key.p8" echo "APPLE_API_KEY=$RUNNER_TEMP/apple-api-key.p8" >> "$GITHUB_ENV" - - name: Pin CMake < 4 for sdist builds - # python-olm (matrix extra) builds libolm from sdist on non-Linux - # targets, and its libolm/CMakeLists.txt requires CMake < 3.5 - # compat (removed in CMake 4, which the darwin runners ship). Pin a - # CMake 3.x first on PATH so the sdist build configures. - shell: bash - run: | - python -m scripts.ci.python_packages cmake==3.31.6 - - name: Build and package shell: bash timeout-minutes: 330 env: HERMES_BUNDLE_ENV_JSON: ${{ inputs.bundle_env }} - GITHUB_SHA: ${{ needs.validate.outputs.sha }} - GITHUB_REF_NAME: ${{ inputs.tag }} PYTHONUTF8: '1' # electron-osx-sign*/electron-notarize* keep the sign+notarize # phase visible: without them NOTHING logs between "signing @@ -780,19 +617,7 @@ jobs: # every Mach-O) — raise the fd limit and let DEBUG show progress. ulimit -n 16384 2>/dev/null || true echo "file descriptor limit: soft=$(ulimit -Sn) hard=$(ulimit -Hn)" - if [ -n "$HERMES_BUILD_COMMIT" ]; then - python scripts/bundles/desktop.py --commit="$HERMES_BUILD_COMMIT" --variant=bundled - else - python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled - fi - - - name: Save warmed Python dependencies even after a packaging failure - if: ${{ !cancelled() && steps.pm.outcome == 'success' }} - uses: ./.github/actions/save-pm-cache - with: - python: ${{ steps.pm.outputs.python-path }} - path: ${{ steps.pm.outputs.uv-cache-path }} - key: ${{ steps.pm.outputs.python-cache-key }} + python scripts/bundles/desktop.py --prepared "$RUNNER_TEMP/desktop-job/prepared.json" --variant bundled - name: Audit bundle architecture shell: bash @@ -843,6 +668,9 @@ jobs: if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} TARGET: ${{ matrix.target.label }} RELEASE_PHASE: ${{ inputs.release-phase }} RELEASE_COMMIT: ${{ needs.validate.outputs.sha }} @@ -866,6 +694,35 @@ jobs: --name "$TARGET" --root apps/desktop/release "${args[@]}" fi + build-darwin-commit: + name: bundled ${{ matrix.target.label }} (commit) + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && inputs.build_commit != '' + needs: [validate, archive-inputs] + runs-on: ${{ matrix.target.runner }} + environment: release-signing + cache-mode: read + timeout-minutes: 360 + strategy: *darwin-strategy + env: *darwin-env + steps: *darwin-steps + + build-darwin: + name: bundled darwin result + # always() crosses the deliberately skipped trust branch; failed admission + # still skips this result, just as it skipped the original native builder. + if: >- + always() && inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') + && needs.validate.result == 'success' && needs.archive-inputs.result == 'success' + needs: [validate, archive-inputs, build-darwin-release, build-darwin-commit] + runs-on: ubuntu-24.04 + permissions: {} + timeout-minutes: 5 + env: + SELECTED_BUILD_SUCCEEDED: >- + ${{ (inputs.build_commit == '' && needs.build-darwin-release.result == 'success' && needs.build-darwin-commit.result == 'skipped') + || (inputs.build_commit != '' && needs.build-darwin-commit.result == 'success' && needs.build-darwin-release.result == 'skipped') }} + steps: *native-build-result-steps + # ── Linux builders (DISABLED for now) ───────────────────────────────────── build-linux: name: bundled linux (disabled for now) @@ -938,13 +795,13 @@ jobs: const el = l.packages["apps/desktop/node_modules/electron"].version const eb = l.packages["node_modules/electron-builder"].version if (!el || !eb) process.exit(1) - console.log(`electron=${el}`) - console.log(`builder=${eb}`) + console.log("electron=" + el) + console.log("builder=" + eb) ' >> "$GITHUB_OUTPUT" # Cache reuse is optional. Bundle scripts provision the pinned SDK # and signing dependencies through electron-builder on a cache miss. - # Keep the path list identical to the build legs for warm reuse. + # This smaller consumer has its own cache, separate from desktop inputs. - name: Resolve electron's default download cache path shell: bash run: | diff --git a/.github/workflows/pm-bundle.yml b/.github/workflows/pm-bundle.yml index 7e3d071c9b..09e46d021a 100644 --- a/.github/workflows/pm-bundle.yml +++ b/.github/workflows/pm-bundle.yml @@ -34,6 +34,8 @@ on: paths: - 'pm/**' - 'scripts/bundles/**' + - 'scripts/ci/desktop_build_cache.py' + - '.github/actions/desktop-build-cache/**' - 'scripts/windows-build-deps.ps1' - 'scripts/build/windows-deps.ps1' - 'scripts/build/windows_deps.py' @@ -81,60 +83,47 @@ jobs: ref: ${{ inputs.ref || github.sha }} fetch-tags: true - # The host and payload toolchains use the same PM pins and installer. - - uses: ./.github/actions/setup-pm - id: pm + - name: Restore native dependency candidates + id: native-cache + uses: ./.github/actions/desktop-build-cache with: - save-python-cache: false + phase: restore + source: ${{ github.workspace }} + work: ${{ runner.temp }}/payload-job + cache: ${{ runner.temp }}/payload-inputs + producer: payload-test - # One cache for the pm store: keyed on the lockfile, so a pin bump - # rotates it. pm verifies every restored entry against the lock - # (hash-named fetches, verify() on entries) — the cache is an - # optimization, never a proof. - - name: Cache pm store - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: build/agent-payload/tools - key: pm-store-v2-${{ matrix.target.label }}-${{ hashFiles('pm/lock.json', 'uv.lock') }} - - # win32-arm64 builds cryptography and obstore from sdist. - - name: Set up native ARM64 build dependencies - if: matrix.target.label == 'win32-arm64' - uses: ./.github/actions/setup-windows-build-deps - - - name: Pin CMake < 4 for sdist builds - # python-olm (matrix extra) builds libolm from sdist on non-Linux - # targets, and its libolm/CMakeLists.txt requires CMake < 3.5 - # compat (removed in CMake 4, which the darwin + win32 runners - # ship). Pin a CMake 3.x first on PATH for those legs so the sdist - # build configures. Linux uses the manylinux wheel — no build, no - # cmake needed. The pip cmake package ships a binary wheel for - # every non-Linux target (macos universal2, win_amd64, win_arm64). - if: startsWith(matrix.target.label, 'darwin-') || startsWith(matrix.target.label, 'win32-') - shell: bash - run: | - python -m scripts.ci.python_packages cmake==3.31.6 - - - name: Stage the payload + - name: Prepare native payload dependencies + id: prepare shell: bash env: - # Windows runners default to cp1252; pm prints UTF-8 (✓/✗). + PAYLOAD_WORK: ${{ runner.temp }}/payload-job + PAYLOAD_CACHE: ${{ runner.temp }}/payload-inputs PYTHONUTF8: '1' run: | - # Archive what actions/checkout actually checked out. On - # pull_request events github.sha names a merge commit that a - # force-push invalidates mid-run ("not a tree object"). - python -m pm.cli bundle \ - --out build/agent-payload \ - --ref HEAD + if [ "$RUNNER_OS" = Windows ]; then bootstrap=python; else bootstrap=python3; fi + "$bootstrap" -S -B scripts/bundles/native_build.py --prepare-only \ + --source "$GITHUB_WORKSPACE" --work "$PAYLOAD_WORK" --cache "$PAYLOAD_CACHE" \ + --out "$GITHUB_WORKSPACE/build/agent-payload" --ref HEAD - - name: Save warmed Python dependencies even after a staging failure - if: ${{ !cancelled() && steps.pm.outcome == 'success' }} - uses: ./.github/actions/save-pm-cache + # PRs can restore candidates but never gain a cache-writing lane here. + - name: Save native dependency candidates before assembly + if: ${{ !cancelled() && steps.prepare.outcome == 'success' && github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && (inputs.ref == '' || inputs.ref == github.sha) }} + uses: ./.github/actions/desktop-build-cache with: - python: ${{ steps.pm.outputs.python-path }} - path: ${{ steps.pm.outputs.uv-cache-path }} - key: ${{ steps.pm.outputs.python-cache-key }} + phase: save + source: ${{ github.workspace }} + work: ${{ runner.temp }}/payload-job + cache: ${{ runner.temp }}/payload-inputs + producer: payload-test + key: ${{ steps.native-cache.outputs.cache-key }} + + - name: Assemble the prepared payload without dependency acquisition + shell: bash + run: | + if [ "$RUNNER_OS" = Windows ]; then bootstrap=python; else bootstrap=python3; fi + "$bootstrap" -S -B scripts/bundles/native_build.py \ + --prepared "$GITHUB_WORKSPACE/build/agent-payload.prepared.json" # The generated command must survive relocation without checkout env/cwd. - name: Smoke test the relocated payload (network disabled) diff --git a/.gitignore b/.gitignore index 4b38b7e709..84908bbd73 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,7 @@ *-snapshots/ .act-sandbox-agent.* .build/ +.cache/desktop-inputs/ .bytecode-fingerprint .bytecode-fingerprint.tmp .codex/ diff --git a/apps/desktop/BUILDING.md b/apps/desktop/BUILDING.md index 8eaa525a21..6e02dc3a69 100644 --- a/apps/desktop/BUILDING.md +++ b/apps/desktop/BUILDING.md @@ -69,39 +69,77 @@ Do not refresh facts in `afterSign`: that changes resources covered by the signa ## Complete native build -From a checkout whose `HEAD` equals the release tag, run: +From a clean checkout whose `HEAD` equals the release tag, run: ```sh -uv run --no-project --python 3.14 python scripts/bundles/desktop.py --tag=vX.Y.Z +python scripts/bundles/desktop.py --tag=vX.Y.Z ``` Replace `vX.Y.Z` with an actual immutable tag. Stable tags must match the version in `pyproject.toml`. Canary tags use the release script's tag grammar. -The host must provide Git, native-architecture Node/npm, and official uv 0.12+ -with a build triple in `uv --version`. Native dependency builds also need the -platform's compiler and libraries. +Start with host Python 3.11+ and Git. Use `python3` if that is your host's command. +Preparation asks PM for the pinned Python, Node, npm and private installer; +preinstalling a separate Node/npm/uv toolchain is not required. Native dependency +builds still need the platform's compiler, SDK and libraries. Windows ARM64 uses +the shared Visual Studio/Clang/Rust/static OpenSSL preparation provider, which +can require Administrator permissions for missing system components. macOS +requires its native developer tools. This is not a hermetic host SDK. The builder: -1. Checks the tag, checkout, Node architecture, and workspace engine constraints. -2. Installs the locked root JS workspace when its install stamp differs. -3. Builds the TUI and dashboard for variants with a payload. -4. Stages the PM payload, places JS assets, relocates links, and generates launchers. -5. Builds Electron and packages MSIX, DMG/ZIP, or AppImage for the current OS. +1. Admits the clean source revision and prepares managed tools in isolated build state. +2. Prepares the locked JS workspace union, icon environment, Electron-native bindings, + packaging utilities, and the application/independent PM environments when selected. +3. Compiles icons and the selected TUI, dashboard and desktop products. +4. Assembles the PM payload, places JS assets, relocates links, and generates launchers. +5. Consumes the prepared Electron archive and utilities to package the current OS. -Use `--variant store` on Windows, or `--variant light` for the remote client. -Arguments after `--` go to Electron Builder. `electron-builder.config.cjs` +Use `--variant store` with a stable tag on Windows, or `--variant light` for the remote client. +Arguments after `--` go to the prepared wrapper, which rejects overrides of +the admitted target, tools, output and configuration. `electron-builder.config.cjs` is the sole packaging configuration. The wrapper disables automatic publishing; the release workflow owns uploads and channel promotion. `pm bundle --out DIR --ref REF` stages the native runtime only. -`scripts/bundles/stage.py` also generates its launchers. Neither command builds -the TUI/dashboard outputs or creates a signed installer by itself. +`scripts/bundles/stage.py` also builds the TUI/dashboard unless both products +are supplied explicitly. Both generate launchers; neither creates an Electron installer. The launcher stage checks the payload and records its relative launch paths. The Electron build bakes these paths into its stamp. Desktop startup does not inspect, create, or repair a PM payload. Non-bundled builds carry no placeholder payload. See [shared bundle builds](../../docs/shared-bundle-builds.md) for Termux reuse. +### Prepare once, then build + +CI and local builds share the same split interface: + +```sh +python scripts/bundles/desktop.py --tag vX.Y.Z --variant bundled --prepare-only \ + --work "$PWD/.build/desktop-job" --cache "$PWD/.cache/desktop-inputs" +python scripts/bundles/desktop.py --prepared "$PWD/.build/desktop-job/prepared.json" +``` + +The displayed paths are the defaults. Use separate build-owned work/cache roots; +do not pre-create the work directory. `prepared.json` is published after every +provider succeeds and contains absolute paths for this job, not a portable cache. +After relocation or a source/lock/tool change, prepare again. Consumption rejects +missing or changed dependencies without repairing or downloading them. Stable +Windows builds can consume one preparation for `--variant bundled` and then +`--variant store`; light and commit preparations cannot switch to Store. + +Release jobs restore candidates, prepare, save reusable inputs, and only then +build/sign. Job-local environments, credentials, products and signature results +are not dependency snapshot inputs. The signature cache retains its own lifetime. +Commit jobs request token-enforced read-only cache access; the conditional YAML +mode still needs GitHub acceptance (see the +[cache-policy caveat](../../docs/shared-bundle-builds.md#cache-ownership)). A +separate key or skipped save alone would not protect release caches. Archival remains an independent +prerequisite, and R2 upload credentials are not exposed to desktop preparation. + +Strict consumption means no dependency acquisition, not offline signing. +Timestamp services, Azure signing, Apple notarization and publication remain +online operations. Validate unsigned packaging with dependency networking denied +on each target, then verify signed installers and launchers on their native hosts. + ## Commit-only builds To preview a build for a pushed revision, run: @@ -153,7 +191,7 @@ It rejects mixed tag, release-phase, channel-publication, and upgrade inputs. Builder jobs check out the admitted SHA. Their artifacts and completion receipts go to `releases/commit/FULL_SHA/`, separate from tag archives and update channels. -The run summary lists Windows packages and both universal bundles, macOS DMG/ZIP +The run summary lists Windows sideload packages and their universal bundle, macOS DMG/ZIP files, and the Termux package. Linux release legs remain disabled and are listed as not built. Only receipt-listed artifacts that exist in storage get download links. Missing receipts show the failed or incomplete leg. @@ -166,8 +204,8 @@ staged, and a re-run of an older tag never replaces a newer channel page. `release.py --build-commit` prints the commit page URL before dispatching. Commit builds require the signing credentials used by their release legs. -Store bundle envelopes remain unsigned for Partner Center, but these builds -never submit them. No GitHub release, updater feed, or APT channel is changed. +They do not produce Store packages or submit to Partner Center. No GitHub +release, updater feed, or APT channel is changed. An identical upload retry can succeed. Different bytes at an existing commit object key fail rather than replace that object. @@ -178,8 +216,8 @@ python scripts/bundles/desktop.py --commit=FULL_SHA --variant=bundled ``` The builder uses that commit's project version. Sideload MSIX versions append -`.0`. Store package versions use the commit timestamp with the existing UTC -calendar policy, even when the app version starts with zero. +`.0`. The shared MSIX version helper can derive Store versions from commit timestamps, +but this desktop preparation interface only admits Store packaging for stable tags. Commit-built stamps disable automatic release-channel updates. Local command and transport tests do not replace signed-package installation and update acceptance on each native host. @@ -273,7 +311,10 @@ The queries use noninteractive `sudo` when available. Permission failures and query timeouts are reported explicitly. Empty output does not prove that the image has no holder. The shim does not stop processes or change detach results, retry settings, signing or notarization. Explicit `CUSTOM_DMGBUILD_PATH` -overrides bypass the shim because their interpreter layout is not known. +overrides outside the prepared path are not an escape hatch for strict builds. +Prepared packaging supplies the admitted dmgbuild path to the diagnostics shim. +Changes to its preparation provider need native DMG/detach and +signing/notarization acceptance; a successful download is not native execution proof. ## Development, assets, and verification diff --git a/apps/desktop/scripts/before-pack.mjs b/apps/desktop/scripts/before-pack.mjs index 6bef32e31b..960af7bb7a 100644 --- a/apps/desktop/scripts/before-pack.mjs +++ b/apps/desktop/scripts/before-pack.mjs @@ -41,16 +41,8 @@ * resolve rather than throw — worst case electron-builder hits the original * ENOENT, which is no worse than not having this hook at all. * - * 2. Re-stages node-pty's native files for the ACTUAL target platform/arch - * of this pack. `npm run build` already staged node-pty once for the - * host machine (see scripts/stage-native-deps.mjs), which is correct for - * single-arch builds matching the host. But electron-builder can target - * a different arch than the host (cross-build), or pack multiple archs - * from one `npm run build` (e.g. `dist:mac` => x64 + arm64). Only this - * hook knows the real per-target arch, via `context.arch` / - * `context.electronPlatformName` — so it re-stages on top of whatever - * `npm run build` left behind, per target, right before files are read - * for packing. + * 2. Copies the target's admitted native tree. Acquisition belongs to native + * preparation before packaging, never this hook. * * electron-builder passes a context with: * - appOutDir: the unpacked app directory about to be staged @@ -60,8 +52,9 @@ import { existsSync, rmSync, renameSync } from 'node:fs' import path from 'node:path' import { Arch } from 'electron-builder' -import { stageNodePty, stageGetWindows } from './stage-native-deps.mjs' +import { copyNativeInputs } from './prepared-native-deps.mjs' +/** @param {string | null | undefined} appOutDir @returns {boolean} */ export function cleanStaleAppOutDir(appOutDir) { if (!appOutDir || typeof appOutDir !== 'string') { return false @@ -87,6 +80,7 @@ export function cleanStaleAppOutDir(appOutDir) { * A rename failure (AV holding a handle) also returns false — the wipe is the * safe fallback and matches pre-#69179 behavior exactly. */ +/** @param {string | null | undefined} appOutDir @param {string} [productExeName] @returns {boolean} */ export function preserveRollbackBackup(appOutDir, productExeName = 'Hermes.exe') { if (!appOutDir || typeof appOutDir !== 'string' || !existsSync(appOutDir)) { return false @@ -105,6 +99,7 @@ export function preserveRollbackBackup(appOutDir, productExeName = 'Hermes.exe') } } +/** @param {import("app-builder-lib").BeforePackContext} context @returns {Promise} */ export default async function beforePack(context) { const appOutDir = context && context.appOutDir const platformName = context && context.electronPlatformName @@ -125,29 +120,11 @@ export default async function beforePack(context) { console.warn(`[before-pack] could not clean ${appOutDir} (${err.message}); continuing`) } - try { - const platform = context && context.electronPlatformName - const archName = context && typeof context.arch === 'number' ? Arch[context.arch] : undefined - if (platform && archName) { - if (archName === 'universal') { - console.warn( - '[before-pack] target arch is "universal" — node-pty has no universal prebuild; ' + - 'staged binary will be whichever single-arch copy npm run build left behind. ' + - 'lipo-merge x64/arm64 .node files manually if you need a true universal build.' - ) - } else { - await stageNodePty({ platform, arch: archName }) - console.log(`[before-pack] re-staged node-pty for target ${platform}-${archName}`) - } - // The macOS helper is universal, while Windows bindings are arch-specific. - // Pass the target arch so an ARM64 package never stages an x64 binding. - stageGetWindows({ platform, arch: archName }) - console.log(`[before-pack] re-staged get-windows for target ${platform}-${archName}`) - } - } catch (err) { - // This one SHOULD fail the build — a missing/wrong native binary for the - // target arch means a broken package shipped to users, which is worse - // than a build that fails loudly here. - throw new Error(`[before-pack] failed to stage native deps for this target: ${err.message}`) - } + const platform = context && context.electronPlatformName + const arch = context && typeof context.arch === 'number' ? Arch[context.arch] : undefined + if (!platform || !arch) return + const app = context.packager.projectDir + const source = path.resolve(app, '../..') + const nativeDeps = process.env.HERMES_PREPARED_NATIVE_DEPS || path.join(app, 'build/native-deps') + copyNativeInputs({ source, nativeDeps, out: path.join(app, 'dist/node_modules'), platform, arch }) } \ No newline at end of file diff --git a/apps/desktop/scripts/dmgbuild-diagnostics.cjs b/apps/desktop/scripts/dmgbuild-diagnostics.cjs index 71f19356e9..5b5a5449d8 100644 --- a/apps/desktop/scripts/dmgbuild-diagnostics.cjs +++ b/apps/desktop/scripts/dmgbuild-diagnostics.cjs @@ -5,6 +5,7 @@ const { syncBuiltinESMExports } = require('node:module') const path = require('node:path') const { promisify } = require('node:util') +/** @param {typeof import("node:child_process").execFile} execFile @param {(chunk: Buffer) => void} [write] @returns {typeof import("node:child_process").execFile} */ function wrapDmgbuildExecFile(execFile, write = chunk => process.stderr.write(chunk)) { function wrapped(file, args, options, callback) { if ( @@ -12,7 +13,8 @@ function wrapDmgbuildExecFile(execFile, write = chunk => process.stderr.write(ch path.basename(file) !== 'dmgbuild' || !Array.isArray(args) || typeof options !== 'object' || - (options?.env?.CUSTOM_DMGBUILD_PATH || process.env.CUSTOM_DMGBUILD_PATH)?.trim() + ((options?.env?.CUSTOM_DMGBUILD_PATH || process.env.CUSTOM_DMGBUILD_PATH)?.trim() && + !(options?.env?.HERMES_PREPARED_PACKAGING || process.env.HERMES_PREPARED_PACKAGING)) ) { return execFile.apply(this, arguments) } diff --git a/apps/desktop/scripts/dmgbuild-diagnostics.test.mjs b/apps/desktop/scripts/dmgbuild-diagnostics.test.mjs index 62323fb800..af56a0757a 100644 --- a/apps/desktop/scripts/dmgbuild-diagnostics.test.mjs +++ b/apps/desktop/scripts/dmgbuild-diagnostics.test.mjs @@ -35,6 +35,17 @@ test('the resolved dmgbuild uses its paired Python and preserves args, results a expect(output[0].toString()).toContain('[dmg-detach]') }) +test('prepared supplier overrides retain the paired-Python diagnostic route', () => { + const calls = [] + const wrapped = wrapDmgbuildExecFile((...args) => { calls.push(args); return {} }) + const binary = path.resolve('prepared', 'dmgbuild') + wrapped(binary, ['-s', 'settings.json', 'Volume', 'out.dmg'], { env: { + CUSTOM_DMGBUILD_PATH: binary, HERMES_PREPARED_PACKAGING: '/work/prepared.json', + } }, () => {}) + expect(calls[0][0]).toBe(path.join(path.dirname(binary), 'python/bin/python3')) + expect(calls[0][2].env.PYTHONPATH).toBe(path.join(path.dirname(binary), 'python/lib')) +}) + test('promisified callers retain stdout, stderr and the real child handle', async () => { const wrapped = wrapDmgbuildExecFile(execFile) const pending = promisify(wrapped)(process.execPath, [ diff --git a/apps/desktop/scripts/prepare-dmgbuild.mjs b/apps/desktop/scripts/prepare-dmgbuild.mjs new file mode 100644 index 0000000000..84469efbd5 --- /dev/null +++ b/apps/desktop/scripts/prepare-dmgbuild.mjs @@ -0,0 +1,26 @@ +import fs from 'node:fs' +import path from 'node:path' +import { runPython } from '../../../scripts/build/python.mjs' + +/** + * PM supplies the complete vendor tree, including the diagnostic hook's Python. + * Source convenience uses the same Python entrypoint as other build helpers. + * @param {{ source: string, out: string, cache: string, binary?: string }} options + * @returns {string} + */ +export function prepareDmgbuild({ source, out, cache, binary }) { + if (!binary) { + binary = String(runPython([path.join(import.meta.dirname, 'prepare_dmgbuild.py'), + '--out', path.join(cache, 'pm-tools'), '--cache', cache], + { cwd: source, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] })).trim() + } + const vendor = path.dirname(binary) + if (path.basename(binary) !== 'dmgbuild' || !fs.existsSync(binary) || + !fs.statSync(binary).isFile() || !fs.existsSync(path.join(vendor, 'python/bin/python3'))) { + throw new Error(`PM dmgbuild launcher or paired Python is missing: ${binary}`) + } + const destination = path.join(out, 'dmgbuild') + fs.rmSync(destination, { recursive: true, force: true }) + fs.cpSync(vendor, destination, { recursive: true, verbatimSymlinks: true }) + return path.join(destination, 'dmgbuild') +} diff --git a/apps/desktop/scripts/prepare-dmgbuild.test.mjs b/apps/desktop/scripts/prepare-dmgbuild.test.mjs new file mode 100644 index 0000000000..ed29ec08be --- /dev/null +++ b/apps/desktop/scripts/prepare-dmgbuild.test.mjs @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { test } from 'vitest' +import { prepareDmgbuild } from './prepare-dmgbuild.mjs' +import { packagingTargetArch } from './prepare-packaging-tools.mjs' + +test('packager selects either same-OS payload architecture but refuses foreign OS', () => { + for (const arch of ['x64', 'arm64']) assert.equal(packagingTargetArch(`${process.platform}-${arch}`), arch) + const foreign = process.platform === 'darwin' ? 'win32' : 'darwin' + assert.throws(() => packagingTargetArch(`${foreign}-arm64`), /same-OS/) + assert.throws(() => packagingTargetArch(`${process.platform}-ia32`), /same-OS/) +}) + +test('explicit PM supplier is copied with paired Python; missing runtime fails before publication', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dmgbuild-prepare-')) + try { + const vendor = path.join(root, 'pm-tools/vendor') + const binary = path.join(vendor, 'dmgbuild') + const python = path.join(vendor, 'python/bin/python3') + fs.mkdirSync(path.dirname(python), { recursive: true }) + fs.writeFileSync(binary, '#!/bin/bash\nexit 1\n') + fs.writeFileSync(python, 'paired Python fixture') + const out = path.join(root, 'packager') + const copied = prepareDmgbuild({ source: root, out, cache: path.join(root, 'cache'), binary }) + assert.equal(copied, path.join(out, 'dmgbuild/dmgbuild')) + assert.equal(fs.readFileSync(path.join(path.dirname(copied), 'python/bin/python3'), 'utf8'), 'paired Python fixture') + fs.writeFileSync(path.join(path.dirname(copied), 'python/bin/python3'), 'build-local change') + assert.equal(fs.readFileSync(python, 'utf8'), 'paired Python fixture') + fs.rmSync(python) + assert.throws(() => prepareDmgbuild({ source: root, out, cache: root, binary }), /paired Python/) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/scripts/prepare-packaging-tools.mjs b/apps/desktop/scripts/prepare-packaging-tools.mjs new file mode 100644 index 0000000000..2d69f9fa6c --- /dev/null +++ b/apps/desktop/scripts/prepare-packaging-tools.mjs @@ -0,0 +1,134 @@ +#!/usr/bin/env node +import fs from 'node:fs' +import path from 'node:path' +import { createRequire } from 'node:module' +import { pathToFileURL } from 'node:url' +import { parseArgs } from 'node:util' +import { isMain } from './utils.mjs' +import { publishPackagingInputs } from './prepared-packaging.mjs' +import { ensureWindowsBundleTools } from './windows-bundle-tools.mjs' +import { prepareDmgbuild } from './prepare-dmgbuild.mjs' + +/** @param {string} source @param {string} name @returns {string} */ +export function pinnedPackageRoot(source, name) { + const require = createRequire(path.join(source, 'apps/desktop/package.json')) + const entry = require.resolve(name) + let directory = path.dirname(entry) + while (!fs.existsSync(path.join(directory, 'package.json'))) { + const parent = path.dirname(directory) + if (parent === directory) throw new Error(`Cannot locate installed ${name}`) + directory = parent + } + const installed = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8')) + const lock = JSON.parse(fs.readFileSync(path.join(source, 'package-lock.json'), 'utf8')) + const key = path.relative(source, directory).split(path.sep).join('/') + if (!lock.packages?.[key] || lock.packages[key].version !== installed.version || installed.name !== name) { + throw new Error(`Installed ${name} is not the source's lock-pinned package; prepare Node dependencies first`) + } + return directory +} + +/** @param {string} target @returns {'x64' | 'arm64'} */ +export function packagingTargetArch(target) { + if (target === `${process.platform}-x64`) return 'x64' + if (target === `${process.platform}-arm64`) return 'arm64' + throw new Error(`Packaging preparation requires a same-OS x64/arm64 target, got ${target}`) +} + +/** @param {string} from @param {string} to @returns {string} */ +function copyTool(from, to) { + fs.rmSync(to, { recursive: true, force: true }) + fs.cpSync(from, to, { recursive: true, verbatimSymlinks: true }) + return to +} + +/** + * Acquire bytes without signing credentials. Builder modules are loaded only + * after the explicit cache root has been selected, before their lazy state runs. + * @param {{ source: string, out: string, cache: string, target?: string, formats?: string[], dmgbuild?: string }} options + * @returns {Promise} + */ +async function preparePackagingTools({ source, out, cache, target = `${process.platform}-${process.arch}`, formats, dmgbuild }) { + source = fs.realpathSync(source) + out = path.resolve(out) + cache = path.resolve(cache) + fs.mkdirSync(out, { recursive: true }) + fs.rmSync(path.join(out, 'prepared.json'), { force: true }) + packagingTargetArch(target) + const builderRoot = pinnedPackageRoot(source, 'app-builder-lib') + pinnedPackageRoot(source, 'electron-builder') + const require = createRequire(path.join(source, 'apps/desktop/package.json')) + const config = require(path.join(source, 'apps/desktop/electron-builder.config.cjs')) + formats ??= process.platform === 'win32' ? ['msix'] : process.platform === 'darwin' ? ['dmg', 'zip'] : ['AppImage'] + if (process.env.CUSTOM_DMGBUILD_PATH) throw new Error('Preparation must select the pinned dmgbuild supplier, not CUSTOM_DMGBUILD_PATH') + const supported = process.platform === 'win32' ? ['dir', 'msix', 'zip'] : process.platform === 'darwin' ? ['dir', 'dmg', 'zip'] : ['dir', 'AppImage', 'deb', 'rpm', 'zip'] + if (formats.some(format => !supported.includes(format))) throw new Error(`Unsupported prepared package formats: ${formats.join(', ')}`) + const dmg = formats.includes('dmg') ? prepareDmgbuild({ source, out, cache, binary: dmgbuild }) : null + const previousCache = process.env.ELECTRON_BUILDER_CACHE + process.env.ELECTRON_BUILDER_CACHE = path.join(cache, 'builder') + try { + return await acquirePackagingTools({ source, out, cache, target, formats, builderRoot, config, dmgbuild: dmg }) + } finally { + if (previousCache === undefined) delete process.env.ELECTRON_BUILDER_CACHE + else process.env.ELECTRON_BUILDER_CACHE = previousCache + } +} + +/** + * @param {{ source: string, out: string, cache: string, target: string, formats: string[], builderRoot: string, config: import('app-builder-lib').Configuration, dmgbuild: string | null }} options + * @returns {Promise} + */ +async function acquirePackagingTools({ source, out, cache, target, formats, builderRoot, config, dmgbuild }) { + /** @param {string} relative */ + const load = (relative) => import(pathToFileURL(path.join(builderRoot, 'dist', relative)).href) + const [electronGet, sevenZip, icons] = await Promise.all([ + load('util/electronGet.js'), load('toolsets/7zip.js'), load('toolsets/icons.js'), + ]) + const resourcesDir = path.join(source, 'apps/desktop', config.directories?.buildResources || 'build') + const [archive, archiveTool, iconTools] = await Promise.all([ + electronGet.downloadElectronArtifactZip({ version: config.electronVersion, platformName: process.platform, arch: packagingTargetArch(target), + artifactName: 'electron', cacheDir: path.join(cache, 'electron') }), + sevenZip.getPath7za(), icons.getIconsToolsetPath(config.toolsets?.icons, resourcesDir), + ]) + const electron = copyTool(archive, path.join(out, 'electron.zip')) + /** @type {import('./prepared-packaging.mjs').PackagingToolsets} */ + const toolsets = { + sevenZip: copyTool(path.dirname(path.dirname(archiveTool)), path.join(out, 'sevenZip')), + icons: copyTool(iconTools, path.join(out, 'icons')), + } + let windows = null + if (process.platform === 'win32') { + const builder = await load('toolsets/winCodeSign.js') + const tools = await ensureWindowsBundleTools({ config, resourcesDir, signing: true, load: async () => builder, prepared: null }) + const kitRoot = copyTool(path.dirname(path.dirname(tools.makeappx)), path.join(out, 'winCodeSign')) + if (!tools.dlib || !tools.dotnetRoot) throw new Error('Windows preparation requires the ATS dlib and paired .NET runtime') + fs.cpSync(path.dirname(tools.dlib), path.join(kitRoot, path.basename(path.dirname(tools.signtool))), { recursive: true }) + const rcedit = await builder.getRceditBundle(config.toolsets?.winCodeSign, resourcesDir) + fs.copyFileSync(rcedit.x64, path.join(kitRoot, 'rcedit-x64.exe')) + fs.copyFileSync(rcedit.x86, path.join(kitRoot, 'rcedit-x86.exe')) + const kit = path.join(kitRoot, path.basename(path.dirname(tools.makeappx))) + windows = { makeappx: path.join(kit, 'makeappx.exe'), signtool: path.join(kit, 'signtool.exe'), + dlib: path.join(kit, 'Azure.CodeSigning.Dlib.dll'), dotnetRoot: copyTool(tools.dotnetRoot, path.join(out, 'dotnet')) } + toolsets.winCodeSign = kitRoot + } + if (formats.includes('AppImage')) { + const appimage = await load('toolsets/appimage.js') + const { Arch } = await import(pathToFileURL(path.join(builderRoot, 'dist/index.js')).href) + const tools = await appimage.getAppImageTools(config.toolsets?.appimage, Arch[packagingTargetArch(target)], resourcesDir) + toolsets.appimage = copyTool(path.dirname(tools.mksquashfs), path.join(out, 'appimage')) + } + if (formats.some(format => format === 'deb' || format === 'rpm')) { + const fpm = await load('toolsets/fpm.js') + toolsets.fpm = copyTool(path.dirname(await fpm.getFpmPath(config.toolsets?.fpm, resourcesDir)), path.join(out, 'fpm')) + } + return publishPackagingInputs({ source, out, target, formats, electron, toolsets, windows, dmgbuild }) +} + +if (isMain(import.meta.url)) { + const { values } = parseArgs({ options: { + source: { type: 'string' }, out: { type: 'string' }, cache: { type: 'string' }, target: { type: 'string' }, + format: { type: 'string', multiple: true }, dmgbuild: { type: 'string' }, + } }) + if (!values.source || !values.out || !values.cache) throw new Error('Usage: prepare-packaging-tools.mjs --source REPO --out WORK/packager --cache CACHE/packager [--target same-OS-target] [--format FORMAT] [--dmgbuild PM_BINARY]') + console.log(await preparePackagingTools({ source: values.source, out: values.out, cache: values.cache, target: values.target, formats: values.format, dmgbuild: values.dmgbuild })) +} diff --git a/apps/desktop/scripts/prepare_dmgbuild.py b/apps/desktop/scripts/prepare_dmgbuild.py new file mode 100644 index 0000000000..31dd3f1b1e --- /dev/null +++ b/apps/desktop/scripts/prepare_dmgbuild.py @@ -0,0 +1,33 @@ +"""Source-build entrypoint for the independently PM-owned DMG supplier.""" +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + + +def prepare_dmgbuild(out: Path, cache: Path) -> Path: + import pm + from pm import paths + + target = pm.current_target() + if not target.startswith("darwin-"): + raise ValueError("dmgbuild preparation requires native macOS") + store = pm.prepare_tools(["dmgbuild"], out=out, target=target, cache=cache) + package = pm.get_package("dmgbuild") + version = pm.Lockfile(paths.lockfile_path()).version(package.name) + assert version is not None # prepare_tools cannot succeed without a lock pin + binary = package.binary(store / package.store_entry(version, target), target) + assert binary is not None # dmgbuild declares its launcher + return binary + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + args = parser.parse_args() + print(prepare_dmgbuild(args.out, args.cache)) diff --git a/apps/desktop/scripts/prepared-native-deps.mjs b/apps/desktop/scripts/prepared-native-deps.mjs new file mode 100644 index 0000000000..feec7ffc13 --- /dev/null +++ b/apps/desktop/scripts/prepared-native-deps.mjs @@ -0,0 +1,55 @@ +import fs from 'node:fs' +import path from 'node:path' +import { createHash } from 'node:crypto' +import { fileDigest, treeDigest, preparationRequired } from './prepared-packaging.mjs' + +/** @typedef {{ source: string, nativeDeps: string, platform?: string, arch?: string, nativeToolchain?: string }} NativeSelection */ +/** @param {string} source @returns {string} */ +function nativeIdentity(source) { + return createHash('sha256').update(JSON.stringify([ + fileDigest(path.join(source, 'package-lock.json')), + fileDigest(path.join(source, 'apps/desktop/package.json')), + fileDigest(path.join(import.meta.dirname, 'stage-native-deps.mjs')), + fileDigest(path.join(import.meta.dirname, 'prepared-native-deps.mjs')), + ])).digest('hex') +} + +/** + * The sidecar stays outside node_modules so it never ships in the application. + * @param {{ source: string, out: string, platform: string, arch: string, nativeToolchain?: string }} inputs + * @returns {void} + */ +export function recordNativeInputs({ source, out, platform, arch, nativeToolchain }) { + fs.writeFileSync(`${out}.prepared.json`, JSON.stringify({ + schema: 1, source: fs.realpathSync(source), out: fs.realpathSync(out), + platform, arch, nativeToolchain, identity: nativeIdentity(source), digest: treeDigest(out), + }) + '\n') +} + +/** @param {NativeSelection} inputs @returns {string} */ +export function readNativeInputs({ source, nativeDeps, platform = process.platform, arch = process.arch, nativeToolchain }) { + try { + const record = JSON.parse(fs.readFileSync(`${nativeDeps}.prepared.json`, 'utf8')) + if (record.schema !== 1 || record.source !== fs.realpathSync(source) || record.out !== fs.realpathSync(nativeDeps) || + record.platform !== platform || record.arch !== arch || + (nativeToolchain !== undefined && record.nativeToolchain !== nativeToolchain) || + record.identity !== nativeIdentity(source) || record.digest !== treeDigest(nativeDeps)) { + throw preparationRequired('Stale or foreign native inputs') + } + if (!fs.statSync(path.join(nativeDeps, 'node-pty/package.json')).isFile()) throw preparationRequired('Missing prepared node-pty') + return record.out + } catch (error) { + throw preparationRequired(`Cannot consume native inputs: ${error instanceof Error ? error.message : String(error)}`) + } +} + +/** @param {NativeSelection & { out: string }} inputs @returns {void} */ +export function copyNativeInputs({ out, ...inputs }) { + const nativeDeps = readNativeInputs(inputs) + const destination = path.resolve(out) + if (destination === nativeDeps || destination.startsWith(nativeDeps + path.sep) || nativeDeps.startsWith(destination + path.sep)) { + throw preparationRequired('Native input and product directories overlap') + } + fs.rmSync(destination, { recursive: true, force: true }) + fs.cpSync(nativeDeps, destination, { recursive: true, dereference: true }) +} diff --git a/apps/desktop/scripts/prepared-native-deps.test.mjs b/apps/desktop/scripts/prepared-native-deps.test.mjs new file mode 100644 index 0000000000..e54a43283c --- /dev/null +++ b/apps/desktop/scripts/prepared-native-deps.test.mjs @@ -0,0 +1,44 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { test } from 'vitest' +import * as native from './prepared-native-deps.mjs' +import beforePack from './before-pack.mjs' + +test('beforePack refuses absent native preparation rather than staging from npm', async () => { + const source = fs.mkdtempSync(path.join(os.tmpdir(), 'native-hook-')) + try { + await assert.rejects(beforePack({ appOutDir: '', electronPlatformName: 'linux', arch: 1, + packager: { projectDir: path.join(source, 'apps/desktop') } }), /run preparation again/) + } finally { + fs.rmSync(source, { recursive: true, force: true }) + } +}) + +test('native consumption copies admitted inputs and rejects a different target or changed binding', () => { + const source = fs.mkdtempSync(path.join(os.tmpdir(), 'prepared-native-')) + try { + const out = path.join(source, 'native') + fs.mkdirSync(path.join(out, 'node-pty'), { recursive: true }) + fs.mkdirSync(path.join(source, 'apps/desktop'), { recursive: true }) + fs.writeFileSync(path.join(source, 'package-lock.json'), '{}') + fs.writeFileSync(path.join(source, 'apps/desktop/package.json'), '{}') + fs.writeFileSync(path.join(out, 'node-pty/package.json'), '{}') + const binding = path.join(out, 'node-pty/pty.node') + fs.writeFileSync(binding, 'native fixture') + native.recordNativeInputs({ source, out, platform: 'linux', arch: 'x64', nativeToolchain: 'compiler-a' }) + assert.equal(native.readNativeInputs({ source, nativeDeps: out, platform: 'linux', arch: 'x64', nativeToolchain: 'compiler-a' }), out) + assert.throws(() => native.readNativeInputs({ source, nativeDeps: out, platform: 'linux', arch: 'x64', nativeToolchain: 'compiler-b' }), /run preparation again/) + const destination = path.join(source, 'product/node_modules') + native.copyNativeInputs({ source, nativeDeps: out, out: destination, platform: 'linux', arch: 'x64' }) + fs.writeFileSync(path.join(destination, 'node-pty/pty.node'), 'product mutation') + assert.equal(fs.readFileSync(binding, 'utf8'), 'native fixture') + assert.throws(() => native.readNativeInputs({ source, nativeDeps: out, platform: 'linux', arch: 'arm64' }), /run preparation again/) + fs.writeFileSync(binding, 'corrupt') + assert.throws(() => native.copyNativeInputs({ source, nativeDeps: out, out: destination, platform: 'linux', arch: 'x64' }), /run preparation again/) + assert.equal(fs.readFileSync(path.join(destination, 'node-pty/pty.node'), 'utf8'), 'product mutation') + } finally { + fs.rmSync(source, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/scripts/prepared-packaging.mjs b/apps/desktop/scripts/prepared-packaging.mjs new file mode 100644 index 0000000000..5321f04727 --- /dev/null +++ b/apps/desktop/scripts/prepared-packaging.mjs @@ -0,0 +1,132 @@ +import fs from 'node:fs' +import path from 'node:path' +import { createHash } from 'node:crypto' + +/** @typedef {{ path: string, digest: string }} PreparedFile */ +/** @typedef {{ sevenZip: string, icons: string, winCodeSign?: string, appimage?: string, fpm?: string }} PackagingToolsets */ +/** @typedef {{ schema: number, source: string, out: string, identity: string, target: string, formats: string[], electron: string, toolsets: PackagingToolsets, windows: import('./windows-bundle-tools.mjs').WindowsBundleTools | null, dmgbuild: string | null, files: PreparedFile[] }} PreparedPackaging */ + +/** @param {string} message @returns {Error} */ +export function preparationRequired(message) { + return new Error(`${message}; run preparation again`) +} + +/** @param {string} file @returns {string} */ +export function fileDigest(file) { + return createHash('sha256').update(fs.readFileSync(file)).digest('hex') +} + +/** + * Include symlink destinations and modes, not timestamps. A copied supplier + * may use internal symlinks; external links would make its receipt incomplete. + * @param {string} root + * @returns {string} + */ +export function treeDigest(root) { + const hash = createHash('sha256') + const canonicalRoot = fs.realpathSync(root) + /** @param {string} entry @returns {void} */ + function visit(entry) { + const relative = path.relative(root, entry) + const stat = fs.lstatSync(entry) + hash.update(JSON.stringify([relative, stat.mode & 0o777])) + if (stat.isSymbolicLink()) { + const target = fs.realpathSync(entry) + if (target !== canonicalRoot && !target.startsWith(canonicalRoot + path.sep)) { + throw preparationRequired(`Prepared input contains an external link: ${entry}`) + } + hash.update(fs.readlinkSync(entry)) + } else if (stat.isDirectory()) { + for (const name of fs.readdirSync(entry).sort()) visit(path.join(entry, name)) + } else if (stat.isFile()) { + hash.update(fileDigest(entry)) + } else { + throw preparationRequired(`Unsupported prepared input: ${entry}`) + } + } + visit(root) + return hash.digest('hex') +} + +/** @param {string} source @returns {string} */ +export function packagingIdentity(source) { + const files = ['package-lock.json', 'apps/desktop/package.json', 'apps/desktop/electron-builder.config.cjs'] + const recipe = ['prepare-packaging-tools.mjs', 'prepared-packaging.mjs', 'prepare-dmgbuild.mjs', 'prepare_dmgbuild.py', 'windows-bundle-tools.mjs', 'run-electron-builder.mjs'] + return createHash('sha256').update(JSON.stringify([ + ...files.map(file => fileDigest(path.join(source, file))), + ...recipe.map(file => fileDigest(path.join(import.meta.dirname, file))), + fileDigest(path.join(import.meta.dirname, '../../../pm/lock.json')), + ])).digest('hex') +} + +/** @param {string} root @param {string} file @returns {void} */ +function assertOwned(root, file) { + const canonicalRoot = fs.realpathSync(root) + const canonicalFile = fs.realpathSync(file) + if (!path.isAbsolute(file) || !canonicalFile.startsWith(canonicalRoot + path.sep)) { + throw preparationRequired(`Prepared input is outside its work directory: ${file}`) + } +} + +/** + * Publish only after every selected supplier completed. The receipt is job-local, + * not a cache attestation; trusted cache writers remain a prerequisite. + * @param {{ source: string, out: string, target: string, formats: string[], electron: string, toolsets: PackagingToolsets, windows?: import('./windows-bundle-tools.mjs').WindowsBundleTools | null, dmgbuild?: string | null }} inputs + * @returns {Promise} + */ +export async function publishPackagingInputs(inputs) { + const out = fs.realpathSync(inputs.out) + const paths = [inputs.electron, ...Object.values(inputs.toolsets)] + if (inputs.windows?.dotnetRoot) paths.push(inputs.windows.dotnetRoot) + if (inputs.dmgbuild) paths.push(path.dirname(inputs.dmgbuild)) + const files = paths.map(file => { + assertOwned(out, file) + return { path: file, digest: treeDigest(file) } + }) + /** @type {PreparedPackaging} */ + const result = { + schema: 1, source: fs.realpathSync(inputs.source), out, + identity: packagingIdentity(inputs.source), target: inputs.target, + formats: inputs.formats, electron: inputs.electron, toolsets: inputs.toolsets, + windows: inputs.windows ?? null, dmgbuild: inputs.dmgbuild ?? null, files, + } + const manifest = path.join(out, 'prepared.json') + fs.writeFileSync(`${manifest}.tmp`, JSON.stringify(result, null, 2) + '\n') + fs.renameSync(`${manifest}.tmp`, manifest) + return manifest +} + +/** + * Read-only admission. No supplier or installer may be imported on this path. + * @param {string} manifest + * @param {string} source + * @param {string} [target] + * @returns {PreparedPackaging} + */ +export function readPackagingInputs(manifest, source, target = `${process.platform}-${process.arch}`) { + try { + /** @type {PreparedPackaging} */ + const result = JSON.parse(fs.readFileSync(manifest, 'utf8')) + if (result.schema !== 1 || result.source !== fs.realpathSync(source) || result.out !== fs.realpathSync(path.dirname(manifest)) || + result.target !== target || result.identity !== packagingIdentity(source)) { + throw preparationRequired('Stale or foreign packaging inputs') + } + const required = [result.electron, result.toolsets.sevenZip, result.toolsets.icons, ...Object.values(result.toolsets)] + if (target.startsWith('win32-')) { + if (!result.windows || !result.toolsets.winCodeSign || !result.windows.dotnetRoot) throw preparationRequired('Missing Windows tool selection') + required.push(result.windows.dotnetRoot) + } + if (result.formats.includes('dmg')) { + if (!result.dmgbuild) throw preparationRequired('Missing prepared dmgbuild') + required.push(path.dirname(result.dmgbuild)) + } + for (const file of new Set(required)) { + assertOwned(result.out, file) + const record = result.files.find(entry => entry.path === file) + if (!record || record.digest !== treeDigest(file)) throw preparationRequired(`Missing or changed packaging input: ${file}`) + } + return result + } catch (error) { + throw preparationRequired(`Cannot consume packaging inputs ${manifest}: ${error instanceof Error ? error.message : String(error)}`) + } +} diff --git a/apps/desktop/scripts/prepared-packaging.test.mjs b/apps/desktop/scripts/prepared-packaging.test.mjs new file mode 100644 index 0000000000..784c32e921 --- /dev/null +++ b/apps/desktop/scripts/prepared-packaging.test.mjs @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, test } from 'vitest' +import * as prepared from './prepared-packaging.mjs' +import { validatePreparedBuilderArgs } from './run-electron-builder.mjs' +import { spawnSync } from 'node:child_process' + +/** @type {string[]} */ +const roots = [] +afterEach(() => roots.splice(0).forEach(root => fs.rmSync(root, { recursive: true, force: true }))) + +/** @returns {{ source: string, out: string, electron: string, sevenZip: string, icons: string }} */ +function fixture() { + const source = fs.mkdtempSync(path.join(os.tmpdir(), 'prepared-packager-')) + roots.push(source) + const out = path.join(source, 'work') + fs.mkdirSync(path.join(source, 'apps/desktop'), { recursive: true }) + fs.writeFileSync(path.join(source, 'package-lock.json'), '{}') + fs.writeFileSync(path.join(source, 'apps/desktop/package.json'), '{}') + fs.writeFileSync(path.join(source, 'apps/desktop/electron-builder.config.cjs'), 'module.exports = {}') + const electron = path.join(out, 'electron.zip') + const sevenZip = path.join(out, 'sevenZip') + const icons = path.join(out, 'icons') + fs.mkdirSync(path.join(sevenZip, 'bin'), { recursive: true }) + fs.mkdirSync(icons) + fs.writeFileSync(electron, 'verified archive fixture') + fs.writeFileSync(path.join(sevenZip, 'bin', process.platform === 'win32' ? '7za.exe' : '7za'), 'archive utility fixture') + fs.writeFileSync(path.join(icons, 'icon-tool.js'), 'icon tool fixture') + return { source, out, electron, sevenZip, icons } +} + +test('a failed preparation invalidates an earlier completion claim before loading suppliers', () => { + const { source, out } = fixture() + const manifest = path.join(out, 'prepared.json') + fs.writeFileSync(manifest, '{}') + const result = spawnSync(process.execPath, [path.join(import.meta.dirname, 'prepare-packaging-tools.mjs'), + '--source', source, '--out', out, '--cache', path.join(source, 'cache')], { encoding: 'utf8' }) + assert.notEqual(result.status, 0) + assert.equal(fs.existsSync(manifest), false) + assert.match(result.stderr, /lock|install|pinned/i) +}) + +test('prepared inputs are path-bound and reject changed or missing bytes without repair', async () => { + const inputs = fixture() + const manifest = await prepared.publishPackagingInputs({ + ...inputs, target: 'linux-x64', formats: ['dir'], + toolsets: { sevenZip: inputs.sevenZip, icons: inputs.icons }, + }) + const result = prepared.readPackagingInputs(manifest, inputs.source, 'linux-x64') + assert.equal(result.electron, inputs.electron) + assert.throws(() => validatePreparedBuilderArgs([], result), /run preparation again/) + assert.throws(() => validatePreparedBuilderArgs(['--dir', '-c.electronDist=/another.zip'], result), /run preparation again/) + validatePreparedBuilderArgs(['--dir', '-c.extraMetadata.version=1.2.3'], result) + fs.writeFileSync(inputs.electron, 'corrupt') + assert.throws(() => prepared.readPackagingInputs(manifest, inputs.source, 'linux-x64'), /run preparation again/i) + assert.equal(fs.readFileSync(inputs.electron, 'utf8'), 'corrupt') + fs.rmSync(inputs.electron) + assert.throws(() => prepared.readPackagingInputs(manifest, inputs.source, 'linux-x64'), /run preparation again/i) +}) diff --git a/apps/desktop/scripts/prepared-windows-tools.test.mjs b/apps/desktop/scripts/prepared-windows-tools.test.mjs new file mode 100644 index 0000000000..f06b9a1375 --- /dev/null +++ b/apps/desktop/scripts/prepared-windows-tools.test.mjs @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { test } from 'vitest' +import { ensureWindowsBundleTools } from './windows-bundle-tools.mjs' +import { azureSignFile } from './sign-msix.mjs' + +test('signing validates once per packager, including concurrent file hooks, never process-wide', async () => { + const previous = process.env.HERMES_PREPARED_PACKAGING + const previousDotnet = process.env.DOTNET_ROOT + process.env.HERMES_PREPARED_PACKAGING = 'operation-selection.json' + let admissions = 0 + const signed = [] + const dependencies = { + ensureTools: async () => { admissions++; return { dotnetRoot: 'prepared-dotnet' } }, + loadManager: async () => class { + async initialize() {} + async signFile({ path: file }) { signed.push(file) } + }, + } + const resources = os.tmpdir() + const packager = { config: { toolsets: { winCodeSign: { url: `file://${path.join(resources, 'prepared-packaging-tools/winCodeSign')}` } } }, buildResourcesDir: resources, platformOptions: {} } + try { + await Promise.all(['first.exe', 'second.msix'].map(file => azureSignFile(file, packager, dependencies))) + assert.equal(admissions, 1) + await azureSignFile('third.msix', { ...packager }, dependencies) + assert.equal(admissions, 2) + assert.deepEqual(signed, ['first.exe', 'second.msix', 'third.msix']) + } finally { + if (previous === undefined) delete process.env.HERMES_PREPARED_PACKAGING + else process.env.HERMES_PREPARED_PACKAGING = previous + if (previousDotnet === undefined) delete process.env.DOTNET_ROOT + else process.env.DOTNET_ROOT = previousDotnet + } +}) + +test('the per-file Azure signer admits the same prepared selection before constructing its manager', async () => { + const previous = process.env.HERMES_PREPARED_PACKAGING + process.env.HERMES_PREPARED_PACKAGING = path.join(os.tmpdir(), 'absent-signing-selection.json') + try { + await assert.rejects(azureSignFile('output.msix', { config: {}, buildResourcesDir: os.tmpdir() }), /run preparation again/) + } finally { + if (previous === undefined) delete process.env.HERMES_PREPARED_PACKAGING + else process.env.HERMES_PREPARED_PACKAGING = previous + } +}) + +test('a prepared signing selection cannot fall through to a supplier', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'prepared-windows-')) + try { + await assert.rejects(ensureWindowsBundleTools({ + prepared: path.join(root, 'missing.json'), signing: true, config: {}, + load: async () => { throw new Error('supplier must not run') }, + }), /run preparation again/i) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/scripts/probe-prepared-native.mjs b/apps/desktop/scripts/probe-prepared-native.mjs new file mode 100644 index 0000000000..e829959486 --- /dev/null +++ b/apps/desktop/scripts/probe-prepared-native.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node +import fs from 'node:fs' +import path from 'node:path' +import { createRequire } from 'node:module' +import { spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { parseArgs } from 'node:util' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { readNativeInputs } from './prepared-native-deps.mjs' +import { readPackagingInputs } from './prepared-packaging.mjs' + +/** @param {string} nativeDeps @returns {Promise} */ +async function probeChild(nativeDeps) { + if (!process.versions.electron) throw new Error('Electron runtime required') + /** @type {typeof import('node-pty')} */ + const pty = createRequire(import.meta.url)(path.join(nativeDeps, 'node-pty')) + const marker = `hermes-pty-${randomUUID()}` + const windows = process.platform === 'win32' + const shell = windows ? (process.env.ComSpec || 'cmd.exe') : '/bin/sh' + const args = windows ? ['/d', '/s', '/c', `echo ${marker}`] : ['-c', `echo ${marker}`] + const terminal = pty.spawn(shell, args, { cols: 80, rows: 24, cwd: process.cwd(), env: process.env }) + let output = '' + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + terminal.kill() + reject(new Error('Prepared Electron PTY timed out')) + }, 10000) + terminal.onData(data => { output += data }) + terminal.onExit(({ exitCode }) => { + clearTimeout(timer) + if (exitCode !== 0 || !output.includes(marker)) reject(new Error(`Prepared Electron PTY failed: exit=${exitCode}, output=${output}`)) + else resolve(undefined) + }) + }) + console.log(JSON.stringify({ electron: process.versions.electron, node: process.versions.node, + platform: process.platform, arch: process.arch, marker, exitCode: 0 })) +} + +/** + * Run every preparation: this execution receipt is job-local, never a cache hit. + * @param {{source: string, nativeDeps: string, packaging: string, out: string, nativeToolchain?: string}} options + * @returns {{electron: string, node: string, platform: string, arch: string, marker: string, exitCode: number, work: string}} + */ +export function probePreparedNative({ source, nativeDeps, packaging, out, nativeToolchain }) { + const admitted = readNativeInputs({ source, nativeDeps, nativeToolchain }) + const inputs = readPackagingInputs(packaging, source) + fs.mkdirSync(out, { recursive: true }) + const work = fs.mkdtempSync(path.join(path.resolve(out), 'electron-probe-')) + const bin = path.join(inputs.toolsets.sevenZip, 'bin') + const archiveTool = fs.readdirSync(bin).find(name => /^(7zz|7za|7z)(\.exe)?$/.test(name)) + if (!archiveTool) throw new Error('Prepared 7zip executable is missing') + const extract = spawnSync(path.join(bin, archiveTool), ['x', '-y', `-o${work}`, inputs.electron], { + encoding: 'utf8', timeout: 60000, stdio: ['ignore', 'pipe', 'pipe'], + }) + if (extract.error || extract.status !== 0) throw new Error(`Electron extraction failed: ${extract.error?.message || extract.stderr}`) + const executable = new Map([['win32', 'electron.exe'], ['darwin', 'Electron.app/Contents/MacOS/Electron'], ['linux', 'electron']]).get(process.platform) + if (!executable) throw new Error(`Unsupported native probe platform: ${process.platform}`) + const child = spawnSync(path.join(work, executable), [fileURLToPath(import.meta.url), '--child', admitted], { + cwd: work, encoding: 'utf8', timeout: 15000, killSignal: 'SIGKILL', + env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, stdio: ['ignore', 'pipe', 'pipe'], + }) + if (child.error || child.status !== 0) throw new Error(`Prepared Electron PTY admission failed: ${child.error?.message || child.stderr}`) + const result = JSON.parse(child.stdout.trim()) + if (!result.electron || result.platform !== process.platform || result.arch !== process.arch || result.exitCode !== 0) { + throw new Error('Prepared Electron PTY returned an invalid admission result') + } + fs.writeFileSync(path.join(work, 'result.json'), JSON.stringify({ ...result, nativeToolchain, nativeDeps: admitted }, null, 2) + '\n') + return { ...result, work } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + if (process.argv[2] === '--child') { + await probeChild(path.resolve(process.argv[3])) + } else { + const { values } = parseArgs({ options: { + source: { type: 'string' }, 'native-deps': { type: 'string' }, packaging: { type: 'string' }, + out: { type: 'string' }, 'native-toolchain': { type: 'string' }, + } }) + if (!values.source || !values['native-deps'] || !values.packaging || !values.out) { + throw new Error('--source, --native-deps, --packaging and --out are required') + } + console.log(JSON.stringify(probePreparedNative({ source: values.source, nativeDeps: values['native-deps'], + packaging: values.packaging, out: values.out, nativeToolchain: values['native-toolchain'] }))) + } +} diff --git a/apps/desktop/scripts/probe-prepared-native.test.mjs b/apps/desktop/scripts/probe-prepared-native.test.mjs new file mode 100644 index 0000000000..fea41eca9a --- /dev/null +++ b/apps/desktop/scripts/probe-prepared-native.test.mjs @@ -0,0 +1,12 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import { test } from 'vitest' + +test('the native admission child refuses ordinary Node instead of certifying an Electron ABI', () => { + const child = spawnSync(process.execPath, [path.join(import.meta.dirname, 'probe-prepared-native.mjs'), '--child', import.meta.dirname], { + encoding: 'utf8', timeout: 5000, env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, + }) + assert.notEqual(child.status, 0) + assert.match(child.stderr, /Electron runtime required/) +}) diff --git a/apps/desktop/scripts/run-electron-builder.mjs b/apps/desktop/scripts/run-electron-builder.mjs index 7f9ea8a40f..806e1d40be 100644 --- a/apps/desktop/scripts/run-electron-builder.mjs +++ b/apps/desktop/scripts/run-electron-builder.mjs @@ -1,103 +1,162 @@ -// Wraps the electron-builder CLI so the arguments compose in one place, in -// the first spawn with no shell in between. -// -// electron-builder downloads and extracts Electron itself (via electronVersion -// + ELECTRON_MIRROR). Passing the local Electron dist makes v27 copy framework -// links as regular files. Its archive extraction preserves those links. - +// Source convenience prepares first; --prepared only admits and consumes files. import fs from 'node:fs' import path from 'node:path' import { spawnSync } from 'node:child_process' import { createRequire } from 'node:module' +import { isMain } from './utils.mjs' +import { readPackagingInputs, preparationRequired } from './prepared-packaging.mjs' +import { readNativeInputs } from './prepared-native-deps.mjs' +import { pinnedPackageRoot } from './prepare-packaging-tools.mjs' -const require = createRequire(import.meta.url) +const source = path.resolve(import.meta.dirname, '../../..') +const app = path.join(source, 'apps/desktop') +const platformFlags = new Map([['--win', 'win32'], ['-w', 'win32'], ['--windows', 'win32'], + ['--mac', 'darwin'], ['--macos', 'darwin'], ['-m', 'darwin'], ['-o', 'darwin'], ['--linux', 'linux'], ['-l', 'linux']]) +const architectures = ['--x64', '--arm64', '--ia32', '--armv7l', '--universal'] -function electronBuilderCli() { - const entry = require.resolve('electron-builder') - let dir = path.dirname(entry) - while (!fs.existsSync(path.join(dir, 'package.json'))) { - const parent = path.dirname(dir) - if (parent === dir) { - throw new Error('electron-builder package root not found') +/** @param {string[]} args @param {string} name @returns {string | undefined} */ +function takeOption(args, name) { + const index = args.findIndex(arg => arg === name || arg.startsWith(`${name}=`)) + if (index < 0) return undefined + const [option] = args.splice(index, 1) + const value = option.includes('=') ? option.slice(name.length + 1) : args.splice(index, 1)[0] + if (!value || value.startsWith('-')) throw new Error(`${name} requires a path`) + return path.resolve(value) +} + +/** @param {string[]} args @param {import('./prepared-packaging.mjs').PreparedPackaging} inputs @returns {void} */ +export function validatePreparedBuilderArgs(args, inputs) { + const requested = sourceFormats(args) + if (requested.some(format => format !== 'dir' && !inputs.formats.includes(format))) { + throw preparationRequired(`Package formats were not prepared: ${requested.join(', ')}`) + } + for (let index = 0; index < args.length; index++) { + const arg = args[index] + if (platformFlags.has(arg)) { + if (!inputs.target.startsWith(`${platformFlags.get(arg)}-`)) throw preparationRequired(`Conflicting package platform: ${arg}`) + continue } - dir = parent - } - const bin = require(path.join(dir, 'package.json')).bin - const rel = typeof bin === 'string' ? bin : bin['electron-builder'] - return path.join(dir, rel) -} - -// Resolve electronDist at runtime (#38673, #47917): electron-builder 26.8.x can -// re-unpack a broken Electron.app; reusing the installed dist dodges that. -// npm workspace hoisting is non-deterministic — require.resolve finds electron -// wherever it landed. -function electronDistDir() { - try { - return path.join(path.dirname(require.resolve('electron/package.json')), 'dist') - } catch { - return null + if (architectures.includes(arg)) { + if (!inputs.target.endsWith(`-${arg.slice(2)}`)) throw preparationRequired(`Conflicting package architecture: ${arg}`) + continue + } + if (arg === '--publish' || arg === '-p') { + if (args[++index] !== 'never') throw new Error('Publishing belongs to the release adapter') + continue + } + if (arg === '--dir' || arg === '--publish=never' || arg === '--') continue + if (/^(?:-c|--config)\.(?:extraMetadata\.(?:version|shortVersion|shortVersionWindows)|directories\.output|mac\.identity)=/.test(arg)) continue + if (!arg.startsWith('-') && inputs.formats.includes(arg)) continue + throw preparationRequired(`Argument is not admitted by prepared packaging: ${arg}`) } } -function distBinary(dist) { +/** + * Copy mutable toolsets into the build resources directory: upstream's custom + * Windows tool contract requires containment there. Never chmod/sign the cache. + * @param {import('./prepared-packaging.mjs').PreparedPackaging} inputs + * @returns {string[]} + */ +function toolsetArguments(inputs) { + const require = createRequire(path.join(source, 'apps/desktop/package.json')) + const config = require(path.join(app, 'electron-builder.config.cjs')) + const tools = path.join(app, config.directories?.buildResources || 'build', 'prepared-packaging-tools') + fs.mkdirSync(tools, { recursive: true }) + return Object.entries(inputs.toolsets).map(([name, directory]) => { + const destination = path.join(tools, name) + fs.rmSync(destination, { recursive: true, force: true }) + fs.cpSync(directory, destination, { recursive: true, verbatimSymlinks: true }) + // Upstream parses this as a literal path after slicing file:// (not URL decoding). + return `-c.toolsets.${name}.url=file://${destination}` + }) +} + +/** + * Prepare one isolated input set per source target; the strict child cannot acquire. + * @param {string[]} args + * @param {string | undefined} nativeDeps + * @param {typeof spawnSync} spawn + * @returns {number} + */ +function runSourceBuilds(args, nativeDeps, spawn) { + const platform = selectedPlatform(args) + const requested = [...new Set(args.filter(arg => architectures.includes(arg)))] + if (requested.includes('--universal')) throw new Error('No prepared universal native payload; use --x64 --arm64 for separate packages') + if (nativeDeps && requested.length > 1) throw new Error('--native-deps selects one architecture, not multiple source targets') + for (const flag of requested.length ? requested : [`--${process.arch}`]) { + const arch = flag.slice(2) + const target = `${platform}-${arch}` + const out = path.join(app, 'build/packager', target) + const native = nativeDeps || path.join(app, requested.length ? `build/native-deps-${target}` : 'build/native-deps') + const commands = [] + if (!nativeDeps && (requested.length || platform !== process.platform || !fs.existsSync(`${native}.prepared.json`))) { + commands.push([path.join(import.meta.dirname, 'stage-native-deps.mjs'), '--source', source, + '--out', native, '--platform', platform, '--arch', arch]) + } + commands.push([path.join(import.meta.dirname, 'prepare-packaging-tools.mjs'), + '--source', source, '--out', out, '--target', target, '--cache', path.join(source, '.cache/desktop-inputs/packager'), + ...sourceFormats(args).flatMap(format => ['--format', format])]) + commands.push([path.join(import.meta.dirname, 'run-electron-builder.mjs'), + '--prepared', path.join(out, 'prepared.json'), '--native-deps', native, + ...args.filter(arg => !architectures.includes(arg)), flag]) + for (const command of commands) { + const result = spawn(process.execPath, command, { cwd: app, stdio: 'inherit' }) + if (result.error) throw result.error + if (result.status !== 0) return result.status ?? 1 + } + } + return 0 +} + +/** @param {string[]} args @returns {string} */ +function selectedPlatform(args) { + const platforms = [...new Set(args.filter(arg => platformFlags.has(arg)).map(arg => platformFlags.get(arg)))] + if (platforms.length > 1) throw preparationRequired('Select one packaging platform per invocation') + return platforms[0] || process.platform +} + +/** @param {string[]} args @param {{ spawn?: typeof spawnSync }} [options] @returns {number} */ +export function runElectronBuilder(args, { spawn = spawnSync } = {}) { + const validateOnly = args.includes('--validate-only') + args = args.filter(arg => arg !== '--validate-only') + const manifest = takeOption(args, '--prepared') + const nativeDeps = takeOption(args, '--native-deps') + if (validateOnly && !manifest) throw preparationRequired('--validate-only requires --prepared') + if (!manifest) return runSourceBuilds(args, nativeDeps, spawn) + const platform = selectedPlatform(args) + const arch = args.find(arg => architectures.includes(arg))?.slice(2) || process.arch + const inputs = readPackagingInputs(manifest, source, `${platform}-${arch}`) + validatePreparedBuilderArgs(args, inputs) + if (!nativeDeps) throw preparationRequired('--native-deps is required with --prepared') + readNativeInputs({ source, nativeDeps, platform, arch }) + if (validateOnly) return 0 + const builder = pinnedPackageRoot(source, 'electron-builder') + pinnedPackageRoot(source, 'app-builder-lib') + const require = createRequire(path.join(builder, 'package.json')) + const bin = require(path.join(builder, 'package.json')).bin['electron-builder'] + const preloads = [] if (process.platform === 'darwin') { - return path.join(dist, 'Electron.app', 'Contents', 'MacOS', 'Electron') + preloads.push('--import', path.join(import.meta.dirname, 'patch-electron-builder-mac-binary.mjs')) + preloads.push('--require', path.join(import.meta.dirname, 'dmgbuild-diagnostics.cjs')) } - if (process.platform === 'win32') { - return path.join(dist, 'electron.exe') - } - return path.join(dist, 'electron') + /** @type {NodeJS.ProcessEnv} */ + const env = { ...process.env, HERMES_PREPARED_PACKAGING: manifest, + HERMES_PREPARED_NATIVE_DEPS: nativeDeps, HERMES_PREPARED_TARGET: inputs.target } + if (inputs.dmgbuild) env.CUSTOM_DMGBUILD_PATH = inputs.dmgbuild + if (inputs.windows?.dotnetRoot) env.DOTNET_ROOT = inputs.windows.dotnetRoot + const result = spawn(process.execPath, [...preloads, path.join(builder, bin), ...args, + '--config', 'electron-builder.config.cjs', '--publish', 'never', `-c.electronDist=${inputs.electron}`, + ...toolsetArguments(inputs)], { cwd: app, stdio: 'inherit', env }) + if (result.error) throw result.error + return result.status ?? 1 } -const args = [...process.argv.slice(2)] - -// package.json has no "build" field. Name the config file or electron-builder -// would look for one and silently use defaults. -if (!args.some(a => a === '--config' || a.startsWith('--config='))) { - args.push('--config', 'electron-builder.config.cjs') +/** @param {string[]} args @returns {string[]} */ +function sourceFormats(args) { + if (args.includes('--dir')) return ['dir'] + const formats = args.filter(arg => ['dmg', 'zip', 'msix', 'AppImage', 'deb', 'rpm'].includes(arg)) + const platform = selectedPlatform(args) + return formats.length ? formats : platform === 'darwin' ? ['dmg', 'zip'] : platform === 'win32' ? ['msix'] : ['AppImage'] } -// Never let electron-builder publish. On a CI tag build it auto-detects -// GitHub and demands GH_TOKEN after the artifacts are already built. -// The release workflow uploads artifacts in its own step. (Also: the npm -// lifecycle env sets CI=1, and electron-builder treats CI=1 as a signal to -// implicitly resolve a publish target, which reads /.git/config — -// apps/desktop has no .git of its own, so pin publish to "never".) -if (!args.includes('--publish') && !args.some(a => a.startsWith('-p'))) { - args.push('--publish', 'never') -} - -// Reuse the installed Electron dist when present so a broken app is not -// re-unpacked from a fresh download; otherwise electron-builder fetches via -// @electron/get (electronVersion + ELECTRON_MIRROR). -const dist = electronDistDir() -if (dist && fs.existsSync(distBinary(dist))) { - args.push(`-c.electronDist=${dist}`) -} else { - console.warn( - '[run-electron-builder] no local electron dist; electron-builder will fetch ' + - 'via @electron/get (electronVersion + ELECTRON_MIRROR).' - ) -} - -if (args.includes('--win') && process.env.AZURE_SIGN_ENDPOINT && process.env.AZURE_CLIENT_ID) { - console.log( - `[run-electron-builder] Windows signing: Azure Trusted Signing at ${process.env.AZURE_SIGN_ENDPOINT}` - ) -} - -const preloads = [] -if (process.platform === 'darwin') { - // Install the supplier-only probe owner before electron-builder imports osx-sign. - preloads.push('--import', path.join(import.meta.dirname, 'patch-electron-builder-mac-binary.mjs')) - preloads.push('--require', path.join(import.meta.dirname, 'dmgbuild-diagnostics.cjs')) -} - -const result = spawnSync(process.execPath, [...preloads, electronBuilderCli(), ...args], { - stdio: 'inherit' -}) -if (result.error) { - console.error(`[run-electron-builder] spawn failed: ${result.error.message}`) - process.exit(1) -} -process.exit(result.status == null ? 1 : result.status) +if (isMain(import.meta.url)) process.exitCode = runElectronBuilder(process.argv.slice(2)) diff --git a/apps/desktop/scripts/run-electron-builder.test.mjs b/apps/desktop/scripts/run-electron-builder.test.mjs new file mode 100644 index 0000000000..ca96fd82fd --- /dev/null +++ b/apps/desktop/scripts/run-electron-builder.test.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict' +import { test } from 'vitest' +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import { runElectronBuilder } from './run-electron-builder.mjs' +import fs from 'node:fs' +import os from 'node:os' +import { publishPackagingInputs } from './prepared-packaging.mjs' +import { recordNativeInputs } from './prepared-native-deps.mjs' + +test('validate-only admits real prepared inputs without launching tools and rejects unsafe arguments', async () => { + const source = path.resolve(import.meta.dirname, '../../..') + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'builder-validation-')) + try { + const electron = path.join(out, 'electron.zip') + fs.writeFileSync(electron, 'fixture archive') + const toolsets = { sevenZip: path.join(out, 'sevenZip'), icons: path.join(out, 'icons') } + for (const dir of Object.values(toolsets)) fs.mkdirSync(dir) + const manifest = await publishPackagingInputs({ source, out, target: `${process.platform}-${process.arch}`, formats: ['dir'], electron, toolsets }) + const nativeDeps = path.join(out, 'native') + fs.mkdirSync(path.join(nativeDeps, 'node-pty'), { recursive: true }) + fs.writeFileSync(path.join(nativeDeps, 'node-pty/package.json'), '{}') + recordNativeInputs({ source, out: nativeDeps, platform: process.platform, arch: process.arch }) + const args = ['--validate-only', '--prepared', manifest, '--native-deps', nativeDeps, '--dir'] + const options = { spawn: () => { throw new Error('validation must not launch tools') } } + assert.equal(runElectronBuilder(args, options), 0) + assert.throws(() => runElectronBuilder([...args, '-c.npmRebuild=true'], options), /not admitted/) + assert.throws(() => runElectronBuilder(['--validate-only'], options), /--prepared/) + const cli = spawnSync(process.execPath, [path.join(import.meta.dirname, 'run-electron-builder.mjs'), ...args], { encoding: 'utf8' }) + assert.equal(cli.status, 0, cli.stderr) + } finally { + fs.rmSync(out, { recursive: true, force: true }) + } +}) + +test('source multiarch prepares isolated native and packaging inputs before each strict invocation', () => { + const calls = [] + const spawn = (_node, args) => { calls.push(args); return { status: 0 } } + assert.equal(runElectronBuilder(['--mac', '--x64', '--arm64', '--dir'], { spawn }), 0) + for (const arch of ['x64', 'arm64']) { + const native = calls.find(args => args[0].endsWith('stage-native-deps.mjs') && args.includes(arch)) + assert.ok(native) + assert.equal(native[native.indexOf('--platform') + 1], 'darwin') + const prepare = calls.find(args => args[0].endsWith('prepare-packaging-tools.mjs') && args.includes(`darwin-${arch}`)) + assert.ok(prepare) + const strict = calls.find(args => args[0].endsWith('run-electron-builder.mjs') && args.includes(`--${arch}`)) + assert.ok(strict) + assert.equal(strict[strict.indexOf('--prepared') + 1], path.join(prepare[prepare.indexOf('--out') + 1], 'prepared.json')) + assert.equal(strict[strict.indexOf('--native-deps') + 1], native[native.indexOf('--out') + 1]) + assert.equal(strict.filter(arg => ['--x64', '--arm64'].includes(arg)).length, 1) + } + assert.equal(calls.length, 6) + assert.notEqual(calls[0][calls[0].indexOf('--out') + 1], calls[3][calls[3].indexOf('--out') + 1]) + calls.length = 0 + assert.equal(runElectronBuilder(['--dir'], { spawn }), 0) + assert.equal(calls.filter(args => args[0].endsWith('prepare-packaging-tools.mjs')).length, 1) + assert.equal(calls.filter(args => args[0].endsWith('run-electron-builder.mjs')).length, 1) + assert.throws(() => runElectronBuilder(['--mac', '--universal'], { spawn }), /No prepared universal native payload/) + assert.equal(runElectronBuilder(['--mac', '--x64', '--arm64'], { spawn: () => ({ status: 7 }) }), 7) +}) + +test('strict builder refuses absent inputs before loading electron-builder', () => { + const result = spawnSync(process.execPath, [path.join(import.meta.dirname, 'run-electron-builder.mjs'), + '--prepared', path.join(import.meta.dirname, 'missing-prepared.json'), '--native-deps', 'missing', '--dir'], { encoding: 'utf8' }) + assert.notEqual(result.status, 0) + assert.match(result.stderr, /run preparation again/) + assert.doesNotMatch(result.stdout, /electron-builder\s+version/) +}) diff --git a/apps/desktop/scripts/sign-msix.mjs b/apps/desktop/scripts/sign-msix.mjs index 5a39c4d9cd..d216706587 100644 --- a/apps/desktop/scripts/sign-msix.mjs +++ b/apps/desktop/scripts/sign-msix.mjs @@ -25,6 +25,7 @@ import fs from 'node:fs' import { createRequire } from 'node:module' import path from 'node:path' import { pathToFileURL } from 'node:url' +import { ensureWindowsBundleTools } from './windows-bundle-tools.mjs' const require = createRequire(import.meta.url) @@ -45,6 +46,7 @@ const STORE_ARTIFACT_PREFIX = 'Store-' * variant is excluded too: it cannot carry an ATS signature and does not * need one. */ +/** @param {string} file @returns {boolean} */ export function shouldSignFile(file) { if (path.basename(file).startsWith(STORE_ARTIFACT_PREFIX)) return false const lower = file.toLowerCase() @@ -53,8 +55,7 @@ export function shouldSignFile(file) { /** * @param {NodeJS.ProcessEnv} [env] - * @returns {{ type: 'azure' } & Record} the - * win.sign azure configuration, composed from the environment. + * @returns {{ type: 'azure', endpoint: string | undefined, codeSigningAccountName: string | undefined, certificateProfileName: string | undefined, publisherName: string | undefined }} */ export function azureConfigFromEnv(env = process.env) { return { @@ -87,32 +88,48 @@ async function loadAzureManagerClass() { return mod.WindowsSignAzureManager } -// One manager per process: electron-builder calls the hook once per file, -// and the manager memoizes toolset downloads / dlib metadata behind it. -let managerPromise = null +// The packager owns both admission and the manager, including concurrent hooks. +const signingOperation = Symbol('hermes.azureSigningOperation') +/** @typedef {{ ensureTools?: typeof ensureWindowsBundleTools, loadManager?: typeof loadAzureManagerClass }} SigningDependencies */ -function azureManager(packager) { - if (managerPromise == null) { - managerPromise = (async () => { - const WindowsSignAzureManager = await loadAzureManagerClass() - // The manager's constructor re-derives the signing config from - // packager.platformOptions.sign and throws unless type === 'azure' — - // but our config's win.sign is the { type: 'signtool' } hook wiring. - // Shim the packager with the azure config composed from the - // environment; everything else (config.toolsets, buildResourcesDir, - // getTempFile) delegates to the real packager via the prototype. - const shim = Object.create(packager) - Object.defineProperty(shim, 'platformOptions', { - value: { ...packager.platformOptions, sign: azureConfigFromEnv() } - }) - const manager = new WindowsSignAzureManager(shim) - // No-op on the modern signtool /dlib path (winCodeSign >= 1.3.0); - // installs the legacy PowerShell module otherwise. - await manager.initialize() - return manager - })() - } - return managerPromise +/** + * @param {import('app-builder-lib').WinPackager} packager + * @param {SigningDependencies} dependencies + * @returns {Promise<{signFile: (options: {path: string, options: import('app-builder-lib').WindowsConfiguration}) => Promise}>} + */ +function azureManager(packager, { ensureTools = ensureWindowsBundleTools, loadManager = loadAzureManagerClass }) { + /** @type {{value?: ReturnType} | undefined} */ + const existing = Object.getOwnPropertyDescriptor(packager, signingOperation) + if (existing?.value) return existing.value + const operation = (async () => { + if (process.env.HERMES_PREPARED_PACKAGING) { + const tools = await ensureTools({ signing: true, config: packager.config, resourcesDir: packager.buildResourcesDir }) + const selection = packager.config.toolsets?.winCodeSign + const local = { url: `file://${path.join(packager.buildResourcesDir, 'prepared-packaging-tools/winCodeSign')}` } + if (JSON.stringify(selection) !== JSON.stringify(local)) { + throw new Error('Prepared signing requires the local Windows toolset; run preparation again') + } + process.env.DOTNET_ROOT = tools.dotnetRoot ?? undefined + } + const WindowsSignAzureManager = await loadManager() + // The manager's constructor re-derives the signing config from + // packager.platformOptions.sign and throws unless type === 'azure' — + // but our config's win.sign is the { type: 'signtool' } hook wiring. + // Shim the packager with the azure config composed from the + // environment; everything else (config.toolsets, buildResourcesDir, + // getTempFile) delegates to the real packager via the prototype. + const shim = Object.create(packager) + Object.defineProperty(shim, 'platformOptions', { + value: { ...packager.platformOptions, sign: azureConfigFromEnv() } + }) + const manager = new WindowsSignAzureManager(shim) + // No-op on the modern signtool /dlib path (winCodeSign >= 1.3.0); + // installs the legacy PowerShell module otherwise. + await manager.initialize() + return manager + })() + Object.defineProperty(packager, signingOperation, { value: operation }) + return operation } /** @@ -122,10 +139,12 @@ function azureManager(packager) { * product exe (which must be signed after rcedit, not in the afterPack batch). * * @param {string} file - * @param {any} packager WinPackager + * @param {import('app-builder-lib').WinPackager} packager + * @param {SigningDependencies} [dependencies] + * @returns {Promise} */ -export async function azureSignFile(file, packager) { - const mgr = await azureManager(packager) +export async function azureSignFile(file, packager, dependencies = {}) { + const mgr = await azureManager(packager, dependencies) // signFileWithDlib reads only options.path (plus the manager's own // signing config), so platformOptions is sufficient here. await mgr.signFile({ path: file, options: packager.platformOptions }) @@ -135,7 +154,8 @@ export async function azureSignFile(file, packager) { * The electron-builder custom sign hook. * * @param {{ path: string }} configuration CustomWindowsSignTaskConfiguration - * @param {any} packager WinPackager + * @param {import('app-builder-lib').WinPackager} packager + * @returns {Promise} */ export default async function sign(configuration, packager) { if (path.basename(configuration.path).startsWith(STORE_ARTIFACT_PREFIX)) { diff --git a/apps/desktop/scripts/stage-native-deps.mjs b/apps/desktop/scripts/stage-native-deps.mjs index 3681325b42..121a13e584 100644 --- a/apps/desktop/scripts/stage-native-deps.mjs +++ b/apps/desktop/scripts/stage-native-deps.mjs @@ -25,6 +25,7 @@ import { } from 'node:fs' import { spawnSync } from 'node:child_process' import { isMain } from './utils.mjs' +import { recordNativeInputs } from './prepared-native-deps.mjs' import { parseArgs } from 'node:util' import { productOutput, withProduct, workspaceTool } from '../../../scripts/build/frontend-common.mjs' @@ -621,14 +622,19 @@ export function stageGetWindows( return stageGetWindowsInto(srcRoot, destRoot, { platform, arch, install }) } -// Preparation may rebuild/download native bindings. The compiler only consumes -// the resulting tree, and must never call this preparation function. -export async function prepareDesktopNativeDependencies({ source, out, platform = process.platform, arch = process.arch }) { +/** + * Preparation may rebuild/download native bindings; compilation only consumes them. + * @param {{ source: string, out: string, platform?: string, arch?: string, nativeToolchain?: string }} inputs + * @returns {Promise<{out: string}>} + */ +export async function prepareDesktopNativeDependencies({ source, out, platform = process.platform, arch = process.arch, nativeToolchain }) { ;({ source, out } = productOutput(source, out, ['node_modules', 'apps/desktop/node_modules', 'apps/desktop/src', 'apps/desktop/electron'])) + rmSync(`${out}.prepared.json`, { force: true }) await withProduct(out, async product => { stageNodePty({ source, out: product, platform, arch }) stageGetWindows({ source, out: product, platform, arch }) }, { source }) + recordNativeInputs({ source, out, platform, arch, nativeToolchain }) return { out } } @@ -636,6 +642,7 @@ if (isMain(import.meta.url)) { const { values } = parseArgs({ options: { source: { type: 'string', default: resolve(projectRoot, '../..') }, out: { type: 'string' }, platform: { type: 'string', default: process.platform }, arch: { type: 'string', default: process.arch }, + 'native-toolchain': { type: 'string' }, } }) - await prepareDesktopNativeDependencies({ ...values, out: values.out || join(values.source, 'apps/desktop/build/native-deps') }) + await prepareDesktopNativeDependencies({ ...values, nativeToolchain: values['native-toolchain'], out: values.out || join(values.source, 'apps/desktop/build/native-deps') }) } diff --git a/apps/desktop/scripts/utils.mjs b/apps/desktop/scripts/utils.mjs index 7010213ec3..edc66f4f8b 100644 --- a/apps/desktop/scripts/utils.mjs +++ b/apps/desktop/scripts/utils.mjs @@ -3,6 +3,7 @@ import { pathToFileURL } from 'node:url'; // returns true if the passsed file is being invoked from node, // not imported. +/** @param {string} importMetaUrl @returns {boolean} */ export function isMain(importMetaUrl) { return importMetaUrl === pathToFileURL(process.argv[1]).href; } \ No newline at end of file diff --git a/apps/desktop/scripts/windows-bundle-tools.mjs b/apps/desktop/scripts/windows-bundle-tools.mjs index a05b10d28b..558e732bf0 100644 --- a/apps/desktop/scripts/windows-bundle-tools.mjs +++ b/apps/desktop/scripts/windows-bundle-tools.mjs @@ -5,8 +5,14 @@ import { createRequire } from 'node:module' import path from 'node:path' import { pathToFileURL } from 'node:url' +import { readPackagingInputs, preparationRequired } from './prepared-packaging.mjs' + const require = createRequire(import.meta.url) +/** @typedef {{ makeappx: string, signtool: string, dlib: string | null, dotnetRoot: string | null }} WindowsBundleTools */ +/** @typedef {{ WIN_CODESIGN_LATEST: string, getWindowsKitsBundle: (options: {winCodeSign?: NonNullable['winCodeSign'], resourcesDir: string}) => Promise<{kit: string}>, getAtsBundleDir: (version: string) => Promise, getDotnetRuntimeDir: (version: string) => Promise }} BuilderWindowsTools */ + +/** @returns {Promise} */ async function loadBuilderTools() { // app-builder-lib exports only its entry and ./internal. Resolve the // installed, lock-pinned package before loading its toolset implementation. @@ -14,15 +20,39 @@ async function loadBuilderTools() { return import(new URL('./toolsets/winCodeSign.js', entry).href) } +/** @param {import("app-builder-lib").Configuration} config @returns {string} */ +function defaultResourcesDir(config) { + return path.resolve(import.meta.dirname, "..", config.directories?.buildResources || "build") +} + +/** @param {string} manifest @param {string} source @param {boolean} signing @param {string | undefined} target @returns {WindowsBundleTools} */ +function consumeWindowsBundleTools(manifest, source, signing, target) { + const result = readPackagingInputs(manifest, source, target).windows + if (!result || (signing && (!result.dlib || !result.dotnetRoot))) throw preparationRequired('Missing prepared Windows signing tools') + for (const file of [result.makeappx, result.signtool, ...(signing ? [result.dlib] : [])]) { + if (!file || !fs.statSync(file).isFile()) throw preparationRequired(`Missing prepared Windows tool: ${file}`) + } + return result +} + +/** + * @param {{ signing?: boolean, config?: import('app-builder-lib').Configuration, resourcesDir?: string, load?: () => Promise, prepared?: string | null, source?: string, target?: string }} [options] + * @returns {Promise} + */ export async function ensureWindowsBundleTools({ signing = false, config = require('../electron-builder.config.cjs'), - resourcesDir = path.resolve(import.meta.dirname, '..', config.directories?.buildResources || 'build'), + resourcesDir = defaultResourcesDir(config), load = loadBuilderTools, + prepared = process.env.HERMES_PREPARED_PACKAGING, + source = path.resolve(import.meta.dirname, '../../..'), + target = process.env.HERMES_PREPARED_TARGET, } = {}) { + if (prepared) return consumeWindowsBundleTools(prepared, source, signing, target) const builder = await load() const configured = config.toolsets?.winCodeSign const { kit } = await builder.getWindowsKitsBundle({ winCodeSign: configured, resourcesDir }) + /** @type {WindowsBundleTools} */ const result = { makeappx: path.join(kit, 'makeappx.exe'), signtool: path.join(kit, 'signtool.exe'), @@ -42,7 +72,7 @@ export async function ensureWindowsBundleTools({ } } const files = [result.makeappx, result.signtool] - if (signing) files.push(result.dlib) + if (result.dlib) files.push(result.dlib) if (result.dotnetRoot) files.push(path.join(result.dotnetRoot, 'dotnet.exe')) for (const file of files) { if (!fs.statSync(file).isFile()) throw new Error(`Windows bundle tool is not a file: ${file}`) diff --git a/docs/shared-bundle-builds.md b/docs/shared-bundle-builds.md index 8b0946390b..5007477c5e 100644 --- a/docs/shared-bundle-builds.md +++ b/docs/shared-bundle-builds.md @@ -46,23 +46,64 @@ failure aborts completion; an existing stale product is not a successful update. There is no updater-specific npm cache, fallback install, extra refresh, or memory-provider reinstall. PM owns the complete Python union. -Build a desktop distribution from a checkout at its release tag. Use its -PM-prepared Python 3.14. The driver delegates Python dependency preparation to PM: +Build a desktop distribution from a clean checkout at its release tag. Start +with a host Python that can run the entrypoint and Git; preparation acquires the +pinned Python, Node and npm through PM. Native compilers/SDKs remain host inputs: ```sh python -m scripts.bundles.desktop --tag=vX.Y.Z ``` -`desktop.py` requires exactly one of `--tag` or `--commit`. Commit builds +Without `--prepared`, `desktop.py` requires exactly one of `--tag` or `--commit`. Commit builds require the full commit SHA and matching checkout HEAD. `--variant` accepts `bundled`, `store`, or `light`, with `bundled` as the default. `--repo` selects -the checkout. Arguments after `--` go to Electron Builder. +the checkout. Arguments after `--` go to the prepared Electron Builder wrapper; +they cannot replace the admitted target, tools, output or packaging configuration. -The driver prepares Node dependencies, builds the selected products, and runs -Electron packaging. The desktop npm build prepares its stamp and native tree -before the shared desktop compiler. Electron-specific dependency compilation, +The driver first prepares the complete dependency set: managed tools, the locked +Node workspace union, icon environment, Electron-native bindings, packaging +utilities, and (except for light) the application and independent PM environments. +Only then does it compile products, assemble the payload, and package Electron. MSIX metadata, signing, notarization, and package formats remain adapter work. +### Split desktop preparation and consumption + +The one-command build and CI use the same preparation operation. To stop before +product compilation and packaging: + +```sh +python scripts/bundles/desktop.py --tag vX.Y.Z --variant bundled --prepare-only \ + --work "$PWD/.build/desktop-job" --cache "$PWD/.cache/desktop-inputs" +python scripts/bundles/desktop.py --prepared "$PWD/.build/desktop-job/prepared.json" +``` + +`--work` and `--cache` are separate, build-owned directories. Preparation claims +the work directory itself; do not create it beforehand. It writes `prepared.json` +last, after every provider succeeds. That file contains this job's absolute paths +and source identity, not a portable cache receipt. Reprepare after moving a +checkout, changing source or locks, or losing an input. Repeated preparation may +reuse admitted dependency bytes while rebuilding path-bound environments. + +`--prepared` does not accept new tag/commit/work/cache arguments. It validates +the clean checkout and prepared inputs, then fails on stale or missing inputs +instead of installing them. Bundled and Store builds may consume the same +preparation for a stable tag; light and commit builds cannot switch to Store. +Product builds still run each time. Native staging exposes `prepare_native` and +`finish_native` to this composition; `hermes pm bundle` remains a complete +native staging command, not the desktop preparation interface. + +Windows and macOS release jobs use **restore → prepare → save → build**. The +cache action derives provider paths and transports candidates; it neither +creates the preparation workdir nor declares a cache hit valid. Saves run after +successful preparation, before product compilation or signing. Failed +preparation leaves completed provider data locally but does not save an overall +snapshot. The general `setup-pm` action remains available for other workflows. + +The strict boundary forbids dependency acquisition during consumption, not all +network access: signing, timestamping, notarization and publication retain their +online responsibilities. A network-denied unsigned native build is still needed +to prove the boundary on each release target. + Stage a native agent with both frontend products, without Electron packaging: ```sh @@ -83,6 +124,37 @@ the application environment, and the agent. That PM command does not build frontends. `npm run payload --workspace apps/desktop` uses the stage driver that includes them. Neither command creates an Electron installer. +### Split PM Bundle preparation and assembly + +PM Bundle uses the same isolated tool bootstrap and native preparation slice, +without installing desktop workspaces, icons, Electron bindings or packagers: + +```sh +python -S -B scripts/bundles/native_build.py --source "$PWD" \ + --work "$PWD/.build/payload-job" --cache "$PWD/.cache/payload-inputs" \ + --out "$PWD/build/agent-payload" --ref HEAD --prepare-only +python -S -B scripts/bundles/native_build.py \ + --prepared "$PWD/build/agent-payload.prepared.json" +``` + +Use a clean checkout at the selected revision. `--ref` defaults to `HEAD`; +`--commit` accepts an exact full SHA instead. Work, cache and output must be +separate; preparation claims a previously absent work directory. Without +`--prepare-only`, the driver also assembles. `--prepared` takes no new selection +or path arguments and validates the native receipt before assembly; it never +bootstraps or repairs dependencies. The receipt and lock are output siblings +(`agent-payload.prepared.json`, `agent-payload.prepare.lock`), not shipped files. +Assembly deliberately supplies no frontend products, as with `hermes pm bundle`. + +The `payload-test` producer on the existing cache action selects only `tools`, +`python/runtime` and `native`. It excludes source `node_modules`, npm caches, +icon environments and packagers. PM's managed tool bootstrap still includes Node +and npm, but does not run desktop `npm ci`. Native compiler identity and Windows +ARM64 prerequisites use the shared owner once. Cache hits always undergo provider +admission; native SDK/compiler prerequisites still belong to the host. PR jobs +only restore; saves require successful preparation on the default-branch trusted +lane with no alternate source ref. No signing or publication is added. + For Termux, tools and the wheelhouse are prerequisites: ```sh @@ -94,9 +166,8 @@ The Termux driver accepts exactly one of `--tag` or `--commit`. It prepares the TUI workspace, calls the shared TUI builder, and passes that product to `build_deb.sh`. It does not add the dashboard or replace bionic preparation. -The CLI contracts are in `scripts/bundles/desktop.py:129–143`, -`scripts/bundles/stage.py:17–46`, `pm/cli.py:442–444,488–491`, and -`scripts/termux/build.py:21–62`. +The CLI contracts are in `scripts/bundles/desktop.py`, +`scripts/bundles/stage.py`, `pm/cli.py`, and `scripts/termux/build.py`. ## Shared agent and launcher contract @@ -220,17 +291,43 @@ consumers. They are not interchangeable cleanup targets. extracted binaries without scanning build-only artifacts. - PM-runtime and application builds share the provider's persistent uv cache. CI restores/saves that cache, not the temporary build HOME. Failed builds - retain completed wheels. Only v2 keys are restored; there is no legacy fallback. + retain completed wheels. General PM staging uses its v2 cache namespace; + desktop preparation uses its own input snapshot namespace. Plain `uv cache prune` removes dangling entries without discarding offline wheel inputs. It does not remove all historical versions or enforce a size cap. -- Icon preparation uses its own `SOURCE/.cache/icon-build`. Its build-only - dependencies do not belong in the native application cache. +- Standalone icon preparation uses `SOURCE/.cache/icon-build`. Desktop + preparation places its icon environment under the job workdir and its wheel + cache under `CACHE/python/build`. Neither enters the shipped runtime. - Frontend `node_modules` is a provider input, not a frontend product. Docker's runtime TypeScript and Photon selections are separate exceptions. -The native cache behavior is in `scripts/bundles/native.py:55–65,202–216`. +The native cache behavior is in `scripts/bundles/native.py`. Removing all caches from a payload can break offline environment reconstruction. +Desktop transport selects PM tool entries, Python download/wheel caches, npm's +content-addressed downloads, prepared workspace dependencies, and native/package +tool inputs through `scripts/ci/desktop_build_cache.py`. It does not select the +workdir, virtual environments, live user home, signing tokens, or products. +The signature-result cache is separate. Native wheel partitions depend on +observed compiler/SDK/OpenSSL identity; incomplete identity deliberately forgoes +warm reuse. This is not a fully pinned host SDK. + +Cache keys are lookup hints, not authorization. Native builders have separate +release and commit execution jobs with literal `cache-mode: write` and +`cache-mode: read`, respectively. GitHub enforces these permissions on scoped +cache tokens, so commit builds cannot save caches even from their own scripts. +Both branches share their matrix, environment, and steps through YAML anchors. +The original `build-win32` and `build-darwin` IDs aggregate the branches: after +successful validation and input archiving, exactly the selected branch must +succeed and the other must be skipped. A failed, cancelled, or unexpectedly +skipped selected build fails the aggregate; it cannot permit publication. +Existing publication dependencies keep using those IDs. A skipped save or a +different key prefix alone cannot prevent a script from poisoning a writer +namespace. +Default-branch canary dispatch retains default-branch cache scope; changing a +checkout SHA does not change the workflow ref's cache scope. See GitHub's +[cache access contract](https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#controlling-cache-access-with-cache-mode). + ## Pinned binary inputs `python -m scripts.ci.archive_inputs` preserves every HTTPS artifact in @@ -246,11 +343,13 @@ cannot replace an existing object. The all-target workflow runs on pin changes on main, manual dispatch, and as an admitted release prerequisite. It uses runner Python before the pinned -toolchain is available. Protected build jobs opt into the same R2-first step -through `setup-pm`'s `archive-inputs` input. Target payload steps seed the -actual PM store's disposable fetch entries with `--target` and `--store`; -Termux also passes `--payload` for runtime libraries. Cache hits do not skip -preservation. Untrusted PR jobs receive no publication credentials. +toolchain is available. Desktop jobs depend on that prerequisite and then let PM +fetch verified inputs without archival write credentials; they do not need a +second cache-seeding lane. Other protected workflows can opt into archival +through `setup-pm`'s `archive-inputs` input. Targeted archival can seed disposable +fetch entries with `--target` and `--store`; Termux also passes `--payload` for +runtime libraries. Cache hits do not skip preservation. Desktop R2 credentials +are scoped to upload steps. Untrusted PR jobs receive no publication credentials. Installed PM clients, bootstrap installers, and Nix pin consumers use the primary URL followed by the public mirror if the download is unavailable. @@ -270,6 +369,12 @@ Electron/SDK archives independently downloaded by their build tools. The checks below distinguish product tests from distribution acceptance. Signed installers and Android device execution require their native runners. +The macOS dmgbuild acquisition owner is part of packaging preparation, not the +strict build. Native DMG creation, detach diagnostics, signing and notarization +must be exercised after tool-provider changes. Windows x64/ARM64 and +macOS x64/ARM64 cold/warm and signed-artifact acceptance remain separate from +Linux helper tests. Linux desktop release lanes remain disabled. + Focused helper tests cannot establish all of these requirements: - Offline frontend compilation from prepared immutable inputs. diff --git a/hermes_cli/runtime_paths.py b/hermes_cli/runtime_paths.py index 15089574ba..5ef76e7f26 100644 --- a/hermes_cli/runtime_paths.py +++ b/hermes_cli/runtime_paths.py @@ -52,11 +52,19 @@ def base_venv(project_root: Path) -> Path: def store_root(project_root: Path) -> Path: - """Read the stamped byte store before PM dependencies are available.""" + """Resolve a payload-relative or stamped store before PM imports.""" override = os.environ.get("HERMES_RUNTIME_DIR") if override: return Path(override).resolve() root = Path(project_root).resolve() + manifest_path = root.parent / "manifest.json" + if manifest_path.is_file(): + manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig")) + if (root.parent / manifest.get("repo", "")).resolve() == root: + store = (root.parent / manifest["store"]).resolve() + if not store.is_relative_to(root.parent): + raise RuntimeError("payload store escapes its root") + return store for directory in (root, *root.parents): stamp = directory / "install-stamp.json" if stamp.is_file(): diff --git a/pm/__init__.py b/pm/__init__.py index 02de3612dd..05b9eb09c0 100644 --- a/pm/__init__.py +++ b/pm/__init__.py @@ -25,7 +25,7 @@ from pm.ensure import ( from pm.client import ( ensure, sync_venv, build_environment, lock_project, stage_manager_runtime, ensure_environment, ensure_python_tool, venv_is_current, - check_project_lock, export_requirements, build_requirements_environment, prune_cache, + check_project_lock, export_requirements, build_requirements_environment, prune_cache, stage_tools, prepare_tools, ) from pm.operations import environment_python, python_tool from pm.extras import available, ensure_import @@ -51,6 +51,8 @@ __all__ = [ "build_environment", "lock_project", "stage_manager_runtime", + "stage_tools", + "prepare_tools", "ensure_environment", "environment_python", "ensure_python_tool", diff --git a/pm/build_operations.py b/pm/build_operations.py index 57a39ce114..32faf55514 100644 --- a/pm/build_operations.py +++ b/pm/build_operations.py @@ -2,9 +2,13 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from dataclasses import dataclass from pathlib import Path import sys +from types import MappingProxyType +from pm.ensure import InstalledPackage +from pm.lock import Lockfile from pm.package import InstallError @@ -84,6 +88,148 @@ def build_requirements_environment( return environment.executable +def prepare_tools(names: Sequence[str], *, out: Path, target: str, + cache: Path | None = None) -> Path: + """Realize an explicit tool closure in a build-owned store, not live state.""" + from pm.ensure import _install, _lockfile + from pm.lock import Facts + from pm.package import StatePackage + from pm.registry import walk + from pm.store import Store + + if isinstance(names, str): + raise TypeError("names must be a sequence, not a string") + packages = walk(list(names)) + if any(isinstance(package, StatePackage) for package in packages): + raise ValueError("prepare_tools accepts tools, not application state") + out = Path(out).resolve() + store, lock = Store(out), _lockfile() + with store.install_lock(): + facts = Facts(out / "facts.json", strict=True) + for package in packages: + _install(package, lock, facts, store, target, _lock_held=True) + return out + + +def _copy_links(package, entry: Path) -> None: + import os + from pm.filesystem import is_junction + + for directory, dirs, files in os.walk(entry): + for name in dirs + files: + path = Path(directory) / name + if path.is_symlink() or is_junction(path): + if (Path(os.readlink(path)).is_absolute() or not path.exists() + or not path.resolve().is_relative_to(entry)): + raise InstallError(package.name, f"tool copy link escapes entry: {path}") + + +@dataclass(frozen=True) +class VerifiedTools: + entries: Mapping[str, InstalledPackage] + _envs: tuple[dict, ...] + + def environment(self, base: Mapping[str, str]) -> dict[str, str]: + from pm.package import compose_env + + return compose_env(list(self._envs), base=dict(base)) + + +def verified_tools(names: Sequence[str], *, source_store: Path, target: str, + lock: Lockfile | None = None) -> VerifiedTools: + """Read a pinned tool closure without writes, acquisition or binary execution. + + Publication already ran package verification. Admission binds its recorded + digest to the canonical entry, lock and declared environment; it does not + rerun arbitrary version probes just to read a previously published tool. + """ + from pm.ensure import _identity, _lockfile + from pm.lock import Facts + from pm.package import StatePackage + from pm.registry import walk + from pm.store import tree_digest + + if isinstance(names, str): + raise TypeError("names must be a sequence, not a string") + packages = walk(list(names)) + if any(isinstance(package, StatePackage) for package in packages): + raise ValueError("verified_tools accepts tool packages, not application state") + source_store = Path(source_store).resolve() + if not (source_store / "facts.json").is_file(): + raise InstallError("tools", f"source facts missing: {source_store}") + facts = Facts(source_store / "facts.json", strict=True) + lock = lock if lock is not None else _lockfile() + entries, envs = {}, [] + for package in packages: + if getattr(package, "pin_only", False): + continue + version = lock.version(package.name) + identity = _identity(lock, package.name, target) + if not version or identity is None: + raise InstallError(package.name, "tool source failed verification: missing pin") + fact = facts.get(package.name) + expected = package.store_entry(version, target) + entry = source_store / expected + if (not fact or fact.get("entry") != expected + or not entry.is_dir() or not entry.resolve().is_relative_to(source_store) + or not facts.installed(package.name, version, source_store, identity) + or tree_digest(entry) != fact.get("digest")): + raise InstallError(package.name, "tool source failed verification", "run preparation again") + _copy_links(package, entry) + binary = package.binary(entry, target) + env = package.env(entry, target) + if ((binary is not None and (not binary.is_file() or not binary.resolve().is_relative_to(entry))) + or facts.env_for(package.name, source_store) != env): + raise InstallError(package.name, "tool source failed verification: binary or environment") + entries[package.name] = InstalledPackage(entry, version, binary) + envs.append(env) + return VerifiedTools(MappingProxyType(entries), tuple(envs)) + + +def stage_tools(names: Sequence[str], *, source_store: Path, out: Path, target: str) -> Path: + """Copy a current, verified tool closure without acquisition or live selection. + + Stores must be disjoint. Both publication locks cover admission and copying; + the ordinary installer owns verification, independent copies and fresh facts. + """ + from contextlib import ExitStack + from pm.ensure import _entry_verified, _install, _lockfile + from pm.lock import Facts + from pm.package import StatePackage + from pm.registry import walk + from pm.store import Store + + if isinstance(names, str): + raise TypeError("names must be a sequence, not a string") + source_store, out = Path(source_store).resolve(), Path(out).resolve() + if source_store.is_relative_to(out) or out.is_relative_to(source_store): + raise ValueError("tool stores must be disjoint") + if not (source_store / "facts.json").is_file(): + raise InstallError("tools", f"source facts missing: {source_store}") + packages = walk(list(names)) + if any(isinstance(package, StatePackage) for package in packages): + raise ValueError("stage_tools accepts tool packages, not application state") + source, destination = Store(source_store), Store(out) + lock = _lockfile() + with ExitStack() as stack: + for store in sorted((source, destination), key=lambda store: str(store.root)): + stack.enter_context(store.install_lock()) + selection = verified_tools(names, source_store=source_store, target=target, lock=lock) + facts = Facts(source_store / "facts.json", strict=True) + for package in packages: + if package.name not in selection.entries: + continue + fact = facts.get(package.name) + assert fact is not None # Admission requires the selected fact under the same store lock. + if not _entry_verified(package, fact, source, target): + raise InstallError(package.name, "tool copy source failed verification", "run preparation again") + staged = Facts(out / "facts.json", strict=True) + for package in packages: + _install(package, lock, staged, destination, target, + copy_from=(facts, source), _lock_held=True, _fresh_copy=True) + return out + + def prune_cache(cache: Path, *, ci: bool = False) -> None: """Prune unused cache entries; CI mode also discards downloaded wheels.""" from pm.environment import managed_environment diff --git a/pm/client.py b/pm/client.py index 976f7269c9..a1765414df 100644 --- a/pm/client.py +++ b/pm/client.py @@ -42,7 +42,8 @@ def _request(operation, arguments, *, callbacks=None, pause_event=None, project_ update_id = receipt._ambient_update_id() callbacks = callbacks or {} spec = OPERATIONS[operation] - names = list(spec.packages) if spec.packages is not None else [arguments["name"]] + names = list(spec.packages) if spec.packages is not None else ( + arguments["names"] if "names" in arguments else [arguments["name"]]) message = { "id": request_id, "operation": operation, "arguments": arguments, "update_id": update_id, @@ -329,5 +330,24 @@ def build_requirements_environment(requirements: Sequence[str], *, out: Path, })) +def prepare_tools(names: Sequence[str], *, out: Path, target: str, + cache: Path | None = None) -> Path: + """Realize build tools without selecting application or profile state.""" + if isinstance(names, str): + raise TypeError("names must be a sequence, not a string") + return Path(_python_operation("prepare_tools", { + "names": list(names), "out": Path(out), "target": target, "cache": cache, + })) + + +def stage_tools(names: Sequence[str], *, source_store: Path, out: Path, target: str) -> Path: + """Independently copy verified pins through a ready PM; never bootstrap.""" + if isinstance(names, str): + raise TypeError("names must be a sequence, not a string") + return Path(_python_operation("stage_tools", { + "names": list(names), "source_store": Path(source_store), "out": Path(out), "target": target, + })) + + def prune_cache(cache: Path, *, ci: bool = False) -> None: _python_operation("prune_cache", {"cache": Path(cache), "ci": ci}) diff --git a/pm/ensure.py b/pm/ensure.py index 59441a1370..4a9f159fa1 100644 --- a/pm/ensure.py +++ b/pm/ensure.py @@ -272,6 +272,7 @@ def _install( *, copy_from: tuple[Facts, Store] | None = None, _lock_held: bool = False, + _fresh_copy: bool = False, ) -> Path: """Realize one pin. Host installs commit facts; cross-target stages carry a marker.""" version = lockfile.version(package.name) @@ -312,11 +313,11 @@ def _install( ) and _entry_verified(package, previous, store, target) else: try: - recorded = (entry / ".pm-stage-pin.json").read_text(encoding="utf-8") + recorded = (entry / ".pm-stage-pin.json").read_text(encoding="utf-8-sig") except OSError: recorded = None current = recorded == pin and not package.verify(entry, target) - if current: + if current and not _fresh_copy: _remove_downloads(store, artifacts) return entry if not artifacts: diff --git a/pm/lock.json b/pm/lock.json index bd69c5924a..f589768d87 100644 --- a/pm/lock.json +++ b/pm/lock.json @@ -96,6 +96,19 @@ }, "version": "0.21.0" }, + "dmgbuild": { + "artifacts": { + "darwin-arm64": { + "sha256": "793404d0c96687e27d5ee40a668d498c92e36a64d6c2906df511031adb33cbeb", + "url": "https://github.com/electron-userland/electron-builder-binaries/releases/download/dmg-builder@1.2.5/dmgbuild-bundle-arm64-75c8a6c.tar.gz" + }, + "darwin-x64": { + "sha256": "1664972f9cc2d6e8fce3b63e42cd30078aff602669c5856939c4519921200433", + "url": "https://github.com/electron-userland/electron-builder-binaries/releases/download/dmg-builder@1.2.5/dmgbuild-bundle-x86_64-75c8a6c.tar.gz" + } + }, + "version": "1.2.5+75c8a6c" + }, "ffmpeg": { "artifacts": { "darwin-arm64": { diff --git a/pm/packages.py b/pm/packages.py index e488d98607..24b36bacf4 100644 --- a/pm/packages.py +++ b/pm/packages.py @@ -117,6 +117,35 @@ class BinaryPackage(Package): return env_for(*self.deps) +@register +class Dmgbuild(BinaryPackage): + """Build-only DMG supplier, independently pinned by PM rather than dmg-builder. + + The lock version is +. Updates are manual: review + the official electron-builder-binaries bundle and re-pin both Darwin targets. + Keep the paired Python tree intact for the launcher and diagnostic hook. + """ + + name = "dmgbuild" + internal = True + on_path = False + flatten = False + probe_version = False + binary_rel = {"posix": "dmgbuild"} + gaps = {target: "DMG creation requires macOS" for target in ALL_TARGETS if not target.startswith("darwin-")} + + def fetch_url(self, version: str, target: str) -> str: + release, _, revision = version.partition("+") + arch = {"darwin-arm64": "arm64", "darwin-x64": "x86_64"}[target] + return ( + "https://github.com/electron-userland/electron-builder-binaries/releases/download/" + f"dmg-builder@{release}/dmgbuild-bundle-{arch}-{revision}.tar.gz" + ) + + def verify(self, entry: Path, target: str) -> str: + return super().verify(entry, target) or self._binary_reason(entry / "python/bin/python3", entry, target) + + class _BionicDebArm: """Shared bionic-arm behavior for the termux tool packages: extract as a .deb on the bionic target (DebPackage's hardened ar+tar), as the binary diff --git a/pm/worker_operations.py b/pm/worker_operations.py index 2804c257a3..1f6bbd19e1 100644 --- a/pm/worker_operations.py +++ b/pm/worker_operations.py @@ -23,6 +23,8 @@ class Operation: OPERATIONS = { "ensure": Operation("pm.ensure", None, "state"), "stage_only": Operation("pm.ensure", None, "always"), + "stage_tools": Operation("pm.build_operations", None, "never"), + "prepare_tools": Operation("pm.build_operations", None, "always"), "sync_venv": Operation("pm.ensure", ("venv",), "policy"), "venv_is_current": Operation("pm.ensure", ("venv",), "never"), "build_environment": Operation("pm.operations", ("uv",), "policy"), diff --git a/scripts/build/README.md b/scripts/build/README.md index 469c0aba4d..726a3a176a 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -21,6 +21,39 @@ build or a target-native runtime. builders, they can access package registries. There is no universal installer, all-products dispatcher, or cross-platform Python environment. +## Complete desktop preparation + +`../bundles/desktop.py` composes these providers for local and CI packaging: + +```sh +python scripts/bundles/desktop.py --tag vX.Y.Z --variant bundled --prepare-only \ + --work "$PWD/.build/desktop-job" --cache "$PWD/.cache/desktop-inputs" +python scripts/bundles/desktop.py --prepared "$PWD/.build/desktop-job/prepared.json" +``` + +Start from a clean checkout at that tag (or use `--commit FULL_SHA`). A host +Python and Git bootstrap preparation; PM selects the pinned tools. Native +compiler/SDK prerequisites are still platform-specific. Omitting `--prepare-only` +runs both phases. Defaults use the same `.build/desktop-job` and +`.cache/desktop-inputs` roots. Do not pre-create the work directory: preparation +claims it and publishes its job-local path selection only after success. + +Preparation covers the exact workspace union, icon Python environment, runtime +and independent PM dependencies, Electron-native bindings, Electron archive and +selected packaging utilities. Light omits the runtime and TUI/web union. +Compilation and packaging consume those inputs, refusing missing/stale inputs +rather than acquiring replacements. The result is tied to this source revision, +target and absolute paths; it must not be restored as an authoritative CI cache. +Stable bundled/Store variants can share it. Product compilation still reruns. + +The desktop cache action derives reusable paths from provider declarations and +saves them after preparation, before compilation/signing. It excludes job-local +environments and products. Native wheel reuse additionally depends on measured +compiler/SDK inputs. Signing-result caches remain separate. Strict dependency +consumption is not offline signing: timestamps, notarization and publication can +still require network access. Native unsigned network-denied packaging and final +signed launch acceptance are distinct verification gates. + ## Prepared JavaScript workspace Run commands from the repository root. Replace the absolute example paths with @@ -247,9 +280,10 @@ cache. `scripts.bundles.stage --cache PATH` (or `hermes pm bundle --cache PATH`) selects the persistent cache explicitly. Direct staging also accepts the provider's `UV_CACHE_DIR`; otherwise it uses the output parent's `.uv-cache`. The PM runtime and application dependency builds receive this same cache. -CI's `setup-pm` restores it and `save-pm-cache` prunes/saves it after the build, -including a failed build. Cache keys use only the v2 namespace, without legacy -fallback. The packaged `uv-cache/` is a copy, not the writable build cache. +General PM staging uses `setup-pm` and `save-pm-cache` to restore and save it +after the build, including failures, under its v2 namespace. Desktop composition +instead prepares the full dependency set before its single dependency snapshot +save. The packaged `uv-cache/` is a copy, not the writable build cache. ### Windows ARM64 build prerequisites @@ -264,7 +298,8 @@ The PowerShell entrypoint accepts `-StateRoot` for persistent build-tool state and `-EnvironmentFile` for its prepared environment. The Python adapter passes that environment only to build children. Rust's original toolchain homes stay explicit, so temporary HOME isolation cannot hide an initialized toolchain. -CI exports the same compiler, SDK, Rust, and OpenSSL environment to later steps. +General CI setup exports that compiler environment to later steps. Desktop +preparation keeps it child-scoped and records native cache identity there. Warm OpenSSL reuse validates both static libraries and its development header. ## Runnable agent assembly diff --git a/scripts/build/icon_environment.py b/scripts/build/icon_environment.py index d7e82af53b..8e0af84c38 100644 --- a/scripts/build/icon_environment.py +++ b/scripts/build/icon_environment.py @@ -14,6 +14,14 @@ sys.path.insert(0, str(ROOT)) import pm +def prepare_icon_environment(source: Path, out: Path, cache: Path, *, explicit: bool = True) -> Path: + """Prepare a fresh locked build-only interpreter without generating assets.""" + return pm.build_environment( + source=source.resolve(), out=out.resolve(), cache=cache.resolve(), + groups=["icon-build"], only_groups=True, explicit=explicit, + ) + + def main(argv: list[str] | None = None) -> int: argv = list(sys.argv[1:] if argv is None else argv) parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) @@ -24,10 +32,9 @@ def main(argv: list[str] | None = None) -> int: argv.remove("--on-demand") source = args.source.resolve() with TemporaryDirectory(prefix="hermes-icon-build-") as temporary: - python = pm.build_environment( - source=source, out=Path(temporary) / "venv", - groups=["icon-build"], only_groups=True, explicit=not args.on_demand, - cache=source / ".cache/icon-build", + python = prepare_icon_environment( + source, Path(temporary) / "venv", source / ".cache/icon-build", + explicit=not args.on_demand, ) return subprocess.run( [str(python), "-I", str(ROOT / "scripts/generate_icons.py"), *argv], diff --git a/scripts/build/node-deps.mjs b/scripts/build/node-deps.mjs index 2829f34020..59b22bf3c4 100644 --- a/scripts/build/node-deps.mjs +++ b/scripts/build/node-deps.mjs @@ -33,8 +33,20 @@ export function npmCommand({ env = process.env } = {}) { return [process.execPath, cli] } +function completedInstallMatches({ source, receipt, hiddenLock, key, nativeKey }) { + if (!existsSync(receipt) || !existsSync(hiddenLock)) return false + const installed = readFileSync(hiddenLock) + const expected = `${key}\n${createHash('sha256').update(installed).digest('hex')}\n` + if (nativeKey !== undefined) { + const nativeReceipt = `${receipt}.native-toolchain` + if (!existsSync(nativeReceipt) || readFileSync(nativeReceipt, 'utf8') !== `${expected}${nativeKey}\n`) return false + } + return readFileSync(receipt, 'utf8') === expected && Object.keys(JSON.parse(installed).packages) + .every(path => existsSync(join(source, path))) +} + /** Install the full requested workspace union in one strict, locked operation. */ -export function prepareNodeDependencies({ source, workspaces, env = process.env, reuse = false, install = true }) { +export function prepareNodeDependencies({ source, workspaces, env = process.env, reuse = false, install = true, nativeToolchain }) { source = resolve(source) if (!Array.isArray(workspaces) || workspaces.length === 0) { throw new Error('Select at least one workspace; implicit all-workspace installation is not allowed') @@ -65,6 +77,11 @@ export function prepareNodeDependencies({ source, workspaces, env = process.env, // This receipt certifies dependency preparation, never compiled product freshness. // Keep it inside the cached tree so a clean npm ci also removes the receipt. const receipt = join(source, 'node_modules/.hermes-node-deps') + // Ordinary product builders consume the baseline receipt; preparation also + // binds lifecycle outputs to its compiler/SDK identity. On a mismatch npm ci + // removes arbitrary package lifecycle outputs, not just known node-pty paths. + const nativeReceipt = `${receipt}.native-toolchain` + const nativeKey = nativeToolchain === undefined ? undefined : JSON.stringify(nativeToolchain) const hiddenLock = join(source, 'node_modules/.package-lock.json') const inputs = createHash('sha256').update(JSON.stringify({ node: process.versions.node, npm: npmVersion, platform: process.platform, arch: process.arch, args, @@ -79,21 +96,19 @@ export function prepareNodeDependencies({ source, workspaces, env = process.env, inputs.update(file).update('\0').update(existsSync(join(source, file)) ? readFileSync(join(source, file)) : '').update('\0') } const key = inputs.digest('hex') - if (reuse && existsSync(receipt) && existsSync(hiddenLock)) { - const installed = readFileSync(hiddenLock) - const expected = `${key}\n${createHash('sha256').update(installed).digest('hex')}\n` - if (readFileSync(receipt, 'utf8') === expected && Object.keys(JSON.parse(installed).packages) - .every(path => existsSync(join(source, path)))) { - console.log(`node-deps: reusing completed install (${selected.join(', ')})`) - return { source, workspaces: selected } - } + if (reuse && completedInstallMatches({ source, receipt, hiddenLock, key, nativeKey })) { + console.log(`node-deps: reusing completed install (${selected.join(', ')})`) + return { source, workspaces: selected } } if (!install) throw new Error('Workspace dependencies are stale or missing and lazy installs are disabled; run an explicit build/update') // npm can fail during validation before deleting node_modules. Invalidate first. rmSync(receipt, { force: true }) + rmSync(nativeReceipt, { force: true }) execFileSync(node, [npm, ...args], { cwd: source, env, stdio: 'inherit' }) if (reuse) { - writeFileSync(receipt, `${key}\n${createHash('sha256').update(readFileSync(hiddenLock)).digest('hex')}\n`) + const completed = `${key}\n${createHash('sha256').update(readFileSync(hiddenLock)).digest('hex')}\n` + writeFileSync(receipt, completed) + if (nativeKey !== undefined) writeFileSync(nativeReceipt, `${completed}${nativeKey}\n`) } return { source, workspaces: selected } } @@ -110,7 +125,8 @@ if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1] source: { type: 'string' }, workspace: { type: 'string', multiple: true }, reuse: { type: 'boolean', default: false }, 'no-install': { type: 'boolean', default: false }, + 'native-toolchain': { type: 'string' }, } }) if (!values.source) throw new Error('--source is required') - prepareNodeDependencies({ source: values.source, workspaces: values.workspace, reuse: values.reuse, install: !values['no-install'] }) + prepareNodeDependencies({ source: values.source, workspaces: values.workspace, reuse: values.reuse, install: !values['no-install'], nativeToolchain: values['native-toolchain'] }) } diff --git a/scripts/bundles/desktop.py b/scripts/bundles/desktop.py index 4434e9a067..fde83d94c9 100644 --- a/scripts/bundles/desktop.py +++ b/scripts/bundles/desktop.py @@ -41,89 +41,75 @@ def npm_command(node: str) -> list[str]: def build(repo: Path, tag: str | None, variant: str, builder_args: list[str], commit_build: str | None = None) -> None: - from pm.store import current_target - from scripts.releases.commit_build import require_commit, version_at + from scripts.bundles.desktop_prepare import BuildRequest, prepare from scripts.releases.bundle_env import decode - from scripts.termux.deb_version import channel_for_tag + request = BuildRequest.create(repo, tag=tag, commit=commit_build, variant=variant, + work=repo / ".build/desktop-job", cache=repo / ".cache/desktop-inputs", + bundle_env=decode(os.environ.get("HERMES_BUNDLE_ENV_JSON", ""))) + build_prepared(prepare(request), builder_args) - # Reject before preparing a payload that cannot use the Store identity. - if variant == "store" and (commit_build or not tag or channel_for_tag(tag) != "stable"): - raise ValueError("Store packaging requires a stable release tag") - repo = repo.resolve() - bundle_env = decode(os.environ.get("HERMES_BUNDLE_ENV_JSON", "")) - if bundle_env and not commit_build: - raise ValueError("Bundle environment defaults require a commit build") - if commit_build: - commit = require_commit(commit_build) - if tag: - raise ValueError("Commit builds cannot also select a tag") - if capture(["git", "rev-parse", "HEAD"], repo) != commit: - raise ValueError("the build checkout must be at the commit being built") - version = version_at(repo, commit) - else: - version = release_version(repo, tag) - commit = capture(["git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"], repo) - if capture(["git", "rev-parse", "HEAD"], repo) != commit: - raise ValueError("the build checkout must be at the release tag") - node = shutil.which("node") - if not node: - raise FileNotFoundError("Node is required") - npm = npm_command(node) - env = {**os.environ, "CI": "true", "PYTHONUTF8": "1", "GITHUB_SHA": commit, - "HERMES_DESKTOP_VARIANT": variant, "HERMES_PYTHON": sys.executable} - env["HERMES_BUNDLE_ENV_JSON"] = json.dumps(bundle_env, sort_keys=True) - if commit_build: - env["HERMES_PAYLOAD_VERSION"] = version - env["HERMES_BUILD_COMMIT"] = commit - env.pop("HERMES_PAYLOAD_TAG", None) - env.pop("GITHUB_REF_NAME", None) - env.pop("GITHUB_HEAD_REF", None) - else: - env.pop("HERMES_BUILD_COMMIT", None) - env["HERMES_PAYLOAD_TAG"] = tag - target = current_target() - node_arch = capture([node, "-p", "process.arch"], repo) - if node_arch != target.split("-")[1]: - raise ValueError(f"Node {node_arch} does not match build target {target}") - if target == "win32-arm64": - from scripts.build.windows_deps import prepare_windows_environment +def build_prepared(path: Path, builder_args: list[str], variant: str | None = None) -> None: + from scripts.bundles.desktop_prepare import PreparedDesktop + from scripts.bundles.desktop_inputs import build_lock + prepared = PreparedDesktop.load(path) + with build_lock(prepared.request.source): + _build_prepared(prepared, builder_args, variant) - env = prepare_windows_environment(source=repo, state=repo / "apps/desktop/build/.build-deps", env=env) - workspaces = ["apps/desktop"] + ([] if variant == "light" else ["ui-tui", "web"]) - run([node, "scripts/build/node-deps.mjs", "--source", str(repo), "--reuse", - *[arg for workspace in workspaces for arg in ("--workspace", workspace)]], cwd=repo, env=env) - payload = repo / "apps/desktop/build/agent-payload" + +def _build_prepared(prepared, builder_args: list[str], variant: str | None) -> None: + prepared.validate() + from scripts.bundles.desktop_inputs import build_environment, select_variant + from scripts.bundles.native import finish_native + + request = prepared.request + variant = select_variant(prepared, variant) + repo, node = request.source, str(prepared.node) + env = build_environment(prepared, variant, os.environ) + desktop = repo / "apps/desktop" + targets = {"win32": ["--win", "msix"], "darwin": ["--mac", "dmg", "zip"], "linux": ["--linux", "AppImage"]}[sys.platform] + package_args = ["--prepared", str(prepared.packager), "--native-deps", str(prepared.native), + *targets, f"-c.extraMetadata.version={request.version}"] + run([node, "scripts/run-electron-builder.mjs", "--validate-only", *package_args, *builder_args], + cwd=desktop, env=env) + run([node, "scripts/build/node-deps.mjs", "--source", str(repo), "--reuse", "--no-install", + "--native-toolchain", prepared.native_toolchain, + *[arg for name in request.workspaces() for arg in ("--workspace", name)]], cwd=repo, env=env) products = repo / "apps/desktop/build/products" icons = products / "icons" - run([node, "scripts/generate-icons.mjs", "--source", str(repo), "--out", str(icons)], cwd=repo, env=env) - if variant == "light": - shutil.rmtree(payload, ignore_errors=True) - else: - + run([str(prepared.icon_python), "-I", str(repo / "scripts/generate_icons.py"), + "--source", str(repo), "--out", str(icons)], cwd=repo, env=env) + if variant != "light": run([node, "scripts/build/tui.mjs", "--source", str(repo), "--out", str(products / "tui")], cwd=repo, env=env) run([node, "scripts/build/web.mjs", "--source", str(repo), "--icons", str(products / "icons"), "--out", str(products / "web")], cwd=repo, env=env) - run([sys.executable, "-m", "scripts.bundles.stage", "--out", str(payload), "--ref", commit, - "--tui", str(products / "tui"), "--web", str(products / "web")], cwd=repo, env=env) - desktop = repo / "apps/desktop" + if prepared.payload is None: + raise ValueError("payload dependencies were not prepared") + if finish_native(prepared.payload, {"tui": products / "tui", "web": products / "web"}): + raise RuntimeError("prepared payload assembly failed") + from scripts.bundles.desktop_prepare import require_source + require_source(repo, request.commit) + shutil.copytree(icons / "apps/desktop/assets", desktop / "assets", dirs_exist_ok=True) + run([node, "scripts/write-build-stamp.mjs"], cwd=desktop, env=env) + run([node, "scripts/build/desktop.mjs", "--source", str(repo), "--icons", str(icons), + "--stamp", str(desktop / "build/install-stamp.json"), "--native-deps", str(prepared.native), + "--out", str(desktop / "dist")], cwd=repo, env=env) # Windows file-version and MSIX build-number policy remains with its packager. version_args = [] if sys.platform == "win32": - if commit_build: + if request.tag is None: # The plain version needs no canary build-number override. metadata = {"file": None, "build": None} else: script = "const w=require('./apps/desktop/scripts/windows-file-version.mjs');const m=require('./scripts/msix-shared.mjs');console.log(JSON.stringify({file:w.windowsFileVersion(process.argv[1]),build:process.argv[2]!=='store'&&process.argv[1].includes('-canary.')?m.canaryBuildMinutes(process.argv[1],process.cwd()):null}))" - metadata = json.loads(capture([node, "-e", script, tag, variant], repo)) + metadata = json.loads(capture([node, "-e", script, request.tag, variant], repo)) env.pop("BUILD_NUMBER", None) if metadata["build"] is not None and variant != "store": env["BUILD_NUMBER"] = str(metadata["build"]) if metadata["file"]: version_args = [f'-c.extraMetadata.shortVersion={metadata["file"]}', f'-c.extraMetadata.shortVersionWindows={metadata["file"]}'] - targets = {"win32": ["--win", "msix"], "darwin": ["--mac", "dmg", "zip"], "linux": ["--linux", "AppImage"]}[sys.platform] - run([*npm, "run", "build", "--", "--icons", str(icons)], cwd=desktop, env=env) - run([*npm, "run", "builder", "--", *targets, f"-c.extraMetadata.version={version}", *version_args, *builder_args], cwd=desktop, env=env) + require_source(repo, request.commit) + run([node, "scripts/run-electron-builder.mjs", *package_args, *version_args, *builder_args], cwd=desktop, env=env) def main() -> None: @@ -133,14 +119,37 @@ def main() -> None: parser.add_argument("--commit", dest="commit_build", default=None, help="Commit-only build: exact full 40-char SHA the checkout is at; " "version comes from the target pyproject, no tag is referenced") - parser.add_argument("--variant", choices=["bundled", "store", "light"], default="bundled") + parser.add_argument("--variant", choices=["bundled", "store", "light"]) parser.add_argument("--repo", type=Path, default=ROOT) + parser.add_argument("--work", type=Path) + parser.add_argument("--cache", type=Path) + parser.add_argument("--prepare-only", action="store_true") + parser.add_argument("--prepared", type=Path) parser.add_argument("builder_args", nargs=argparse.REMAINDER) args = parser.parse_args() - if bool(args.tag) == bool(args.commit_build): - parser.error("exactly one of --tag or --commit is required") - build(args.repo, args.tag, args.variant, [v for v in args.builder_args if v != "--"], - commit_build=args.commit_build) + builder_args = [v for v in args.builder_args if v != "--"] + try: + if args.prepared: + if args.tag or args.commit_build or args.prepare_only or args.work or args.cache: + parser.error("--prepared supplies the complete build request") + build_prepared(args.prepared, builder_args, args.variant) + else: + from scripts.bundles.desktop_prepare import BuildRequest, prepare + from scripts.releases.bundle_env import decode + request = BuildRequest.create(args.repo, tag=args.tag, commit=args.commit_build, + variant=args.variant or "bundled", + work=args.work or args.repo / ".build/desktop-job", + cache=args.cache or args.repo / ".cache/desktop-inputs", + bundle_env=decode(os.environ.get("HERMES_BUNDLE_ENV_JSON", ""))) + if args.prepare_only and builder_args: + parser.error("builder arguments belong to the build phase") + result = prepare(request) + if args.prepare_only: + print(result) + else: + build_prepared(result, builder_args) + except (ValueError, OSError, RuntimeError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"desktop build: {exc}\n") if __name__ == "__main__": diff --git a/scripts/bundles/desktop_inputs.py b/scripts/bundles/desktop_inputs.py new file mode 100644 index 0000000000..2e6a649213 --- /dev/null +++ b/scripts/bundles/desktop_inputs.py @@ -0,0 +1,67 @@ +"""Build identity and process environment from an admitted preparation.""" +from __future__ import annotations + +from collections.abc import Mapping +from contextlib import contextmanager +import json +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scripts.bundles.desktop_prepare import PreparedDesktop + + +@contextmanager +def build_lock(source: Path): + from hermes_cli.runtime_state import _lock + + directory = source / ".build" + if directory.is_symlink(): + raise ValueError("desktop build scratch must not be a symlink") + directory.mkdir(exist_ok=True) + with (directory / ".desktop-build.lock").open("a+b") as lock: + if not _lock(lock.fileno(), wait=False): + raise ValueError("another desktop build or preparation is using this source checkout") + yield + + +def build_environment(prepared: PreparedDesktop, variant: str, inherited: Mapping[str, str]) -> dict[str, str]: + from pm.build_operations import verified_tools + from pm.lock import Lockfile + from scripts.bundles.desktop_toolchain import bootstrap_environment + + request = prepared.request + env = bootstrap_environment(request.source, request.work, request.cache, inherited) + selection = verified_tools(["python", "node", "npm"], source_store=request.cache / "tools", + target=request.target, lock=Lockfile(request.source / "pm/lock.json")) + if prepared.python != selection.entries["python"].binary or prepared.node != selection.entries["node"].binary: + raise ValueError("prepared Python/Node paths do not match selected tools; prepare again") + env = selection.environment(env) + env.update(CI="true", PYTHONUTF8="1", GITHUB_SHA=request.commit, + HERMES_DESKTOP_VARIANT=variant, HERMES_PYTHON=str(prepared.python), + HERMES_PAYLOAD_VERSION=request.version, + HERMES_BUNDLE_ENV_JSON=json.dumps(request.bundle_env, sort_keys=True)) + env.pop("BUILD_NUMBER", None) + env.pop("GITHUB_HEAD_REF", None) + if request.tag is None: + env["HERMES_BUILD_COMMIT"] = request.commit + env.pop("HERMES_PAYLOAD_TAG", None) + env.pop("GITHUB_REF_NAME", None) + else: + env.pop("HERMES_BUILD_COMMIT", None) + env["HERMES_PAYLOAD_TAG"] = request.tag + env["GITHUB_REF_NAME"] = request.tag + return env + + +def select_variant(prepared: PreparedDesktop, variant: str | None) -> str: + from scripts.termux.deb_version import channel_for_tag + + request = prepared.request + variant = variant or request.variant + if variant != request.variant and not ( + {variant, request.variant} == {"bundled", "store"} and request.tag + and channel_for_tag(request.tag) == "stable" + ): + raise ValueError("requested variant was not prepared") + return variant diff --git a/scripts/bundles/desktop_prepare.py b/scripts/bundles/desktop_prepare.py new file mode 100644 index 0000000000..159fb547dc --- /dev/null +++ b/scripts/bundles/desktop_prepare.py @@ -0,0 +1,280 @@ +"""Complete desktop dependency preparation and its job-local consume contract.""" +from __future__ import annotations + +import argparse +from dataclasses import asdict, dataclass +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def git(source: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=source, text=True, encoding="utf-8").strip() + + +def require_source(source: Path, commit: str) -> None: + if git(source, "rev-parse", "HEAD") != commit: + raise ValueError("preparation source checkout changed revision; prepare again") + if git(source, "status", "--porcelain", "--untracked-files=normal"): + raise ValueError("desktop builds require a clean source checkout at the admitted revision") + + +def fingerprint(path: Path) -> str: + if path.is_dir(): + from pm.store import tree_digest + return tree_digest(path) + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def require_owned(request: BuildRequest, paths: list[Path]) -> None: + require_output_roots(request.source) + roots = (request.work.resolve(), request.cache.resolve(), + (request.source / "apps/desktop/build").resolve()) + for path in paths: + if not path.is_absolute() or not any(path.resolve().is_relative_to(root) for root in roots): + raise ValueError(f"preparation input is outside owned roots: {path}") + + +def require_output_roots(source: Path) -> None: + from pm.filesystem import is_junction + + for relative in (".build", "apps", "apps/desktop", "apps/desktop/build", "apps/desktop/dist", "apps/desktop/release"): + path = source / relative + if path.is_symlink() or (path.exists() and is_junction(path)): + raise ValueError(f"desktop output root must not be a symlink or junction: {path}") + + +def selected_tool_inputs(request: BuildRequest) -> list[Path]: + from pm.build_operations import verified_tools + from pm.lock import Lockfile + + selection = verified_tools(["python", "node", "npm"], source_store=request.cache / "tools", + target=request.target, lock=Lockfile(request.source / "pm/lock.json")) + return [entry.path for entry in selection.entries.values()] + + +@dataclass(frozen=True) +class BuildRequest: + source: Path + work: Path + cache: Path + commit: str + tag: str | None + version: str + variant: str + target: str + bundle_env: dict[str, str | None] + + @classmethod + def create(cls, source: Path, *, tag: str | None, commit: str | None, variant: str, + work: Path, cache: Path, bundle_env: dict[str, str | None]) -> BuildRequest: + from pm.store import current_target + from scripts.bundles.desktop import release_version + from scripts.releases.bundle_env import validate + from scripts.releases.commit_build import require_commit, version_at + from scripts.termux.deb_version import channel_for_tag + + if variant == "store" and (commit or not tag or channel_for_tag(tag) != "stable"): + raise ValueError("Store packaging requires a stable release tag") + if variant not in {"bundled", "store", "light"}: + raise ValueError("invalid desktop variant") + if bool(tag) == bool(commit): + raise ValueError("exactly one of --tag or --commit is required") + bundle_env = validate(bundle_env) + if bundle_env and tag: + raise ValueError("Bundle environment defaults require a commit build") + source, work, cache = source.resolve(), work.resolve(), cache.resolve() + require_output_roots(source) + if work == cache or work.is_relative_to(cache) or cache.is_relative_to(work): + raise ValueError("preparation work and cache must be separate directories") + for destination in (work, cache): + if source == destination or source.is_relative_to(destination): + raise ValueError("preparation output must not contain the source checkout") + if commit: + commit = require_commit(commit) + require_source(source, commit) + version = version_at(source, commit) + else: + assert tag is not None # The exclusive selection was checked above. + version = release_version(source, tag) + commit = git(source, "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}") + require_source(source, commit) + return cls(source, work, cache, commit, tag, version, variant, current_target(), bundle_env) + + def data(self) -> dict: + return {**asdict(self), "source": str(self.source), "work": str(self.work), "cache": str(self.cache)} + + @classmethod + def from_data(cls, data: dict) -> BuildRequest: + return cls(**{**data, **{name: Path(data[name]) for name in ("source", "work", "cache")}}) + + def workspaces(self) -> list[str]: + return ["apps/desktop"] + ([] if self.variant == "light" else ["ui-tui", "web"]) + + +@dataclass(frozen=True) +class PreparedDesktop: + request: BuildRequest + python: Path + node: Path + icon_python: Path + native: Path + packager: Path + payload: Path | None + digests: dict[str, str] + native_toolchain: str + + @classmethod + def record(cls, request: BuildRequest, *, python: Path, node: Path, icon_python: Path, + native: Path, packager: Path, payload: Path | None, native_toolchain: str) -> PreparedDesktop: + files = [python, node, icon_python.parent.parent, native, packager] + if payload is not None: + files.append(payload) + require_owned(request, files) + files.extend(selected_tool_inputs(request)) + return cls(request, python, node, icon_python, native, packager, payload, + {str(path): fingerprint(path) for path in files}, native_toolchain) + + def write(self, path: Path) -> None: + from pm.lock import _write + _write(path, {"schema": 1, "request": self.request.data(), + **{name: str(value) if value is not None else None for name, value in ( + ("python", self.python), ("node", self.node), ("icon_python", self.icon_python), + ("native", self.native), ("packager", self.packager), ("payload", self.payload))}, + "digests": self.digests, "native_toolchain": self.native_toolchain}) + + @classmethod + def load(cls, path: Path) -> PreparedDesktop: + try: + data = json.loads(path.read_text(encoding="utf-8-sig")) + if data.pop("schema") != 1: + raise ValueError("unsupported preparation schema") + request = BuildRequest.from_data(data.pop("request")) + for name in ("python", "node", "icon_python", "native", "packager", "payload"): + if data[name] is not None: + data[name] = Path(data[name]) + return cls(request=request, **data) + except (OSError, ValueError, KeyError, TypeError) as exc: + raise ValueError(f"desktop preparation is missing or invalid: {path}; run preparation again") from exc + + def validate(self) -> None: + from pm.store import current_target + require_source(self.request.source, self.request.commit) + if self.request.target != current_target(): + raise ValueError("preparation target differs from this host") + paths = [self.python, self.node, self.icon_python.parent.parent, self.native, self.packager] + paths.extend(selected_tool_inputs(self.request)) + if self.payload is not None: + paths.append(self.payload) + require_owned(self.request, paths) + if set(self.digests) != {str(path) for path in paths}: + raise ValueError("preparation input inventory changed") + for path in paths: + if not path.is_absolute() or not path.exists() or fingerprint(path) != self.digests[str(path)]: + raise ValueError(f"preparation input changed or is missing: {path}; prepare again") + + +def prepare(request: BuildRequest) -> Path: + from scripts.bundles.desktop_inputs import build_lock + + require_source(request.source, request.commit) + with build_lock(request.source): + return _prepare(request) + + +def _prepare(request: BuildRequest) -> Path: + from scripts.bundles.desktop_toolchain import run_preparation + from pm.lock import _write + from hermes_cli.runtime_state import _lock + + require_source(request.source, request.commit) + owner = request.work / ".desktop-preparation" + if request.work.exists(): + if not owner.is_file() or owner.read_text(encoding="utf-8-sig") != str(request.source): + raise ValueError(f"preparation work directory is not owned by this checkout: {request.work}") + else: + request.work.mkdir(parents=True) + owner.write_text(str(request.source), encoding="utf-8") + with (request.work / ".lock").open("a+b") as lock: + if not _lock(lock.fileno(), wait=False): + raise ValueError("another desktop preparation is using this work directory") + result = request.work / "prepared.json" + result.unlink(missing_ok=True) + request_file = request.work / "request.json" + _write(request_file, request.data()) + status = run_preparation(request.source, request.work, request.cache, request_file) + if status: + result.unlink(missing_ok=True) + raise RuntimeError(f"desktop dependency preparation failed (exit {status}); see provider output above") + PreparedDesktop.load(result).validate() + return result + + +def prepare_in_worker(request: BuildRequest) -> Path: + from scripts.bundles.desktop import run + from scripts.bundles.desktop_toolchain import prepare_tools + from scripts.build.icon_environment import prepare_icon_environment + from scripts.bundles.native import prepare_native + + require_source(request.source, request.commit) + python, node, env = prepare_tools(request.source, request.work, request.cache, os.environ) + native_toolchain = Path(env["UV_CACHE_DIR"]).name + run([str(node), "scripts/build/node-deps.mjs", "--source", str(request.source), "--reuse", + "--native-toolchain", native_toolchain, + *[arg for name in request.workspaces() for arg in ("--workspace", name)]], cwd=request.source, env=env) + icon_environment = request.work / "icon-environment" + if icon_environment.exists(): + if icon_environment.is_symlink(): + raise ValueError("icon environment must be preparation-owned, not a symlink") + shutil.rmtree(icon_environment) + icon_python = prepare_icon_environment(request.source, icon_environment, request.cache / "python/build") + native = request.source / "apps/desktop/build/native-deps" + run([str(node), "apps/desktop/scripts/stage-native-deps.mjs", "--source", str(request.source), + "--out", str(native), "--native-toolchain", native_toolchain], cwd=request.source, env=env) + packager = request.work / "packager" + dmg_args = [] + if request.target.startswith("darwin-"): + from apps.desktop.scripts.prepare_dmgbuild import prepare_dmgbuild + dmg = prepare_dmgbuild(request.cache / "tools", request.cache / "python/build") + dmg_args = ["--dmgbuild", str(dmg)] + run([str(node), "apps/desktop/scripts/prepare-packaging-tools.mjs", "--source", str(request.source), + "--out", str(packager), "--cache", str(request.cache / "packager"), "--target", request.target, *dmg_args], + cwd=request.source, env=env) + run([str(node), "apps/desktop/scripts/probe-prepared-native.mjs", "--source", str(request.source), + "--native-deps", str(native), "--packaging", str(packager / "prepared.json"), + "--out", str(request.work / "native-probe"), "--native-toolchain", native_toolchain], + cwd=request.source, env=env) + payload = None + if request.variant != "light": + payload = prepare_native(out=request.source / "apps/desktop/build/agent-payload", ref=request.commit, + source=request.source, cache=Path(env["UV_CACHE_DIR"]), + tools=request.cache / "tools", env=env) + require_source(request.source, request.commit) + prepared = PreparedDesktop.record(request, python=python, node=node, icon_python=icon_python, + native=native, packager=packager / "prepared.json", payload=payload, + native_toolchain=native_toolchain) + result = request.work / "prepared.json" + prepared.write(result) + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", required=True, type=Path) + parser.add_argument("--worker", action="store_true", required=True) + args = parser.parse_args() + request = BuildRequest.from_data(json.loads(args.request.read_text(encoding="utf-8-sig"))) + prepare_in_worker(request) + + +if __name__ == "__main__": + main() diff --git a/scripts/bundles/desktop_toolchain.py b/scripts/bundles/desktop_toolchain.py new file mode 100644 index 0000000000..e014df989c --- /dev/null +++ b/scripts/bundles/desktop_toolchain.py @@ -0,0 +1,193 @@ +"""Child-scoped desktop build tools, independent of the invoking Hermes install. + +This module is stdlib-only until preparation runs inside the isolated child. +""" +from __future__ import annotations + +from collections.abc import Mapping +import hashlib +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import uuid + + +_INSTALL_ENV = { + "HERMES_INSTALL_ROOT", "HERMES_PAYLOAD_ROOT", "HERMES_PAYLOAD_TAG", "HERMES_BUILD_COMMIT", + "HERMES_SITE", "HERMES_PYTHON", "HERMES_NODE", "HERMES_PROFILE", "HERMES_REAL_HOME", + "HERMES_DATA_DIR_SUFFIX", "HERMES_BUNDLED_SKILLS", "HERMES_OPTIONAL_SKILLS", + "HERMES_BUNDLED_PLUGINS", "HERMES_BUNDLED_LOCALES", "HERMES_OPTIONAL_MCPS", + "PYTHONPATH", "PYTHONHOME", "PYTHONUSERBASE", "VIRTUAL_ENV", "CONDA_PREFIX", +} + + +def bootstrap_environment(source: Path, work: Path, cache: Path, + env: Mapping[str, str]) -> dict[str, str]: + """Return launch state; never import PM, mutate the process, or write files.""" + source, work, cache = source.resolve(), work.resolve(), cache.resolve() + owned = {"HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "HERMES_HOME", "HERMES_RUNTIME_DIR", + "HERMES_PYTHON_SRC_ROOT", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", + "UV_CACHE_DIR", "NPM_CONFIG_CACHE", "NPM_EXECPATH", "NPM_NODE_EXECPATH", + "NPM_CONFIG_USERCONFIG", "NPM_CONFIG_GLOBALCONFIG", "NPM_CONFIG_PREFIX", + "PYTHONUTF8", "PYTHONDONTWRITEBYTECODE", "CARGO_HOME", "RUSTUP_HOME"} + result = {key: value for key, value in env.items() if key.upper() not in _INSTALL_ENV | owned} + canonical = {key.upper(): value for key, value in env.items()} + # Rustup must still find the runner's installed toolchain after HOME moves. + home_key = "USERPROFILE" if os.name == "nt" else "HOME" + original_home = Path(canonical.get("HERMES_REAL_HOME") or canonical.get(home_key) or Path.home()) + for key, directory in (("CARGO_HOME", ".cargo"), ("RUSTUP_HOME", ".rustup")): + result[key] = canonical.get(key) or str(original_home / directory) + home = work / "home" + result.update({ + "HOME": str(home), "USERPROFILE": str(home), + "LOCALAPPDATA": str(home / "AppData/Local"), + "APPDATA": str(home / "AppData/Roaming"), + "HERMES_HOME": str(work / "hermes-home"), + "HERMES_RUNTIME_DIR": str(cache / "tools"), + "HERMES_PYTHON_SRC_ROOT": str(source), + "XDG_CONFIG_HOME": str(home / "config"), "XDG_CACHE_HOME": str(home / "cache"), + "XDG_DATA_HOME": str(home / "data"), "XDG_STATE_HOME": str(home / "state"), + "UV_CACHE_DIR": str(cache / "python/runtime"), + "npm_config_cache": str(cache / "npm"), + "npm_config_userconfig": os.devnull, + "PYTHONUTF8": "1", "PYTHONDONTWRITEBYTECODE": "1", + }) + return result + + +def run_preparation(source: Path, work: Path, cache: Path, request_file: Path, + *, worker: Path | None = None) -> int: + """Bootstrap PM after process isolation, then run the preparation worker. + + The request contains paths/selection only; credentials stay in the inherited + child environment. Return the worker's exit status without hiding failures. + """ + source, work, cache = source.resolve(), work.resolve(), cache.resolve() + environment = bootstrap_environment(source, work, cache, os.environ) + Path(environment["HOME"]).mkdir(parents=True, exist_ok=True) + launcher = ( + "import subprocess, sys; from pathlib import Path; " + "source, cache, request, entry, worker = map(Path, sys.argv[1:]); " + "sys.path.insert(0, str(source)); from pm.runtime import runtime_command; " + "command = runtime_command(entry, " + "[str(worker), '--request', str(request), '--worker'], " + "cache=cache / 'python/runtime'); " + "sys.exit(subprocess.run(command, stdin=subprocess.DEVNULL).returncode)" + ) + return subprocess.run( + [sys.executable, "-I", "-S", "-B", "-c", launcher, + str(source), str(cache), str(request_file.resolve()), str(Path(__file__).resolve()), + str(worker or source / "scripts/bundles/desktop_prepare.py")], + cwd=source, env=environment, stdin=subprocess.DEVNULL, + ).returncode + + +def prepare_tools(source: Path, work: Path, cache: Path, + env: Mapping[str, str]) -> tuple[Path, Path, dict[str, str]]: + """Run only in the bootstrapped worker; return native Python, Node and env. + + PM owns pin selection and verification. The caller passes the resulting env + to every build child, including its explicit native-wheel UV_CACHE_DIR. + """ + import pm + from pm.paths import store_root + + source, work, cache = source.resolve(), work.resolve(), cache.resolve() + if store_root() != cache / "tools": + raise ValueError("desktop tools require the isolated preparation worker") + prepared = dict(env) + if pm.current_target() == "win32-arm64": + from scripts.build.windows_deps import prepare_windows_environment + + prepared = prepare_windows_environment(source=source, state=cache / "native/prerequisites", env=prepared) + if pm.current_target().startswith("darwin"): + # python-build-standalone's sysconfig names its absent build-host tools. + prepared.setdefault("AR", "/usr/bin/ar") + prepared.setdefault("CC", "clang") + for name in ("uv", "npm"): + pm.ensure(name, explicit=True) + binaries = {} + for name in ("python", "node"): + installed = pm.installed_package(name) + if installed is None or installed.binary is None or not installed.binary.is_file(): + raise FileNotFoundError(f"PM did not prepare a native {name} executable") + binaries[name] = installed.binary + prepared = pm.env_for("python", "npm", base_env=prepared) + prepared.update({"HERMES_PYTHON": str(binaries["python"]), "HERMES_NODE": str(binaries["node"]), + "UV_CACHE_DIR": str(native_cache_path(cache, prepared))}) + return binaries["python"], binaries["node"], prepared + + +def native_cache_path(cache: Path, env: Mapping[str, str]) -> Path: + """Partition built wheels by observed native inputs, not just the OS label. + + Missing tool/SDK identity deliberately gets a fresh partition. No inherited + environment or credential-bearing Cargo configuration is written to cache. + """ + from pm.store import current_target + + target = current_target() + keys = {"CC", "CXX", "AR", "CFLAGS", "CXXFLAGS", "LDFLAGS", "RUSTFLAGS", "RUSTUP_TOOLCHAIN", + "SDKROOT", "MACOSX_DEPLOYMENT_TARGET", "WindowsSDKVersion", "VCToolsVersion", + "WindowsSdkDir", "INCLUDE", "LIB", "LIBPATH", "OPENSSL_DIR", "OPENSSL_LIB_DIR", + "OPENSSL_INCLUDE_DIR", "OPENSSL_STATIC"} + selected = {key: value for key, value in env.items() if key in keys or key.startswith("CC_")} + identity = {"recipe": 1, "target": target, "host": platform.platform(), + "libc": platform.libc_ver(), "inputs": selected, "probes": [], "openssl": {}} + commands = [["rustc", "-vV"]] + if target.startswith("win32"): + commands.append(["cl", "/Bv"]) + complete = bool(env.get("WindowsSDKVersion") and env.get("VCToolsVersion")) + else: + commands.extend([[env.get("CC") or "cc", "--version"], [env.get("CXX") or "c++", "--version"]]) + complete = True + commands.extend([[value, "--version"] for key, value in selected.items() if key.startswith("CC_")]) + if target.startswith("darwin"): + commands.extend([["xcrun", "--show-sdk-path"], ["xcrun", "--show-sdk-build-version"]]) + openssl = env.get("OPENSSL_DIR") + if openssl or env.get("OPENSSL_LIB_DIR") or env.get("OPENSSL_INCLUDE_DIR"): + roots = {"include": Path(env["OPENSSL_INCLUDE_DIR"]) if env.get("OPENSSL_INCLUDE_DIR") + else Path(openssl or "") / "include", + "lib": Path(env["OPENSSL_LIB_DIR"]) if env.get("OPENSSL_LIB_DIR") + else Path(openssl or "") / "lib"} + for kind, root in roots.items(): + files = sorted(path for path in root.rglob("*") if path.is_file()) if root.is_dir() else [] + complete = complete and bool(files) + for path in files: + with path.open("rb") as stream: + identity["openssl"][f"{kind}/{path.relative_to(root).as_posix()}"] = hashlib.file_digest(stream, "sha256").hexdigest() + else: + commands.append(["openssl", "version", "-a"]) + if target == "win32-arm64": + complete = False # The provider must identify the linked static libraries. + for command in commands: + try: + result = subprocess.run(command, env=dict(env), capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=30, stdin=subprocess.DEVNULL) + output = result.stdout + result.stderr + # cl /Bv reports its identity and exits 2 when no source is supplied. + valid = result.returncode in ((0, 2) if command == ["cl", "/Bv"] else (0,)) and bool(output.strip()) + identity["probes"].append([command, output]) + complete = complete and valid + except (OSError, subprocess.TimeoutExpired): + complete = False + # Build-owned locations move across runners; their bytes and relative layout + # identify them. Keep external compiler/SDK paths significant. + encoded = json.dumps(identity, sort_keys=True) + owned = str(cache.resolve()) + encoded = encoded.replace(json.dumps(owned)[1:-1], "") + digest = hashlib.sha256(encoded.encode()).hexdigest() if complete else uuid.uuid4().hex + return cache.resolve() / "python/runtime" / f"native-{target}-{digest}" + + +if __name__ == "__main__": + import runpy + import truststore + + # Like pm/launch.py: initialize platform trust before PM creates HTTPS clients. + truststore.inject_into_ssl() + sys.argv = sys.argv[1:] + runpy.run_path(sys.argv[0], run_name="__main__") diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 4cde9e5e6e..001d1a12ac 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -7,10 +7,10 @@ import subprocess import sys import tempfile import argparse +from dataclasses import asdict from pathlib import Path -from pm.cli import _install_names -from pm.ensure import _store, _facts, _lockfile +from pm.ensure import _lockfile from pm.lock import Facts from pm.registry import get_package, walk from pm.store import current_target @@ -137,28 +137,40 @@ def prune_staged_store(store_dir: Path, names: list[str]) -> None: shutil.rmtree(entry) -def _stage_native(args) -> int: - """Stage a complete payload for THIS machine's target into --out: - repo snapshot + store + facts (via the normal install path, redirected) - + a relocatable venv built on the staged interpreter and synced from - uv.lock. Built natively per (os, arch); there is no cross-target - staging.""" - import os +def prepare_native(*, out: Path, ref: str, source: Path, cache: Path, + tools: Path | None = None, env: dict | None = None) -> Path: + """Prepare final payload dependencies inside the caller's isolated PM process. + The caller supplies its compiler environment; only the full standalone + wrapper provisions machine prerequisites and isolates HOME. + """ + from scripts.bundles.native_prepared import preparation_lock, prepared_path + + out = Path(out).absolute() + if out != out.resolve(): + raise ValueError("symlinked native output") + out.mkdir(parents=True, exist_ok=True) + with preparation_lock(out): + prepared_path(out).unlink(missing_ok=True) + (out / "manifest.json").unlink(missing_ok=True) + return _prepare_native(out=out, ref=ref, source=Path(source).resolve(), + cache=Path(cache).resolve(), tools=tools, env=env) + + +def _prepare_native(*, out: Path, ref: str, source: Path, cache: Path, + tools: Path | None, env: dict | None) -> Path: from pm import paths + from pm.package import InstallError - out = Path(args.out).resolve() store_dir = out / "tools" store_dir.mkdir(parents=True, exist_ok=True) - # A manifest from a previous run would make this payload look sealed - # and refuse its own staging; it is rewritten at the end. - (out / "manifest.json").unlink(missing_ok=True) - repo_dir = out / "hermes-agent" - ref = args.ref or "HEAD" from scripts.bundles.payload import snapshot print(f"staging repo snapshot ({ref})…", flush=True) - snapshot(getattr(args, "source", None) or paths.repo_root(), ref, repo_dir) + revision = subprocess.check_output( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=source, + text=True, encoding="utf-8").strip() + snapshot(source, revision, repo_dir) # PM's provider code reads its adjacent lock. Never combine that tool graph # with a revision selecting different pins. if (repo_dir / "pm/lock.json").read_bytes() != paths.lockfile_path().read_bytes(): @@ -168,32 +180,35 @@ def _stage_native(args) -> int: n for n in _bundle_package_names() if get_package(n).missing_reason(current_target()) is None ] - failed = _install_names(names) + from pm import prepare_tools, stage_tools + + prepare_tools(names, out=Path(tools) if tools is not None else store_dir, + target=current_target(), cache=cache) + if tools is not None: + stage_tools(names, source_store=Path(tools), out=store_dir, target=current_target()) # Prune the staged store BEFORE the venv sync and packaging: drop the # fetch- download-cache archives (needed only at install time — dead # weight in the shipped payload AND in the CI cache that restores this # dir) and any orphaned package versions left over from an older lock # the cache carried in. A lean staged store = a lean CI cache. - if failed: - return 1 + # Only this build's store is ours to prune; machine-wide partials are not. # Cached facts may still name packages removed from the current selection. # Retain the dependency closure before using facts as the deletion roots. prune_staged_store(store_dir, names) - python_fact = _facts().get("python") + facts = Facts(store_dir / "facts.json") + python_fact = facts.get("python") if python_fact is None: - print("✗ venv: no staged interpreter to build on") - return 1 + raise InstallError("venv", "no staged interpreter to build on") python_bin = get_package("python").binary( - _store().entry(python_fact["entry"]), current_target() + store_dir / python_fact["entry"], current_target() ) if python_bin is None: raise FileNotFoundError("staged Python executable is missing") - cache = Path(os.environ["UV_CACHE_DIR"]) stage_pm_runtime(out, python_bin, repo_dir, cache=cache) print("✓ pm-runtime (independent locked dependencies)", flush=True) @@ -202,28 +217,19 @@ def _stage_native(args) -> int: venv_dir = out / "venv" if venv_dir.exists(): shutil.rmtree(venv_dir) - env = dict(os.environ) + env = dict(os.environ if env is None else env) from pm import build_environment - from pm.package import InstallError - try: - # Cold native wheels need a larger budget than interactive installs. - build_environment(source=repo_dir, python=python_bin, out=venv_dir, - env=env, cache=cache, all_extras=True, sealed=True, explicit=True, - timeout=2 * 60 * 60) - except InstallError as exc: - print(f"✗ venv: {exc}") - return 1 + # Cold native wheels need a larger budget than interactive installs. + build_environment(source=repo_dir, python=python_bin, out=venv_dir, + env=env, cache=cache, all_extras=True, sealed=True, explicit=True, + timeout=2 * 60 * 60) print("✓ venv (all extras, on the staged interpreter)") # Inventory the staged interpreter before publishing the bundle contract. - from pm.features import FeatureProbeError, installed_extras, write_features + from pm.features import installed_extras, write_features - try: - features = installed_extras(repo_dir, venv_dir, python_exe=python_bin) - except FeatureProbeError as exc: - print(f"✗ features: {exc}") - return 1 + features = installed_extras(repo_dir, venv_dir, python_exe=python_bin) write_features(features, out) print(f"✓ enabled-features.json ({len(features)} extras recorded)") @@ -241,31 +247,71 @@ def _stage_native(args) -> int: stage_uv_cache(src_cache, payload_cache) print("✓ uv-cache (staged — warm rebuilds for the mutable venv)") else: - print(" uv-cache: none warm (first bundle on this machine?)") + raise InstallError("uv-cache", "runtime dependency cache is missing") bad = _arch_guard(store_dir) for line in bad: print(f"✗ arch: {line}") - failed += 1 - - if failed: - return 1 + if bad: + raise InstallError("tools", "native architecture verification failed") from scripts.bundles.payload import record_tools - recorded = {name: fact["entry"] for name in names if (fact := _facts().get(name)) and "entry" in fact} + recorded = {name: fact["entry"] for name in names if (fact := facts.get(name)) and "entry" in fact} record_tools(out, paths.lockfile_path(), current_target(), recorded) - from scripts.build.agent import assemble from scripts.build.inputs import AgentInputs, RESOURCE_ENV, dependency_site + from scripts.bundles.native_prepared import publish_prepared - assemble(AgentInputs( + site = dependency_site(venv_dir, python_fact["version"], current_target()) + (site / "hermes-agent.pth").write_text( + Path(os.path.relpath(repo_dir, site)).as_posix() + "\n", encoding="utf-8") + from scripts.bundles.payload import relativize_links + relativize_links(out) + inputs = AgentInputs( project=repo_dir / "pyproject.toml", code=repo_dir, repo="hermes-agent", placement="contained", target=current_target(), python=python_bin, - site_packages=dependency_site(venv_dir, python_fact["version"], current_target()), environment=venv_dir, + site_packages=site, environment=venv_dir, tools=store_dir, pm_runtime=out / "pm-runtime", ref=ref, resources={name: repo_dir / name for name in RESOURCE_ENV}, - frontends=getattr(args, "frontends", {}), features=out / "enabled-features.json", - ), out) + features=out / "enabled-features.json", + ) + return publish_prepared(out, source, revision, inputs) + + +def finish_native(prepared: Path, frontends: dict[str, Path]) -> int: + """Consume verified job-local paths. No dependency acquisition or repair.""" + from scripts.build.agent import assemble + from scripts.build.inputs import AgentInputs + from scripts.bundles.native_prepared import load_prepared, preparation_lock + + prepared = Path(prepared).absolute() + if not prepared.name.endswith(".prepared.json") or not prepared.is_file(): + raise ValueError("native preparation is missing or invalid; run preparation again") + out = prepared.with_name(prepared.name.removesuffix(".prepared.json")) + with preparation_lock(out): + (out / "manifest.json").unlink(missing_ok=True) + inputs = load_prepared(prepared) + values = asdict(inputs) + values["frontends"] = {name: Path(path).absolute() for name, path in frontends.items()} + assemble(AgentInputs.from_dict(values), out) print(f"✓ manifest ({out / 'manifest.json'})") - return 1 if failed else 0 + return 0 + + +def _stage_native(args) -> int: + from pm import paths + from pm.features import FeatureProbeError + from pm.package import InstallError + + try: + prepared = prepare_native( + out=Path(args.out), ref=args.ref or "HEAD", + source=getattr(args, "source", None) or paths.repo_root(), + cache=Path(getattr(args, "cache", None) or os.environ["UV_CACHE_DIR"]), + tools=getattr(args, "tools", None), + ) + return finish_native(prepared, getattr(args, "frontends", {})) + except (InstallError, FeatureProbeError) as exc: + print(f"✗ {exc}") + return 1 diff --git a/scripts/bundles/native_build.py b/scripts/bundles/native_build.py new file mode 100644 index 0000000000..ee9c26b3af --- /dev/null +++ b/scripts/bundles/native_build.py @@ -0,0 +1,114 @@ +"""Prepare or consume the standalone PM payload, without desktop products.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def prepare(source: Path, work: Path, cache: Path, out: Path, ref: str) -> Path: + from hermes_cli.runtime_state import _lock + from scripts.bundles.desktop_prepare import git, require_source + from scripts.bundles.desktop_toolchain import run_preparation + from scripts.bundles.native_prepared import prepared_path + from pm.lock import _write + + for path in (work, cache, out): + if path.absolute() != path.resolve(): + raise ValueError("native build roots must not be symlinked") + source, work, cache, out = (path.resolve() for path in (source, work, cache, out)) + revision = git(source, "rev-parse", "--verify", f"{ref}^{{commit}}") + require_source(source, revision) + for destination in (work, cache, out): + if source.is_relative_to(destination): + raise ValueError("native build output must not contain the source checkout") + for left, right in ((work, cache), (out, cache), (work, out)): + if left.is_relative_to(right) or right.is_relative_to(left): + raise ValueError("native work, cache and output must be separate directories") + owner = work / ".native-preparation" + if work.exists(): + if not owner.is_file() or owner.read_text(encoding="utf-8-sig") != str(source): + raise ValueError("native work directory is not owned by this checkout") + else: + work.mkdir(parents=True) + owner.write_text(str(source), encoding="utf-8") + with (work / ".lock").open("a+b") as lock: + if not _lock(lock.fileno(), wait=False): + raise ValueError("native preparation work directory is already in use") + result = prepared_path(out) + result.unlink(missing_ok=True) + request = work / "request.json" + _write(request, {"source": str(source), "work": str(work), "cache": str(cache), + "out": str(out), "ref": revision}) + status = run_preparation(source, work, cache, request, worker=Path(__file__).resolve()) + if status: + result.unlink(missing_ok=True) + raise RuntimeError(f"native preparation failed (exit {status})") + if not result.is_file(): + raise RuntimeError("native preparation did not publish its result") + return result + + +def prepare_in_worker(request: dict) -> Path: + from scripts.bundles.desktop_prepare import require_source + from scripts.bundles.desktop_toolchain import prepare_tools + from scripts.bundles.native import prepare_native + + source, work, cache, out = (Path(request[name]) for name in ("source", "work", "cache", "out")) + require_source(source, request["ref"]) + _, _, env = prepare_tools(source, work, cache, os.environ) + return prepare_native(out=out, ref=request["ref"], source=source, + cache=Path(env["UV_CACHE_DIR"]), tools=cache / "tools", env=env) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path) + parser.add_argument("--work", type=Path) + parser.add_argument("--cache", type=Path) + parser.add_argument("--out", type=Path) + selection = parser.add_mutually_exclusive_group() + selection.add_argument("--ref") + selection.add_argument("--commit") + parser.add_argument("--prepare-only", action="store_true") + parser.add_argument("--prepared", type=Path) + parser.add_argument("--request", type=Path, help=argparse.SUPPRESS) + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + try: + if args.worker: + if not args.request: + parser.error("worker requires --request") + prepare_in_worker(json.loads(args.request.read_text(encoding="utf-8-sig"))) + return 0 + if args.request: + parser.error("--request is worker-only") + if args.prepared: + if any((args.source, args.work, args.cache, args.out, args.ref, args.commit, args.prepare_only)): + parser.error("--prepared supplies the complete native build request") + prepared = args.prepared.absolute() + else: + if not all((args.source, args.work, args.cache, args.out)): + parser.error("preparation requires --source, --work, --cache and --out") + if args.commit: + from scripts.releases.commit_build import require_commit + require_commit(args.commit) + prepared = prepare(args.source, args.work, args.cache, args.out, args.commit or args.ref or "HEAD") + if args.prepare_only: + print(prepared) + return 0 + from scripts.bundles.native import finish_native + return finish_native(prepared, {}) + except (ValueError, OSError, RuntimeError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"native build: {exc}\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/bundles/native_prepared.py b/scripts/bundles/native_prepared.py new file mode 100644 index 0000000000..89b26adf6c --- /dev/null +++ b/scripts/bundles/native_prepared.py @@ -0,0 +1,141 @@ +"""Job-local native preparation admission; never a portable environment cache.""" +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import asdict +import hashlib +import json +import os +import re +import tomllib +from pathlib import Path + +from pm.store import current_target, tree_digest +from scripts.build.inputs import AgentInputs, RESOURCE_ENV + + +def prepared_path(out: Path) -> Path: + return out.with_name(out.name + ".prepared.json") + + +@contextmanager +def preparation_lock(out: Path): + from hermes_cli.runtime_state import _lock + from pm.filesystem import is_junction + + if out != out.resolve(): + raise ValueError("symlinked native output") + for name in ("hermes-agent", "tools", "venv", "pm-runtime", "uv-cache", "bin", + "enabled-features.json", "manifest.json"): + path = out / name + if path.is_symlink() or (path.exists() and is_junction(path)): + raise ValueError(f"symlinked native output: {path}") + with out.with_name(out.name + ".prepare.lock").open("a+b") as lock: + if not _lock(lock.fileno(), wait=False): + raise ValueError("native output is already in use") + yield + + +def _digest(path: Path) -> str: + return tree_digest(path) if path.is_dir() else hashlib.sha256(path.read_bytes()).hexdigest() + + +def _source_digest(code: Path) -> str: + """Ignore only assembly-owned products, leaving admitted source immutable.""" + project = tomllib.loads((code / "pyproject.toml").read_text(encoding="utf-8-sig"))["project"] + dist = re.sub(r"[-_.]+", "_", project["name"]) + generated = {"install-stamp.json", "hermes_cli/tui_dist", "hermes_cli/web_dist", + f"{dist}-{project['version']}.dist-info"} + files = {} + for directory, dirs, names in os.walk(code): + dirs[:] = [name for name in dirs if name != "__pycache__" + and (Path(directory) / name).relative_to(code).as_posix() not in generated] + for name in dirs: + path = Path(directory) / name + if path.is_symlink(): + files[path.relative_to(code).as_posix()] = os.readlink(path) + for name in names: + path = Path(directory) / name + relative = path.relative_to(code).as_posix() + if relative not in generated: + files[relative] = os.readlink(path) if path.is_symlink() else _digest(path) + return hashlib.sha256(json.dumps(files, sort_keys=True).encode()).hexdigest() + + +def _owned(path: Path, out: Path) -> None: + if not path.is_absolute() or not path.exists() or not path.resolve().is_relative_to(out): + raise ValueError(f"missing or escaped prepared path: {path}") + + +def _check_links(path: Path, out: Path) -> None: + from pm.filesystem import is_junction + + for directory, dirs, files in os.walk(path): + for name in dirs + files: + entry = Path(directory) / name + if entry.is_symlink() or is_junction(entry): + _owned(entry, out) + + +def _input_paths(inputs: AgentInputs, out: Path) -> list[Path]: + expected = {"code": out / "hermes-agent", "project": out / "hermes-agent/pyproject.toml", + "tools": out / "tools", "environment": out / "venv", "pm_runtime": out / "pm-runtime", + "features": out / "enabled-features.json"} + if (any(getattr(inputs, key) != path for key, path in expected.items()) + or inputs.placement != "contained" or inputs.repo != "hermes-agent" + or inputs.frontends or inputs.stamp or inputs.command_dir or inputs.env + or inputs.resources != {name: inputs.code / name for name in RESOURCE_ENV} + or not inputs.site_packages.is_relative_to(inputs.environment)): + raise ValueError("native prepared input layout changed") + paths = [inputs.code, inputs.tools, inputs.environment, inputs.pm_runtime, + inputs.python, inputs.features, out / "uv-cache"] + for path in [inputs.project, inputs.site_packages, *inputs.resources.values(), *paths]: + _owned(path, out) + return paths + + +def publish_prepared(out: Path, source: Path, revision: str, inputs: AgentInputs) -> Path: + from hermes_cli.runtime_state import _atomic_bytes + from pm.paths import lockfile_path + + inputs.validate(out) + paths = _input_paths(inputs, out) + digests = {} + for path in paths: + _owned(path, out) + _check_links(path, out) + digests[path.relative_to(out).as_posix()] = _source_digest(path) if path == inputs.code else _digest(path) + data = {"schema": 1, "out": str(out), "source": str(source), "revision": revision, + "lock": _digest(lockfile_path()), "inputs": asdict(inputs), "digests": digests} + prepared = prepared_path(out) + _atomic_bytes(prepared, (json.dumps(data, default=str, indent=2) + "\n").encode()) + return prepared + + +def load_prepared(prepared: Path) -> AgentInputs: + from pm.paths import lockfile_path + + try: + data = json.loads(prepared.read_text(encoding="utf-8-sig")) + out = Path(data["out"]) + if (prepared.is_symlink() or out != out.resolve() or data["schema"] != 1 + or prepared != prepared_path(out) or data["lock"] != _digest(lockfile_path()) + or not re.fullmatch(r"[a-f0-9]{40}", data["revision"])): + raise ValueError("native preparation identity changed") + inputs = AgentInputs.from_dict(data["inputs"]) + if inputs.target != current_target() or not inputs.ref: + raise ValueError("native preparation target/ref changed") + inputs.validate(out) + required = {path.relative_to(out).as_posix() for path in _input_paths(inputs, out)} + if set(data["digests"]) != required: + raise ValueError("native preparation inventory changed") + for name, digest in data["digests"].items(): + path = out / name + _owned(path, out) + _check_links(path, out) + actual = _source_digest(path) if path == inputs.code else _digest(path) + if actual != digest: + raise ValueError(f"prepared bytes changed: {name}") + return inputs + except (OSError, ValueError, KeyError, TypeError) as exc: + raise ValueError(f"invalid native preparation; run preparation again: {exc}") from exc \ No newline at end of file diff --git a/scripts/ci/desktop_build_cache.py b/scripts/ci/desktop_build_cache.py new file mode 100644 index 0000000000..e80612ee08 --- /dev/null +++ b/scripts/ci/desktop_build_cache.py @@ -0,0 +1,126 @@ +"""Describe the desktop dependency snapshot using only the runner's stdlib. + +Actions transports provider directories directly, including npm's existing +receipt. Restored bytes are candidates: preparation must still admit them. +This module never provisions tools or certifies a cache hit as prepared. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import platform +import re +import sys + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from scripts.ci.setup_toolchain import current_target, file_commands + + +CACHE_DOMAINS = ("tools", "python/runtime", "python/build", "npm/_cacache", "native", "packager") +_INPUT_FILES = ( + "package-lock.json", "pyproject.toml", "uv.lock", "pm/lock.json", "pm/pyproject.toml", "pm/uv.lock", + "scripts/ci/desktop_build_cache.py", "scripts/ci/setup_toolchain.py", + "scripts/bundles/desktop_prepare.py", "scripts/bundles/desktop_toolchain.py", + "scripts/bundles/desktop_inputs.py", "scripts/bundles/native.py", "scripts/bundles/native_prepared.py", + "scripts/bundles/native_build.py", + "scripts/build/node-deps.mjs", "scripts/build/icon_environment.py", + "scripts/build/windows_deps.py", "scripts/windows-build-deps.ps1", + "apps/desktop/scripts/stage-native-deps.mjs", "apps/desktop/scripts/prepare-packaging-tools.mjs", + "apps/desktop/scripts/windows-bundle-tools.mjs", "apps/desktop/scripts/prepared-native-deps.mjs", + "apps/desktop/scripts/prepared-packaging.mjs", "apps/desktop/scripts/prepare-dmgbuild.mjs", + "apps/desktop/scripts/prepare_dmgbuild.py", "apps/desktop/scripts/probe-prepared-native.mjs", +) + + +def _input_digest(source: Path, lock: dict) -> str: + # A coarse lookup hint, not a second receipt or dependency resolver. Owner + # admission remains necessary even when this digest matches exactly. + files: set[str] = set(_INPUT_FILES) + files.update(str(path.relative_to(source)) for path in (source / "pm").glob("*.py")) + files.update(str(Path(path) / "package.json") for path in lock["packages"] + if "node_modules" not in path.split("/")) + digest = hashlib.sha256() + for name in sorted(files): + path = source / name + digest.update(name.encode() + b"\0") + digest.update(path.read_bytes() if path.is_file() else b"") + digest.update(b"\0") + return digest.hexdigest() + + +def _workspace_path(source: Path, name: str) -> None: + if (not isinstance(name, str) or not name or + any(char in name for char in "\\:*?[]!\r\n\0") or + PurePosixPath(name).is_absolute() or + any(part in ("", ".", "..", "node_modules") for part in name.split("/"))): + raise ValueError(f"invalid locked workspace path: {name!r}") + path = source / name + if not path.resolve().is_relative_to(source) or not (path / "package.json").is_file(): + raise ValueError(f"missing or foreign locked workspace: {name!r}") + + +def _cache_path(path: Path) -> str: + if any(char in str(path) for char in "*?[]!\r\n\0") or path.resolve() != path: + raise ValueError(f"unsafe cache path: {path}") + return str(path) + + +def describe_cache(source: Path, cache: Path, producer: str, *, work: Path | None = None) -> dict: + source, cache = source.resolve(), cache.resolve() + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}", producer): + raise ValueError("producer must be a nonempty cache namespace") + if os.environ.get("HERMES_HOME") and cache.is_relative_to(Path(os.environ["HERMES_HOME"]).resolve()): + raise ValueError("cache must be separate from HERMES_HOME") + payload_only = producer == "payload-test" + lock = {"packages": {}} if payload_only else json.loads( + (source / "package-lock.json").read_text(encoding="utf-8-sig")) + # Match the Node owner's locked layout without needing Node before restore. + workspaces = sorted({entry["resolved"] for entry in lock["packages"].values() if entry.get("link")}) + for workspace in set(workspaces) | {name for name in lock["packages"] if name and "node_modules" not in name.split("/")}: + _workspace_path(source, workspace) + domains = ("tools", "python/runtime", "native") if payload_only else CACHE_DOMAINS + paths = [_cache_path(cache / domain) for domain in domains] + if not payload_only: + paths += [_cache_path(source / workspace / "node_modules") for workspace in ["", *workspaces]] + private_roots = {"source": source, "work": work, + "HERMES_HOME": Path(os.environ["HERMES_HOME"]) if os.environ.get("HERMES_HOME") else None} + for name, private in private_roots.items(): + if private is not None and any(private.resolve().is_relative_to(Path(path)) for path in paths): + raise ValueError(f"{name} must be outside every reusable cache path") + target = current_target() + host = hashlib.sha256(json.dumps([ + platform.platform(), os.environ.get("ImageOS", ""), os.environ.get("ImageVersion", ""), + ]).encode()).hexdigest()[:16] + prefix = f"desktop-inputs-v1-{producer}-{target}-{host}-" + input_prefix = f"{prefix}{_input_digest(source, lock)}-" + run = "-".join(os.environ.get(name, default) for name, default in ( + ("GITHUB_RUN_ID", "local"), ("GITHUB_RUN_ATTEMPT", "1"), ("GITHUB_JOB", "desktop"), + )) + return {"target": target, "paths": paths, "restore_keys": [input_prefix, prefix], + "key": input_prefix + run} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=["describe"]) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--work", type=Path) + parser.add_argument("--producer", required=True) + args = parser.parse_args() + description = describe_cache(args.source, args.cache, args.producer, work=args.work) + if os.environ.get("GITHUB_OUTPUT"): + file_commands("GITHUB_OUTPUT", { + "cache-key": description["key"], "cache-paths": json.dumps(description["paths"]), + "input-prefix": description["restore_keys"][0], "restore-prefix": description["restore_keys"][1], + }) + print(json.dumps(description)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ci/desktop_commands.py b/scripts/ci/desktop_commands.py new file mode 100644 index 0000000000..b643198950 --- /dev/null +++ b/scripts/ci/desktop_commands.py @@ -0,0 +1,45 @@ +"""Expose already-prepared tools to later CI audit/upload steps; never install.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +from scripts.bundles.desktop_prepare import PreparedDesktop +from scripts.ci.setup_toolchain import add_path, file_commands + + +def export_commands(path: Path) -> None: + prepared = PreparedDesktop.load(path) + prepared.validate() + directories = [str(prepared.python.parent), str(prepared.node.parent)] + # PBS provides python3 on POSIX. Keep the workflow's `python` command + # job-local: adding an alias inside the verified tool store invalidates it. + if os.name != "nt": + commands = prepared.request.work / "commands" + if commands.is_symlink(): + raise ValueError("CI commands directory must not be a symlink") + commands.mkdir(exist_ok=True) + alias = commands / "python" + alias.unlink(missing_ok=True) + alias.symlink_to(prepared.python) + directories.insert(0, str(commands)) + file_commands("GITHUB_ENV", { + "HERMES_PYTHON": prepared.python, + "HERMES_NODE": prepared.node, + "HERMES_HOME": prepared.request.work / "hermes-home", + "HERMES_RUNTIME_DIR": prepared.request.cache / "tools", + "PYTHONUTF8": "1", + "PYTHONDONTWRITEBYTECODE": "1", + }) + add_path(directories) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("prepared", type=Path) + export_commands(parser.parse_args().prepared) + + +if __name__ == "__main__": + main() diff --git a/tests-js/desktop-builder.test.mjs b/tests-js/desktop-builder.test.mjs index 5a44be30cf..7011acece3 100644 --- a/tests-js/desktop-builder.test.mjs +++ b/tests-js/desktop-builder.test.mjs @@ -162,6 +162,7 @@ test('a prepared input changing during desktop compilation cannot publish a curr test('native preparation stages the selected source into an explicit tree before compilation', async () => { const { prepareDesktopNativeDependencies } = await import('../apps/desktop/scripts/stage-native-deps.mjs') const input = fixture() + cpSync(join(repo, 'package-lock.json'), join(input.source, 'package-lock.json')) const nativeOut = join(dirname(input.out), 'prepared-native') await prepareDesktopNativeDependencies({ source: input.source, out: nativeOut }) expect(existsSync(join(nativeOut, 'node-pty/package.json'))).toBe(true) diff --git a/tests-js/node-deps.test.mjs b/tests-js/node-deps.test.mjs index 1d6b96e335..616f3263c3 100644 --- a/tests-js/node-deps.test.mjs +++ b/tests-js/node-deps.test.mjs @@ -66,6 +66,25 @@ test('read-only dependency preparation reuses complete receipts but refuses miss expect(readFileSync(join(source, 'node_modules/.hermes-node-deps'))).toEqual(receipt) }, 30000) +test('native toolchain admission rejects a changed compiler without breaking ordinary builders', async () => { + const { prepareNodeDependencies } = await import('../scripts/build/node-deps.mjs') + const source = fixture() + const options = { source, workspaces: ['web'], reuse: true, + env: { ...process.env, npm_config_offline: 'true', npm_config_cache: join(source, '.npm-cache') } } + prepareNodeDependencies({ ...options, nativeToolchain: 'native-first' }) + const artifact = join(source, 'node_modules/compiled-output') + writeFileSync(artifact, 'old compiler') + prepareNodeDependencies({ ...options, install: false }) + prepareNodeDependencies({ ...options, nativeToolchain: 'native-first', install: false }) + expect(() => prepareNodeDependencies({ ...options, nativeToolchain: 'native-second', install: false })).toThrow(/disabled/) + expect(readFileSync(artifact, 'utf8')).toBe('old compiler') + const cli = [join(repo, 'scripts/build/node-deps.mjs'), '--source', source, '--workspace', 'web', '--reuse', '--native-toolchain', 'native-second'] + execFileSync(process.execPath, cli, { env: options.env, stdio: 'pipe' }) + expect(existsSync(artifact)).toBe(false) + prepareNodeDependencies({ ...options, nativeToolchain: 'native-second', install: false }) + prepareNodeDependencies({ ...options, install: false }) +}, 30000) + test('one locked preparation retains the requested union without provisioning desktop', async () => { const { prepareNodeDependencies } = await import('../scripts/build/node-deps.mjs') const source = fixture() diff --git a/tests/hermes_cli/test_runtime_paths.py b/tests/hermes_cli/test_runtime_paths.py new file mode 100644 index 0000000000..ef1338f133 --- /dev/null +++ b/tests/hermes_cli/test_runtime_paths.py @@ -0,0 +1,67 @@ +"""Payload stores follow the payload, not the launching shell's home.""" + +import json + +import pytest + +from hermes_cli.runtime_paths import store_root +from hermes_constants import get_default_hermes_root + + +def test_store_resolution_follows_relocated_payload(tmp_path, monkeypatch): + monkeypatch.delenv("HERMES_RUNTIME_DIR", raising=False) + payload = tmp_path / "agent-payload" + repo = payload / "hermes-agent" + repo.mkdir(parents=True) + stamp = repo / "install-stamp.json" + stamp.write_text(json.dumps({"payload": "bundled", "runtime": { + "repoDir": "hermes-agent", "toolsDir": "tools", + }})) + manifest = payload / "manifest.json" + manifest.write_text(json.dumps({"schema": 1, "repo": "hermes-agent", + "venv": "venv", "store": "tools", + "runtime": {"toolsDir": "tools"}})) + tools = payload / "tools" + tools.mkdir() + facts = {"schema": 1, "packages": {"node": {"entry": "node-test"}}} + (tools / "facts.json").write_text(json.dumps(facts)) + (tools / "node-test").mkdir() + + for destination in (payload, tmp_path / "relocated payload"): + if destination != payload: + payload.rename(destination) + repo = destination / "hermes-agent" + resolved = store_root(repo) + assert resolved == destination / "tools" + installed = json.loads((resolved / "facts.json").read_text()) + assert (resolved / installed["packages"]["node"]["entry"]).is_dir() + + override = tmp_path / "stage-tools" + monkeypatch.setenv("HERMES_RUNTIME_DIR", str(override)) + assert store_root(repo) == override + monkeypatch.delenv("HERMES_RUNTIME_DIR") + (repo / "install-stamp.json").write_text(json.dumps({"runtimeDir": str(override)})) + (destination / "manifest.json").write_text(json.dumps({"repo": "other-repo"})) + assert store_root(repo) == override + (repo / "install-stamp.json").unlink() + assert store_root(repo) == get_default_hermes_root() / "tools" + + +@pytest.mark.parametrize("escape", ["relative", "absolute", "symlink"]) +def test_payload_store_cannot_escape_payload(tmp_path, monkeypatch, escape): + monkeypatch.delenv("HERMES_RUNTIME_DIR", raising=False) + payload = tmp_path / "agent-payload" + repo = payload / "hermes-agent" + repo.mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + values = {"relative": "../outside", "absolute": str(outside), "symlink": "tools"} + if escape == "symlink": + (payload / "tools").symlink_to(outside, target_is_directory=True) + (payload / "manifest.json").write_text(json.dumps({ + "repo": "hermes-agent", "store": values[escape], + })) + with pytest.raises(RuntimeError, match="payload store escapes its root"): + store_root(repo) + monkeypatch.setenv("HERMES_RUNTIME_DIR", str(outside)) + assert store_root(repo) == outside \ No newline at end of file diff --git a/tests/pm/test_build_operations.py b/tests/pm/test_build_operations.py index e2b30cce53..b67663de0c 100644 --- a/tests/pm/test_build_operations.py +++ b/tests/pm/test_build_operations.py @@ -15,9 +15,154 @@ from tests.pm._fixtures import ( build_worker as build_worker, client as client, isolated_python as isolated_python, + served as served, ) +def test_stage_tools_copies_verified_closure_without_acquiring_or_live_state(tmp_path, client, monkeypatch, served): + import importlib.util + import pm + from pm.lock import Facts, Lockfile + from pm.registry import _packages + from pm.store import current_target, tree_digest + + target = current_target() + source = tmp_path / "canonical" + source.mkdir() + lock = Lockfile(tmp_path / "lock.json") + facts = Facts(source / "facts.json") + definition = tmp_path / "copy_fixture.py" + definition.write_text( + "from pm.package import Package\n" + "class CopyLeaf(Package):\n" + " name = 'copy-leaf'\n" + " def env(self, entry, target): return {'COPY_ROOT': str(entry)}\n" + "class CopyTool(CopyLeaf):\n" + " name = 'copy-tool'\n" + " deps = ('copy-leaf',)\n", encoding="utf-8") + spec = importlib.util.spec_from_file_location("copy_fixture", definition) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, "copy_fixture", module) + spec.loader.exec_module(module) + for package in (module.CopyLeaf(), module.CopyTool()): + monkeypatch.setitem(_packages, package.name, package) + lock.set_pin(package.name, "1.0", {target: {"url": "https://invalid.test/tool.tgz", "sha256": "a" * 64}}) + entry = source / package.store_entry("1.0", target) + entry.mkdir() + (entry / "data").write_text(package.name, encoding="utf-8") + facts.record(package.name, "1.0", entry.name, package.env(entry, target), source, + target=target, artifacts=["a" * 64], digest=tree_digest(entry)) + lock.save() + from tests.pm._fixtures import make_tar + import shutil + directory, base = served + for name in ("copy-leaf", "copy-tool"): + archive, sha = make_tar(directory, name + ".tgz", {"data": name}) + lock.set_pin(name, "1.0", {target: {"url": base + "/" + archive, "sha256": sha}}) + lock.save() + shutil.rmtree(source) + assert pm.prepare_tools(["copy-tool"], out=source, target=target, cache=tmp_path / "python-cache") == source + facts = Facts(source / "facts.json") + before = (source / "facts.json").read_bytes() + home_before = sorted(str(path) for path in (tmp_path / "home").rglob("*")) + destination = tmp_path / "payload-tools" + result = pm.stage_tools(["copy-tool"], source_store=source, out=destination, target=target) + assert result == destination + # An old output may have been populated with hardlinks. A fresh staging + # request must establish independent bytes, not reuse those current facts. + tool_entry = facts.get("copy-tool")["entry"] + linked_copy = destination / tool_entry / "data" + linked_copy.unlink() + os.link(source / tool_entry / "data", linked_copy) + pm.stage_tools(["copy-tool"], source_store=source, out=destination, target=target) + staged = Facts(destination / "facts.json") + for name in ("copy-leaf", "copy-tool"): + entry = facts.get(name)["entry"] + copied, original = destination / entry / "data", source / entry / "data" + assert copied.read_bytes() == original.read_bytes() + assert not copied.samefile(original) + copied.write_text("signed output", encoding="utf-8") + assert original.read_text() == name + assert staged.get(name)["digest"] == facts.get(name)["digest"] + assert staged.env_for(name, destination)["COPY_ROOT"] == str(destination / entry) + assert (source / "facts.json").read_bytes() == before + assert sorted(str(path) for path in (tmp_path / "home").rglob("*")) == home_before + assert not (tmp_path / "home/config.yaml").exists() + assert not (tmp_path / "store/facts.json").exists() + + # Every call must admit the source, even if destination facts are warm. + original = source / facts.get("copy-tool")["entry"] / "data" + original.write_text("corrupt", encoding="utf-8") + with pytest.raises(InstallError, match="source failed verification"): + pm.stage_tools(["copy-tool"], source_store=source, out=destination, target=target) + original.write_text("copy-tool", encoding="utf-8") + lock.set_pin("copy-tool", "1.0", {target: {"url": "https://invalid.test/repin", "sha256": "b" * 64}}) + lock.save() + with pytest.raises(InstallError, match="source failed verification"): + pm.stage_tools(["copy-tool"], source_store=source, out=destination, target=target) + lock.set_pin("copy-tool", "1.0", {target: {"url": "https://invalid.test/tool.tgz", "sha256": facts.get("copy-tool")["artifacts"][0]}}) + lock.save() + # A locally recorded link must not turn an independent copy into a shared + # mutable file in the cache or another caller's tree. + external = tmp_path / "external" + external.write_text("shared", encoding="utf-8") + original.unlink() + original.symlink_to(external) + package = module.CopyTool() + entry = original.parent + facts.record(package.name, "1.0", entry.name, package.env(entry, target), source, + target=target, artifacts=facts.get("copy-tool")["artifacts"], digest=tree_digest(entry)) + with pytest.raises(InstallError, match="source.*verification|escapes"): + pm.stage_tools(["copy-tool"], source_store=source, out=tmp_path / "other-output", target=target) + + +@pytest.mark.parametrize("damage", [None, "entry", "env", "bytes", "pin", "binary"]) +def test_verified_tools_admits_only_locked_entries_without_execution(tmp_path, monkeypatch, damage): + import subprocess + from pm import build_operations + from pm.lock import Facts, Lockfile + from pm.registry import get_package + from pm.store import current_target, tree_digest + + target, store = current_target(), tmp_path / "tools" + lock = Lockfile(tmp_path / "lock.json") + package = get_package("python") + entry = store / package.store_entry("1.0", target) + binary = package.binary(entry, target) + assert binary is not None + binary.parent.mkdir(parents=True) + binary.write_bytes(b"already verified at publication") + lock.set_pin("python", "1.0", {target: {"url": "https://invalid.test/python", "sha256": "a" * 64}}) + facts = Facts(store / "facts.json") + env = package.env(entry, target) + if damage == "entry": + renamed = entry.with_name("noncanonical") + entry.rename(renamed) + entry = renamed + env = package.env(entry, target) + if damage == "env": + env["PATH"] = [str(tmp_path / "unverified")] + if damage == "binary": + binary.unlink() + facts.record("python", "1.0", entry.name, env, store, target=target, + artifacts=["a" * 64], digest=tree_digest(entry)) + if damage == "bytes": + binary.write_bytes(b"changed") + if damage == "pin": + lock.set_pin("python", "1.0", {target: {"url": "https://invalid.test/python", "sha256": "b" * 64}}) + before = {p.relative_to(store): p.read_bytes() for p in store.rglob("*") if p.is_file()} + monkeypatch.setattr(subprocess, "run", lambda *a, **kw: pytest.fail("read-only admission executed a process")) + if damage: + with pytest.raises(InstallError, match="source failed verification"): + build_operations.verified_tools(["python"], source_store=store, target=target, lock=lock) + else: + selected = build_operations.verified_tools(["python"], source_store=store, target=target, lock=lock) + assert selected.entries["python"].path == entry + assert selected.entries["python"].binary == binary + assert selected.environment({"PATH": "inherited"})["PATH"] == str(binary.parent) + os.pathsep + "inherited" + assert {p.relative_to(store): p.read_bytes() for p in store.rglob("*") if p.is_file()} == before + + @pytest.fixture def build_tools(tmp_path, monkeypatch, build_worker): env = {key: value for key, value in os.environ.items() diff --git a/tests/pm/test_dmgbuild_package.py b/tests/pm/test_dmgbuild_package.py new file mode 100644 index 0000000000..b07b387253 --- /dev/null +++ b/tests/pm/test_dmgbuild_package.py @@ -0,0 +1,46 @@ +"""The build-only DMG supplier preserves its paired runtime without executing it.""" +import struct +import tarfile + +import pytest + +from pm import Lockfile, Store, get_package, paths +from pm.store import ALL_TARGETS +from scripts.bundles.native import _bundle_package_names + + +@pytest.mark.parametrize("target,cpu", [("darwin-arm64", 0x0100000C), ("darwin-x64", 0x01000007)]) +def test_dmgbuild_stages_paired_runtime_and_stays_out_of_payload(tmp_path, monkeypatch, target, cpu): + package = get_package("dmgbuild") + lock = Lockfile(paths.lockfile_path()) + version = lock.version(package.name) + assert version is not None + assert lock.artifacts(package.name, target)[0]["url"] == package.fetch_url(version, target) + assert package.latest_versions(target, locked=version) == [] + assert package.internal and not package.on_path + assert package.name not in _bundle_package_names() + assert all(package.missing_reason(t) for t in ALL_TARGETS if not t.startswith("darwin-")) + payload = tmp_path / "payload" + python = payload / "python/bin/python3" + python.parent.mkdir(parents=True) + python.write_bytes(struct.pack(" tuple[Path, str]: + from pm.store import current_target + source = tmp_path / "source with spaces" + source.mkdir() + (source / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.2.3"\n', encoding="utf-8") + (source / "package.json").write_text('{"workspaces": ["apps/desktop"]}', encoding="utf-8") + (source / "package-lock.json").write_text('{"packages": {}}', encoding="utf-8") + (source / ".gitignore").write_text(".build/\n", encoding="utf-8") + (source / "pm").mkdir() + (source / "pm/lock.json").write_text(json.dumps({"schema": 1, "packages": { + name: {"version": "1.0.0", "artifacts": {current_target(): {"url": "https://example.test/tool", "sha256": "a" * 64}}} + for name in ("python", "node", "npm") + }}), encoding="utf-8") + subprocess.run(["git", "init", str(source)], check=True, capture_output=True) + subprocess.run(["git", "add", "."], cwd=source, check=True) + subprocess.run(["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.test", "commit", "-m", "fixture"], cwd=source, check=True, capture_output=True) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + return source, commit + + +def test_prepared_input_roundtrip_rejects_mutation_and_foreign_source(tmp_path): + from scripts.bundles.desktop_prepare import BuildRequest, PreparedDesktop + + source, commit = _project(tmp_path) + work = tmp_path / "work" + cache = tmp_path / "cache" + request = BuildRequest.create(source, tag=None, commit=commit, variant="light", work=work, cache=cache, bundle_env={}) + from pm.lock import Facts + from pm.store import tree_digest + from pm.registry import get_package + store = cache / "tools" + binaries = {} + for name in ("python", "node", "npm"): + package = get_package(name) + entry = store / package.store_entry("1.0.0", request.target) + binary = package.binary(entry, request.target) + assert binary is not None + binary.parent.mkdir(parents=True) + binary.write_bytes(b"published tool fixture") + binaries[name] = binary + Facts(store / "facts.json").record(name, "1.0.0", entry.name, package.env(entry, request.target), store, + target=request.target, artifacts=["a" * 64], digest=tree_digest(entry)) + work.mkdir() + executable, node = binaries["python"], binaries["node"] + icons = work / "icon-environment" / "bin" / "python" + icons.parent.mkdir(parents=True) + icons.write_bytes(b"prepared icon interpreter") + icon_library = work / "icon-environment" / "library" + icon_library.write_bytes(b"prepared library") + native = work / "native" + native.mkdir() + (native / "binding.node").write_bytes(b"prepared native input") + packager = work / "packager.json" + packager.write_text("{}", encoding="utf-8") + prepared = PreparedDesktop.record(request, python=executable, node=node, icon_python=icons, + native=native, packager=packager, payload=None, native_toolchain="fixture-toolchain") + path = work / "prepared.json" + prepared.write(path) + restored = PreparedDesktop.load(path) + restored.validate() + assert restored.request.commit == commit + assert restored.request.version == "1.2.3" + assert "environment" not in json.loads(path.read_text()) + icon_library.unlink() + with pytest.raises(ValueError, match="changed|stale|missing"): + restored.validate() + icon_library.write_bytes(b"prepared library") + (native / "binding.node").write_bytes(b"changed") + with pytest.raises(ValueError, match="changed|stale"): + restored.validate() + (native / "binding.node").write_bytes(b"prepared native input") + (source / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="9.9.9"\n', encoding="utf-8") + with pytest.raises(ValueError, match="source|checkout"): + restored.validate() + + +def test_prepared_paths_cannot_escape_their_owned_roots(tmp_path): + from scripts.bundles.desktop_prepare import BuildRequest, PreparedDesktop + + source, commit = _project(tmp_path) + work, cache = tmp_path / "work", tmp_path / "cache" + request = BuildRequest.create(source, tag=None, commit=commit, variant="light", work=work, cache=cache, bundle_env={}) + external = tmp_path / "external" + external.write_bytes(b"not preparation-owned") + with pytest.raises(ValueError, match="outside|owned"): + PreparedDesktop.record(request, python=external, node=external, icon_python=external, + native=external, packager=external, payload=None, native_toolchain="fixture-toolchain") + + +def test_prepared_cli_rejects_missing_result_without_provisioning(tmp_path): + source = Path(__file__).resolve().parents[2] + absent = tmp_path / "absent" / "prepared.json" + result = subprocess.run([sys.executable, str(source / "scripts/bundles/desktop.py"), "--prepared", str(absent)], + cwd=tmp_path, capture_output=True, text=True, timeout=30) + assert result.returncode != 0 + assert "preparation" in result.stderr.lower() + assert "unrecognized arguments" not in result.stderr + assert not absent.parent.exists() + + +def test_source_admission_rejects_dirty_checkout_and_invalid_store_before_writes(tmp_path): + from scripts.bundles.desktop_prepare import BuildRequest + + source, commit = _project(tmp_path) + work = tmp_path / "work" + cache = tmp_path / "cache" + for tag, selected in [(None, commit), ("v1.2.0-canary.20260911120000", None)]: + with pytest.raises(ValueError, match="Store.*stable"): + BuildRequest.create(source, tag=tag, commit=selected, variant="store", work=work, cache=cache, bundle_env={}) + (source / "package.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="source|checkout"): + BuildRequest.create(source, tag=None, commit=commit, variant="light", work=work, cache=cache, bundle_env={}) + assert not work.exists() + assert not cache.exists() + + +def test_preparation_never_reuses_an_unowned_work_directory(tmp_path): + from scripts.bundles.desktop_prepare import BuildRequest, prepare + + source, commit = _project(tmp_path) + work = tmp_path / "someone-elses-files" + work.mkdir() + precious = work / "request.json" + precious.write_text("do not replace", encoding="utf-8") + request = BuildRequest.create(source, tag=None, commit=commit, variant="light", + work=work, cache=tmp_path / "cache", bundle_env={}) + with pytest.raises(ValueError, match="owned"): + prepare(request) + assert precious.read_text() == "do not replace" + + +def test_checkout_lock_excludes_a_second_build_process(tmp_path): + from scripts.bundles.desktop_inputs import build_lock + + source, _ = _project(tmp_path) + project = Path(__file__).resolve().parents[2] + probe = ( + "import sys; from pathlib import Path; sys.path.insert(0,sys.argv[1]); " + "from scripts.bundles.desktop_inputs import build_lock; " + "lock=build_lock(Path(sys.argv[2])); lock.__enter__(); print('acquired'); lock.__exit__(None,None,None)" + ) + command = [sys.executable, "-I", "-c", probe, str(project), str(source)] + with build_lock(source): + blocked = subprocess.run(command, capture_output=True, text=True, timeout=30) + assert blocked.returncode != 0 and "another desktop" in blocked.stderr + released = subprocess.run(command, capture_output=True, text=True, timeout=30) + assert released.returncode == 0, released.stderr + assert released.stdout.strip() == "acquired" diff --git a/tests/scripts/test_desktop_toolchain.py b/tests/scripts/test_desktop_toolchain.py new file mode 100644 index 0000000000..f73a937c2e --- /dev/null +++ b/tests/scripts/test_desktop_toolchain.py @@ -0,0 +1,286 @@ +"""Desktop preparation keeps dependency acquisition out of live installs.""" +from __future__ import annotations + +import importlib +import os +from pathlib import Path +import json +import subprocess +import sys +from types import SimpleNamespace + +import pytest + + +def test_bootstrap_environment_isolates_owned_paths_without_mutating_caller(tmp_path): + toolchain = importlib.import_module("scripts.bundles.desktop_toolchain") + source, work, cache = (tmp_path / name for name in ("source", "work", "cache")) + original_home = tmp_path / "live-user" + inherited = { + "HOME": str(original_home), "USERPROFILE": str(original_home), + "LOCALAPPDATA": str(original_home / "AppData/Local"), + "APPDATA": str(original_home / "AppData/Roaming"), + "HERMES_HOME": str(original_home / "profile"), + "HERMES_RUNTIME_DIR": str(original_home / "tools"), + "HERMES_INSTALL_ROOT": str(original_home / "installed"), + "HERMES_PAYLOAD_ROOT": str(original_home / "payload"), + "HERMES_PAYLOAD_TAG": "v1.2.3", "HERMES_BUILD_COMMIT": "a" * 40, + "HERMES_SITE": str(original_home / "site"), + "HERMES_PYTHON_SRC_ROOT": str(original_home / "repo"), + "HERMES_PYTHON": str(original_home / "python"), + "HERMES_NODE": str(original_home / "node"), + "HERMES_PROFILE": "live", "HERMES_REAL_HOME": str(original_home), + "HERMES_BUNDLED_SKILLS": str(original_home / "skills"), + "HERMES_OPTIONAL_MCPS": str(original_home / "mcps"), + "VIRTUAL_ENV": str(original_home / "venv"), + "PYTHONHOME": str(original_home / "python-home"), + "PYTHONPATH": str(original_home / "site"), + "UV_CACHE_DIR": str(original_home / "uv-cache"), + "npm_config_cache": str(original_home / "npm-cache"), + "npm_execpath": str(original_home / "foreign-npm.js"), + "NPM_CONFIG_USERCONFIG": str(original_home / "credentials.npmrc"), + "CARGO_HOME": str(original_home / "custom-cargo"), + "RUSTUP_HOME": str(original_home / "custom-rustup"), + "PATH": os.defpath, "SIGNING_TOKEN": "inherited-not-serialized", + } + before = inherited.copy() + process_before = dict(os.environ) + environment = toolchain.bootstrap_environment(source, work, cache, inherited) + assert inherited == before + assert dict(os.environ) == process_before + assert environment["CARGO_HOME"] == inherited["CARGO_HOME"] + assert environment["RUSTUP_HOME"] == inherited["RUSTUP_HOME"] + for key in ("HOME", "USERPROFILE", "LOCALAPPDATA", "APPDATA", "HERMES_HOME", + "XDG_CONFIG_HOME", "XDG_CACHE_HOME"): + assert Path(environment[key]).is_relative_to(work) + assert Path(environment["HERMES_RUNTIME_DIR"]) == cache / "tools" + assert Path(environment["UV_CACHE_DIR"]) == cache / "python/runtime" + assert Path(environment["npm_config_cache"]) == cache / "npm" + assert "npm_execpath" not in environment + assert "NPM_CONFIG_USERCONFIG" not in environment + assert environment["npm_config_userconfig"] == os.devnull + assert "npm_config_globalconfig" not in environment + assert environment["HERMES_PYTHON_SRC_ROOT"] == str(source) + for key in ("HERMES_INSTALL_ROOT", "HERMES_PAYLOAD_ROOT", "HERMES_PAYLOAD_TAG", + "HERMES_BUILD_COMMIT", "HERMES_SITE", "HERMES_PROFILE", "HERMES_REAL_HOME", + "HERMES_PYTHON", "HERMES_NODE", "HERMES_BUNDLED_SKILLS", "HERMES_OPTIONAL_MCPS", + "VIRTUAL_ENV", "PYTHONHOME", "PYTHONPATH"): + assert key not in environment + assert environment["SIGNING_TOKEN"] == inherited["SIGNING_TOKEN"] + assert environment["PATH"] == inherited["PATH"] + assert not work.exists() # Forming the child environment is pure. + + defaults = toolchain.bootstrap_environment(source, work, cache, {"HOME": str(original_home), + "USERPROFILE": str(original_home)}) + assert defaults["CARGO_HOME"] == str(original_home / ".cargo") + assert defaults["RUSTUP_HOME"] == str(original_home / ".rustup") + + mixed = toolchain.bootstrap_environment(source, work, cache, { + "Home": str(original_home), "UserProfile": str(original_home), "LocalAppData": "live-local", + "Hermes_Home": "live-profile", "Hermes_Runtime_Dir": "live-store", "Cargo_Home": "custom-cargo", + "Rustup_Home": "custom-rustup", "NPM_CONFIG_CACHE": "live-npm", "Path": os.defpath, + }) + assert mixed["CARGO_HOME"] == "custom-cargo" and mixed["RUSTUP_HOME"] == "custom-rustup" + assert len([key for key in mixed if key.upper() == "HOME"]) == 1 + assert len([key for key in mixed if key.upper() == "HERMES_RUNTIME_DIR"]) == 1 + + +@pytest.mark.parametrize("exit_code", [0, 7]) +@pytest.mark.platforms("linux", "macos", "windows") +def test_run_preparation_bootstraps_before_worker_in_isolated_child(tmp_path, monkeypatch, exit_code): + from scripts.bundles import desktop_toolchain + + source, work, cache = (tmp_path / name for name in ("source space", "work", "cache")) + (source / "pm").mkdir(parents=True) + (source / "pm/__init__.py").write_text("", encoding="utf-8") + # A bootstrap double runs in a REAL child: imports must already be isolated, + # not temporarily redirected in the parent's process around the PM call. + (source / "pm/runtime.py").write_text( + "import os, sys\n" + "from pathlib import Path\n" + "assert Path(os.environ['HOME']).name == 'home'\n" + "assert 'HERMES_INSTALL_ROOT' not in os.environ\n" + "assert sys.flags.isolated and sys.flags.no_site\n" + "def runtime_command(script, args, *, cache):\n" + " assert cache == Path(os.environ['UV_CACHE_DIR'])\n" + " return [sys.executable, '-I', '-B', str(script), *args]\n", + encoding="utf-8", + ) + worker = source / "scripts/bundles/desktop_prepare.py" + worker.parent.mkdir(parents=True) + worker.write_text( + "import json, os, sys\n" + "from pathlib import Path\n" + "assert sys.argv[1] == '--request' and sys.argv[3:] == ['--worker']\n" + "import ssl\n" + "assert ssl.SSLContext.__module__.startswith('truststore')\n" + "request = json.loads(Path(sys.argv[2]).read_text())\n" + "assert os.environ['HERMES_RUNTIME_DIR'] == request['tools']\n" + "Path(request['receipt']).write_text(str(Path.cwd()))\n" + "sys.exit(request['exit_code'])\n", encoding="utf-8", + ) + request_file = tmp_path / "request space.json" + receipt = tmp_path / "worker-ran" + request_file.write_text(json.dumps({"receipt": str(receipt), "tools": str(cache / "tools"), + "exit_code": exit_code}), encoding="utf-8") + monkeypatch.setenv("HERMES_INSTALL_ROOT", str(tmp_path / "live-install")) + before = dict(os.environ) + assert desktop_toolchain.run_preparation(source, work, cache, request_file) == exit_code + assert dict(os.environ) == before + assert receipt.read_text(encoding="utf-8-sig") == str(source) + + +def test_bootstrap_real_pm_resolves_only_build_owned_state(tmp_path): + from scripts.bundles.desktop_toolchain import bootstrap_environment + + source = Path(__file__).resolve().parents[2] + work, cache = tmp_path / "work", tmp_path / "cache" + env = bootstrap_environment(source, work, cache, os.environ) + probe = ( + "import json, sys; from pathlib import Path; sys.path.insert(0, sys.argv[1]); " + "from pm import paths; from pm.packages import uv_cache_dir; " + "from hermes_cli.runtime_paths import install_state_dir; " + "print(json.dumps([str(paths.store_root()), str(paths.partials_root()), " + "str(uv_cache_dir()), str(install_state_dir(Path(sys.argv[1])))]))" + ) + result = subprocess.run([sys.executable, "-I", "-S", "-B", "-c", probe, str(source)], + env=env, capture_output=True, text=True, check=True) + store, partials, uv_default, state = map(Path, json.loads(result.stdout)) + assert store == cache / "tools" + assert all(path.is_relative_to(work) for path in (partials, uv_default, state)) + + +@pytest.mark.platforms("linux", "macos", "windows") +def test_prepare_tools_uses_pm_native_pins_and_separate_cache(tmp_path, monkeypatch): + import pm + from scripts.bundles import desktop_toolchain + from scripts.build import windows_deps + + source, work, cache = (tmp_path / name for name in ("source", "work", "cache")) + env = desktop_toolchain.bootstrap_environment(source, work, cache, os.environ) + monkeypatch.setenv("HERMES_RUNTIME_DIR", env["HERMES_RUNTIME_DIR"]) + acquired, native_calls = [], [] + bins = {name: cache / "tools" / name / "bin" / name for name in ("python", "node")} + for binary in bins.values(): + binary.parent.mkdir(parents=True) + binary.touch() + + def ensure(name, *, explicit): + assert explicit + acquired.append(name) + + def native(**kwargs): + native_calls.append(kwargs) + return {**kwargs["env"], "OPENSSL_DIR": str(cache / "native/openssl")} + + monkeypatch.setattr(pm, "ensure", ensure) + monkeypatch.setattr(pm, "installed_package", lambda name: SimpleNamespace(binary=bins[name])) + monkeypatch.setattr(pm, "env_for", lambda *names, base_env: {**base_env, "PATH": "pm-tools"}) + monkeypatch.setattr(windows_deps, "prepare_windows_environment", native) + monkeypatch.setattr(desktop_toolchain, "native_cache_path", lambda cache, env: cache / "python/runtime/native-identity", + raising=False) + before = dict(os.environ) + python, node, prepared = desktop_toolchain.prepare_tools(source, work, cache, env) + assert acquired == ["uv", "npm"] + assert python == bins["python"] and node == bins["node"] + assert prepared["HERMES_PYTHON"] == str(python) + assert prepared["HERMES_NODE"] == str(node) + assert prepared["PATH"] == "pm-tools" + assert Path(prepared["UV_CACHE_DIR"]) == cache / "python/runtime/native-identity" + assert dict(os.environ) == before + if pm.current_target() == "win32-arm64": + assert len(native_calls) == 1 + assert native_calls[0]["state"].is_relative_to(cache / "native") + assert prepared["OPENSSL_DIR"] == str(cache / "native/openssl") + else: + assert native_calls == [] + + +@pytest.mark.platforms("linux", "macos", "windows") +def test_native_cache_identity_tracks_compilers_sdk_and_openssl(tmp_path, monkeypatch): + from scripts.bundles import desktop_toolchain + + openssl = tmp_path / "openssl" + (openssl / "include/openssl").mkdir(parents=True) + (openssl / "include/openssl/opensslv.h").write_bytes(b"version one") + (openssl / "lib").mkdir() + library = openssl / "lib/libcrypto.lib" + library.write_bytes(b"first build") + (openssl / "lib/libssl.lib").write_bytes(b"ssl build") + env = {"PATH": os.defpath, "OPENSSL_DIR": str(openssl), + "WindowsSDKVersion": "10.0.1", "VCToolsVersion": "14.1"} + version = ["compiler one"] + + def probe(command, **kwargs): + assert kwargs["env"] == env + return SimpleNamespace(returncode=0, stdout=version[0], stderr="") + + # Let stdlib cache its real-host probe before substituting compiler commands. + desktop_toolchain.platform.platform() + monkeypatch.setattr(desktop_toolchain.subprocess, "run", probe) + cache = tmp_path / "cache" + first = desktop_toolchain.native_cache_path(cache, env) + assert first.is_relative_to(cache / "python/runtime") + assert desktop_toolchain.native_cache_path(cache, env) == first + moved_cache = tmp_path / "relocated-cache" + moved_openssl = moved_cache / "native/openssl" + import shutil + shutil.copytree(openssl, cache / "native/openssl") + shutil.copytree(openssl, moved_openssl) + env["OPENSSL_DIR"] = str(cache / "native/openssl") + portable = desktop_toolchain.native_cache_path(cache, env) + env["OPENSSL_DIR"] = str(moved_openssl) + assert desktop_toolchain.native_cache_path(moved_cache, env).name == portable.name + env["OPENSSL_DIR"] = str(openssl) + version[0] = "compiler two" + second = desktop_toolchain.native_cache_path(cache, env) + assert second != first + library.write_bytes(b"changed build") + third = desktop_toolchain.native_cache_path(cache, env) + assert third != second + env["WindowsSDKVersion"] = "10.0.2" + assert desktop_toolchain.native_cache_path(cache, env) != third + monkeypatch.setattr(desktop_toolchain.subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError())) + assert desktop_toolchain.native_cache_path(cache, env) != desktop_toolchain.native_cache_path(cache, env) + + +@pytest.mark.parametrize("on_demand", [False, True]) +def test_icon_environment_prepares_locked_group_before_generation(tmp_path, monkeypatch, on_demand): + import pm + from scripts.build import icon_environment + + source, out, cache = (tmp_path / name for name in ("source", "icons-venv", "build-cache")) + python = out / "bin/python" + acquired = [] + + def build(**kwargs): + acquired.append(kwargs) + return python + + monkeypatch.setattr(pm, "build_environment", build) + assert icon_environment.prepare_icon_environment(source, out, cache) == python + assert acquired == [{"source": source, "out": out, "cache": cache, + "groups": ["icon-build"], "only_groups": True, "explicit": True}] + calls = [] + + def prepare(source, out, cache, *, explicit=True): + calls.append((source, out, cache, explicit)) + return python + + launched = [] + + def run(command, *, cwd): + launched.append(command) + assert calls and cwd == source + return SimpleNamespace(returncode=9) + + monkeypatch.setattr(icon_environment, "prepare_icon_environment", prepare) + monkeypatch.setattr(icon_environment.subprocess, "run", run) + args = ["--source", str(source), "--out", str(tmp_path / "icons")] + if on_demand: + args.append("--on-demand") + assert icon_environment.main(args) == 9 + assert calls[0][0] == source and calls[0][2] == source / ".cache/icon-build" + assert calls[0][3] is not on_demand + assert launched[0][:2] == [str(python), "-I"] + assert "--on-demand" not in launched[0] diff --git a/tests/scripts/test_native_build.py b/tests/scripts/test_native_build.py new file mode 100644 index 0000000000..71a1024293 --- /dev/null +++ b/tests/scripts/test_native_build.py @@ -0,0 +1,99 @@ +"""PM Bundle shares preparation, not the desktop product dependency closure.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/bundles/native_build.py" + + +def test_payload_transport_does_not_require_or_export_desktop_inputs(tmp_path): + from scripts.ci.desktop_build_cache import describe_cache + + source = tmp_path / "source" + source.mkdir() + cache = tmp_path / "cache" + description = describe_cache(source, cache, "payload-test") + assert set(description["paths"]) == {str(cache / name) for name in ("tools", "python/runtime", "native")} + assert "payload-test" in description["key"] + assert not cache.exists() + + +def test_native_worker_passes_shared_toolchain_to_native_owner(tmp_path, monkeypatch): + from scripts.bundles import native_build + from scripts.bundles import desktop_toolchain, native + + request = {name: str(tmp_path / name) for name in ("source", "work", "cache", "out")} + request["ref"] = "a" * 40 + selected = {"UV_CACHE_DIR": str(tmp_path / "native-wheel-identity"), "CC": "prepared-compiler"} + monkeypatch.setattr("scripts.bundles.desktop_prepare.require_source", lambda *args: None) + prepared_tools = [] + def tools(source, work, cache, inherited): + prepared_tools.append((source, work, cache)) + return Path("python"), Path("node"), selected + monkeypatch.setattr(desktop_toolchain, "prepare_tools", tools) + calls = [] + def prepare_native(**kwargs): + calls.append(kwargs) + return tmp_path / "out.prepared.json" + monkeypatch.setattr(native, "prepare_native", prepare_native) + assert native_build.prepare_in_worker(request) == tmp_path / "out.prepared.json" + assert prepared_tools == [(tmp_path / "source", tmp_path / "work", tmp_path / "cache")] + assert calls == [{"source": tmp_path / "source", "out": tmp_path / "out", "ref": request["ref"], + "cache": Path(selected["UV_CACHE_DIR"]), "tools": tmp_path / "cache/tools", "env": selected}] + + +@pytest.mark.platforms("posix") +def test_native_cli_consumes_real_minimal_preparation_without_bootstrap(tmp_path): + from scripts.build.inputs import AgentInputs, RESOURCE_ENV + from scripts.bundles.native_prepared import publish_prepared + from pm.store import current_target + + out = tmp_path / "payload" + code = out / "hermes-agent" + code.mkdir(parents=True) + (code / "pyproject.toml").write_text('[project]\nname="native-fixture"\nversion="1.0"\n[project.scripts]\nprobe="entry:main"\n') + (code / "entry.py").write_text('def main():\n print("native fixture")\n return 0\n') + for name in RESOURCE_ENV: + (code / name).mkdir() + python = out / "tools/python/bin/python3" + python.parent.mkdir(parents=True) + shutil.copy2(Path(getattr(sys, "_base_executable")).resolve(), python) + site = out / "venv/lib/site-packages" + site.mkdir(parents=True) + runtime = out / "pm-runtime" + (runtime / "lib/site-packages").mkdir(parents=True) + (runtime / "pm-runtime.json").write_text(json.dumps({"python": "../tools/python/bin/python3", "sitePackages": "lib/site-packages"})) + features = out / "enabled-features.json" + features.write_text('{"extras":[]}') + (out / "uv-cache").mkdir() + prepared = publish_prepared(out, ROOT, "a" * 40, AgentInputs( + project=code / "pyproject.toml", code=code, repo="hermes-agent", placement="contained", + target=current_target(), python=python, site_packages=site, environment=out / "venv", + tools=out / "tools", pm_runtime=runtime, features=features, ref="fixture", + resources={name: code / name for name in RESOURCE_ENV}, + )) + env = {**os.environ, "UV_OFFLINE": "1", "HERMES_HOME": str(tmp_path / "private")} + command = [sys.executable, "-S", "-B", str(SCRIPT), "--prepared", str(prepared)] + result = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + manifest = json.loads((out / "manifest.json").read_text()) + assert manifest["runtime"]["commands"] == {"probe": "bin/probe"} + assert not (code / "hermes_cli/tui_dist").exists() + assert not (code / "hermes_cli/web_dist").exists() + probe = subprocess.run([str(out / "bin/probe")], cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30) + assert probe.returncode == 0, probe.stderr + assert probe.stdout.strip() == "native fixture" + assert not (tmp_path / "private").exists() + (site / "changed.py").write_text("changed = True") + result = subprocess.run(command, cwd=tmp_path, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode != 0 + assert "prepare" in result.stderr + assert not (out / "manifest.json").exists() diff --git a/tests/scripts/test_pm_runtime_bundle.py b/tests/scripts/test_pm_runtime_bundle.py index 708cf1a8d3..bfd56c3720 100644 --- a/tests/scripts/test_pm_runtime_bundle.py +++ b/tests/scripts/test_pm_runtime_bundle.py @@ -139,10 +139,9 @@ def test_native_stage_builds_pm_before_application_environment(tmp_path, monkeyp shutil.copy2(Path(__file__).resolve().parents[2] / "pm/lock.json", lock) monkeypatch.setattr(payload, "snapshot", lambda *args: None) monkeypatch.setattr(native, "_bundle_package_names", lambda: []) - monkeypatch.setattr(native, "_install_names", lambda names: 0) + monkeypatch.setattr("pm.prepare_tools", lambda *args, **kwargs: tmp_path / "prepared-tools") + monkeypatch.setattr("pm.stage_tools", lambda *args, **kwargs: tmp_path / "tools") monkeypatch.setattr(native, "Facts", Facts) - monkeypatch.setattr(native, "_facts", Facts) - monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(entry=lambda name: tmp_path / "tools" / name)) monkeypatch.setattr(native, "get_package", lambda name: SimpleNamespace(binary=lambda path, target: path / "python")) cache_dir = tmp_path / "cache" diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.md index 8fe7a2bf52..a80f53615d 100644 --- a/website/docs/reference/package-management.md +++ b/website/docs/reference/package-management.md @@ -66,6 +66,25 @@ Native desktop bundles stage the supported tool set and all target-compatible Python extras before packaging. `--extra all` and `--all-extras` are not synonyms. Platform markers still exclude dependencies that cannot run on a target. +The complete desktop builder composes PM with the Node/native packaging providers. +From a clean checkout at a release tag, `python scripts/bundles/desktop.py --tag vX.Y.Z` +prepares dependencies and builds the installer. Add `--prepare-only` to stop after +preparation; consume its job-local result with +`python scripts/bundles/desktop.py --prepared .build/desktop-job/prepared.json`. +`--work` and `--cache` select separate build-owned roots. Preparation, not the +caller, creates the work directory. A full commit SHA can replace the tag through +`--commit`; the checkout must match. + +Preparation uses isolated PM state, pinned tools, fresh path-bound Python +environments, the complete JS workspace union, native bindings and packaging +utilities. Reusable dependency caches are not live installations or portable +virtual environments. The prepared result binds the source, target and paths; +missing or changed inputs fail consumption rather than trigger a download. +Reprepare after a move or input change. Signing and notarization can still use +the network. See the +[desktop build guide](https://github.com/NousResearch/hermes-agent/blob/main/apps/desktop/BUILDING.md) +for native compiler requirements and release verification limits. + A packaged application's base payload is immutable. Hermes runs its backend from that payload, rather than copying a source checkout on first launch. The bundle builder checks its files and writes the launch paths into the desktop