diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index b4f214371a..4af7fb7f11 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -39,19 +39,21 @@ name: Desktop Bundled Release # green, so a failed leg can never publish a partial channel. Stable candidate # jobs only stage tag-scoped objects; stable-release.yml gates feed promotion # on package acceptance and artifact publication. No GitHub job artifacts -# carry release files. Non-publishing builds leave output on the runner only. +# carry release files. Non-publishing tag builds leave output on the runner. +# Commit builds stage under releases/commit// and write a run summary. +# They never publish a release, updater feed, Store submission, or APT channel. # # Payload staging is `hermes pm bundle` on the native runner. There is # no cross-target staging. The darwin legs sign (CSC_LINK) and notarize # (afterSign notarize.mjs) when the release-signing environment carries the -# Apple credentials, and FAIL rather than publish unsigned — forks without -# the credentials can only build (upload_release=false), never publish. +# Apple credentials. Downloadable commit artifacts require the same signing +# checks as release artifacts. Unsigned tag builds cannot publish. # # scripts/bundles/desktop.py is the one driver. Local and CI run # the same command. This workflow adds caching and upload only. # -# Triggers: workflow_dispatch with an explicit tag. A tag push does not -# start this workflow (a bot-pushed canary tag would never fire). +# Manual dispatch selects a tag or a pushed full commit. Commit dispatches +# use the default-branch workflow and require maintainer permission. # # R2 secrets (repo-level or the release-signing environment): the R2 # account id + an R2 API token (S3-compatible) with read/write on the @@ -95,8 +97,14 @@ on: inputs: tag: description: 'Release tag to bundle (vX.Y.Z or vX.Y.0-canary.YYYYMMDDHHMMSS). Must exist on the repo.' - required: true + required: false type: string + default: '' + build_commit: + description: 'Commit-only build: EXACT full 40-hex SHA already pushed to this repo (workflow_dispatch from the default branch only). Mutually exclusive with tag.' + required: false + type: string + default: '' termux_only: description: 'Build and publish only the Termux package' required: false @@ -120,43 +128,71 @@ permissions: id-token: write concurrency: - group: desktop-bundled-release-${{ inputs.tag }}-${{ inputs.release-phase || 'canary' }} + group: desktop-bundled-release-${{ inputs.tag || inputs.build_commit || 'canary' }}-${{ inputs.release-phase || 'canary' }} cancel-in-progress: false jobs: validate: - name: Validate the tag + name: Validate the build source runs-on: ubuntu-24.04 timeout-minutes: 5 + permissions: + contents: read outputs: - # The tag's commit, resolved ONCE here and exported as a full SHA. - # Every privileged job checks out THIS — never the tag ref, which a - # force-push can move between the validate and build jobs. + # Resolve the build source once. Every privileged job uses this SHA, + # not a mutable tag or the dispatch branch. # The tag's channel (stable | canary). publish-darwin-updater scopes # its concurrency group on this so two dispatches for the SAME # channel can never race their feed writes (a stable and a canary # publish in parallel by design — different feed files). channel: ${{ steps.admission.outputs.channel }} sha: ${{ steps.admission.outputs.sha }} + payload-version: ${{ steps.admission.outputs.payload-version }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.tag }} + ref: ${{ inputs.build_commit != '' && github.sha || inputs.tag }} # Full history: the admission gate below runs merge-base against # origin/main, which needs the shared ancestry, not a depth-1 tip. fetch-depth: 0 fetch-tags: true - # but we only need one file :3 filter: tree:0 - sparse-checkout: pyproject.toml + sparse-checkout: | + pyproject.toml + scripts/releases sparse-checkout-cone-mode: false - name: Validate tag shape, pyproject lockstep, and ancestry on origin/main id: admission env: TAG: ${{ inputs.tag }} + BUILD_COMMIT: ${{ inputs.build_commit }} RELEASE_PHASE: ${{ inputs.release-phase }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + UPLOAD_RELEASE: ${{ inputs.upload_release }} + TERMUX_UPGRADE_FROM_TAG: ${{ inputs.termux_upgrade_from_tag }} + GH_TOKEN: ${{ github.token }} run: | + if [ -n "$BUILD_COMMIT" ]; then + # Reject mixed inputs before loading code from the checkout. + if [ -n "$TAG" ] || [ -n "$RELEASE_PHASE" ] || [ -n "$TERMUX_UPGRADE_FROM_TAG" ] || [ "$UPLOAD_RELEASE" != false ]; then + echo '::error::commit builds cannot select release publication or upgrade inputs' + exit 1 + fi + if ! [[ "$BUILD_COMMIT" =~ ^[a-f0-9]{40}$ ]]; then + echo '::error::commit builds require an exact full SHA' + exit 1 + fi + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [ -z "$DEFAULT_BRANCH" ] || [ "$GITHUB_REF" != "refs/heads/$DEFAULT_BRANCH" ] || [ "$GITHUB_WORKFLOW_REF" != "$GITHUB_REPOSITORY/.github/workflows/desktop-bundled-release.yml@refs/heads/$DEFAULT_BRANCH" ]; then + echo '::error::commit builds require the default-branch dispatch workflow' + exit 1 + fi + python -m scripts.releases.commit_build admit + exit 0 + fi + if [ -z "$TAG" ]; then + echo "::error::either tag or build_commit is required"; exit 1 + fi case "$RELEASE_PHASE" in candidate|publish|promote) [[ "$TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { echo "::error::not a release tag: $TAG"; exit 1; } @@ -204,8 +240,7 @@ jobs: echo "sha=$SHA" >> "$GITHUB_OUTPUT" # ── Windows builders (REAL) ─────────────────────────────────────────────── - # The only active builder legs. Every win32 downstream job (updater feed + - # Store submission) is gated on THIS job, never on mac/linux. + # Windows assembly and publication depend on these native Windows legs. build-win32: name: bundled ${{ matrix.target.label }} if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') @@ -224,6 +259,8 @@ jobs: env: HERMES_DESKTOP_VARIANT: bundled HERMES_PAYLOAD_TAG: ${{ inputs.tag }} + HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} + HERMES_PAYLOAD_VERSION: ${{ needs.validate.outputs.payload-version }} ELECTRON_BUILDER_CACHE: ${{ github.workspace }}/.cache/electron-builder ELECTRON_CACHE: ${{ github.workspace }}/.cache/electron electron_config_cache: ${{ github.workspace }}/.cache/electron @@ -439,12 +476,15 @@ jobs: AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} AZURE_TOKEN_CREDENTIALS: prod run: | - python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled - # The Store-submission MSIX is the same bundled payload re-packed - # with the Partner Center packaging identity (publish-win32-store - # bundles these into the universal Store .msixbundle and submits it; - # they also land in the tag archive, never a feed dir). - python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=store + if [ -n "$HERMES_BUILD_COMMIT" ]; then + python scripts/bundles/desktop.py --commit="$HERMES_BUILD_COMMIT" --variant=bundled + python scripts/bundles/desktop.py --commit="$HERMES_BUILD_COMMIT" --variant=store + else + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + # Repack the payload with the Partner Center identity. + # Store packages enter the tag archive, never an updater feed. + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=store + fi - name: Verify native signature cache contracts shell: bash @@ -459,7 +499,7 @@ jobs: --arch="${MATRIX_LABEL##*-}" --root=apps/desktop/release - name: Stage Windows packages to R2 - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: TARGET: ${{ matrix.target.label }} @@ -475,8 +515,15 @@ jobs: --out "apps/desktop/release/metadata-windows-${TARGET##*-}.json" args+=(--include 'metadata-*.json') fi - python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ - --name "$TARGET" --root apps/desktop/release "${args[@]}" + if [ -n "$HERMES_BUILD_COMMIT" ]; then + # Commit-only mode: schema-2 receipts under releases/commit//. + python -m scripts.releases.handoff stage --commit-build "$HERMES_BUILD_COMMIT" \ + --commit "$RELEASE_COMMIT" \ + --name "$TARGET" --root apps/desktop/release "${args[@]}" + else + python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ + --name "$TARGET" --root apps/desktop/release "${args[@]}" + fi # ── macOS builders (REAL) ────────────────────────────────────────────────── # Native per-arch darwin builds via scripts/bundles/desktop.py (the @@ -507,6 +554,8 @@ jobs: env: HERMES_DESKTOP_VARIANT: bundled HERMES_PAYLOAD_TAG: ${{ inputs.tag }} + HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} + HERMES_PAYLOAD_VERSION: ${{ needs.validate.outputs.payload-version }} ELECTRON_BUILDER_CACHE: ${{ github.workspace }}/.cache/electron-builder ELECTRON_CACHE: ${{ github.workspace }}/.cache/electron electron_config_cache: ${{ github.workspace }}/.cache/electron @@ -625,13 +674,10 @@ jobs: restore-keys: | node-pty-prebuilds-${{ matrix.target.label }}- - # Signing/notarization gate. A publishing run MUST have the Apple - # credentials; missing credentials fail the leg here (before any - # build work) instead of producing an unsigned artifact that a later - # job would publish. A non-publishing run (upload_release=false, - # e.g. forks) builds unsigned on purpose. + # Commit artifacts are downloadable too. Require signing for them, + # candidates, and published tags before building any payload. - name: Require signing credentials when publishing - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: CSC_LINK: ${{ secrets.CSC_LINK }} @@ -649,7 +695,7 @@ jobs: [ -z "$CLOUDFLARE_R2_PUBLIC_URL" ] && missing+=(CLOUDFLARE_R2_PUBLIC_URL) [ -z "$CLOUDFLARE_R2_BUCKET" ] && missing+=(CLOUDFLARE_R2_BUCKET) if [ ${#missing[@]} -gt 0 ]; then - echo "::error::upload_release=true but required signing/notarization credentials are not set in the release-signing environment: ${missing[*]} — refusing to produce an unsigned publishable build" + echo "::error::required signing/notarization credentials are missing from release-signing: ${missing[*]}" exit 1 fi @@ -657,7 +703,7 @@ jobs: # notarytool takes a FILE PATH for --key; raw .p8 content in argv # dies with `Invalid option: ***`. The build step must NOT re-declare # APPLE_API_KEY in its env: step env shadows GITHUB_ENV. - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }} @@ -696,7 +742,11 @@ jobs: # every Mach-O) — raise the fd limit and let DEBUG show progress. ulimit -n 16384 2>/dev/null || true echo "file descriptor limit: soft=$(ulimit -Sn) hard=$(ulimit -Hn)" - python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + if [ -n "$HERMES_BUILD_COMMIT" ]; then + python scripts/bundles/desktop.py --commit="$HERMES_BUILD_COMMIT" --variant=bundled + else + python scripts/bundles/desktop.py --tag="$HERMES_PAYLOAD_TAG" --variant=bundled + fi - name: Audit bundle architecture shell: bash @@ -709,7 +759,7 @@ jobs: # The backstop against a silent unsigned publish: assess the packed # app against the real Gatekeeper policy (requires a Developer ID # signature AND a stapled notarization ticket to pass offline). - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash run: | shopt -s nullglob @@ -744,7 +794,7 @@ jobs: mv "$f" "apps/desktop/release/${arch}-${channel}-mac.yml" - name: Stage macOS packages and feed inputs to R2 - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: TARGET: ${{ matrix.target.label }} @@ -759,8 +809,16 @@ jobs: --out "apps/desktop/release/metadata-macos-${TARGET##*-}.json" args+=(--include 'metadata-*.json') fi - python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ - --name "$TARGET" --root apps/desktop/release "${args[@]}" + if [ -n "$HERMES_BUILD_COMMIT" ]; then + # Commit-only mode: binaries only — no feed yml exists without a tag. + python -m scripts.releases.handoff stage --commit-build "$HERMES_BUILD_COMMIT" \ + --commit "$RELEASE_COMMIT" \ + --name "$TARGET" --root apps/desktop/release \ + --include '*.dmg' --include '*.zip' --include '*.blockmap' + else + python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ + --name "$TARGET" --root apps/desktop/release "${args[@]}" + fi # ── Linux builders (DISABLED for now) ───────────────────────────────────── build-linux: @@ -783,12 +841,14 @@ jobs: publish-win32-updater: name: Assemble Windows bundle and optionally publish canary feed needs: [validate, build-win32] - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' runs-on: windows-2025 environment: release-signing timeout-minutes: 45 env: HERMES_PAYLOAD_TAG: ${{ inputs.tag }} + HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} + HERMES_PAYLOAD_VERSION: ${{ needs.validate.outputs.payload-version }} ELECTRON_BUILDER_CACHE: ${{ github.workspace }}/.cache/electron-builder CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} @@ -863,8 +923,14 @@ jobs: env: RELEASE_COMMIT: ${{ needs.validate.outputs.sha }} run: | - python -m scripts.releases.handoff fetch --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ - --name win32-x64 --name win32-arm64 --root apps/desktop/release --include '*.msix' + if [ -n "$HERMES_BUILD_COMMIT" ]; then + python -m scripts.releases.handoff fetch --commit-build "$HERMES_BUILD_COMMIT" \ + --commit "$RELEASE_COMMIT" \ + --name win32-x64 --name win32-arm64 --root apps/desktop/release --include '*.msix' + else + python -m scripts.releases.handoff fetch --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ + --name win32-x64 --name win32-arm64 --root apps/desktop/release --include '*.msix' + fi - name: Azure login (OIDC) uses: azure/login@f5d393ae46f8fde4be8b75f32e3fc50e654ad0ca # v3.0.1 @@ -896,6 +962,12 @@ jobs: run: | # Candidate mode builds the bundles without writing a feed. # Canary mode publishes the App Installer bundle and pointer. + # Commit mode assembles both products without publishing feeds. + if [ -n "$HERMES_BUILD_COMMIT" ]; then + node scripts/stage-msixbundle.mjs --commit "$HERMES_BUILD_COMMIT" --version "$HERMES_PAYLOAD_VERSION" --variant bundled --no-upload + node scripts/bundle-store-msixbundle.mjs --commit "$HERMES_BUILD_COMMIT" --version "$HERMES_PAYLOAD_VERSION" + exit 0 + fi args=() if [ "$RELEASE_PHASE" = candidate ]; then args+=(--candidate); fi node scripts/stage-msixbundle.mjs --tag "$HERMES_PAYLOAD_TAG" --variant bundled "${args[@]}" @@ -903,14 +975,20 @@ jobs: node scripts/bundle-store-msixbundle.mjs --tag "$HERMES_PAYLOAD_TAG" fi - - name: Stage stable universal bundles to R2 - if: inputs.release-phase == 'candidate' + - name: Stage universal bundles to R2 + if: inputs.release-phase == 'candidate' || inputs.build_commit != '' shell: bash env: RELEASE_COMMIT: ${{ needs.validate.outputs.sha }} run: | - python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ - --name windows-universal --root apps/desktop/release --include '*.msixbundle' + if [ -n "$HERMES_BUILD_COMMIT" ]; then + python -m scripts.releases.handoff stage --commit-build "$HERMES_BUILD_COMMIT" \ + --commit "$RELEASE_COMMIT" \ + --name windows-universal --root apps/desktop/release --include '*.msixbundle' + else + python -m scripts.releases.handoff stage --tag "$HERMES_PAYLOAD_TAG" --commit "$RELEASE_COMMIT" \ + --name windows-universal --root apps/desktop/release --include '*.msixbundle' + fi # ── Windows Store submission (REAL — PARALLEL with publish-win32-updater) ─ # Bundles the two Store-*.msix into one universal Store .msixbundle and @@ -930,7 +1008,8 @@ jobs: name: Publish the Windows Store submission (production + canary flight) needs: [validate, build-win32] if: | - inputs.upload_release == true + inputs.build_commit == '' + && inputs.upload_release == true && vars.MS_STORE_PRODUCT_ID != '' && (contains(inputs.tag, '-canary.') == false || vars.MS_STORE_CANARY_FLIGHT_ID != '') runs-on: windows-2025 @@ -1085,7 +1164,7 @@ jobs: publish-darwin-updater: name: Publish the macOS updater feed needs: [validate, build-darwin] - if: inputs.upload_release == true && inputs.termux_only != true + if: inputs.build_commit == '' && inputs.upload_release == true && inputs.termux_only != true runs-on: ubuntu-24.04 environment: release-signing timeout-minutes: 15 @@ -1147,12 +1226,13 @@ jobs: termux-deb: name: Build + publish the termux .deb (aarch64) needs: [validate] - if: inputs.upload_release == true || inputs.release-phase == 'candidate' + if: inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' runs-on: ubuntu-24.04-arm environment: release-signing timeout-minutes: 90 env: HERMES_PAYLOAD_TAG: ${{ inputs.tag }} + HERMES_BUILD_COMMIT: ${{ needs.validate.outputs.sha && inputs.build_commit != '' && needs.validate.outputs.sha || '' }} # termux_build.sh gates on `gh release view ` (refuse to build # before the release exists); gh needs GH_TOKEN or it errors out and # the gate misreads that as "release not found". @@ -1179,6 +1259,7 @@ jobs: - name: Derive the channel from the tag id: channel + if: inputs.build_commit == '' shell: bash # Single source of truth: deb_version.py --channel uses the same # _TAG_RE as the Debian version derivation, so the channel and the @@ -1309,10 +1390,18 @@ jobs: # termux_build.sh lands wheelhouse/ + index.json + SHA256SUMS in the # payload root, where build_deb.sh's --no-index pip install finds it. run: | - bash scripts/termux/termux_build.sh \ - --repo . \ - --tag "$HERMES_PAYLOAD_TAG" \ - --out termux-build/payload + if [ -n "$HERMES_BUILD_COMMIT" ]; then + # The checkout is the admitted commit. No release tag is required. + bash scripts/termux/termux_build.sh \ + --repo . \ + --commit "$HERMES_BUILD_COMMIT" \ + --out termux-build/payload + else + bash scripts/termux/termux_build.sh \ + --repo . \ + --tag "$HERMES_PAYLOAD_TAG" \ + --out termux-build/payload + fi - name: Save the wheelhouse # Only proven wheels enter the cache. Saving before deb assembly @@ -1346,11 +1435,19 @@ jobs: # opt-out flag); the channel is derived from the tag inside the # script via deb_version.py. run: | - bash scripts/termux/build_deb.sh \ - --repo . \ - --tag "$HERMES_PAYLOAD_TAG" \ - --payload termux-build/payload \ - --out termux-build/deb + if [ -n "$HERMES_BUILD_COMMIT" ]; then + bash scripts/termux/build_deb.sh \ + --repo . \ + --commit "$HERMES_BUILD_COMMIT" \ + --payload termux-build/payload \ + --out termux-build/deb + else + bash scripts/termux/build_deb.sh \ + --repo . \ + --tag "$HERMES_PAYLOAD_TAG" \ + --payload termux-build/payload \ + --out termux-build/deb + fi - name: Retrieve the previous published Termux package from R2 if: inputs.termux_upgrade_from_tag != '' @@ -1375,7 +1472,15 @@ jobs: fetch(tag, commit, ['termux'], Path('termux-build/previous'), ['deb/*.deb']) PY + - name: Stage the commit-build deb to R2 + if: inputs.build_commit != '' + shell: bash + run: | + python -m scripts.releases.handoff stage --commit-build "$HERMES_BUILD_COMMIT" \ + --name termux --root termux-build --include 'deb/*.deb' + - name: Write the APT signing key + if: inputs.build_commit == '' shell: bash env: TERMUX_APT_GPG_KEY: ${{ secrets.TERMUX_APT_GPG_KEY }} @@ -1387,6 +1492,7 @@ jobs: printf '%s\n' "$TERMUX_APT_GPG_KEY" > "$RUNNER_TEMP/termux-apt-gpg.asc" - name: Stage the APT repo and publish to R2 + if: inputs.build_commit == '' shell: bash env: CHANNEL: ${{ steps.channel.outputs.channel }} @@ -1474,10 +1580,13 @@ jobs: builds-pending: name: Mark the builds table as in progress - if: inputs.upload_release == true && inputs.termux_only != true + needs: validate + if: inputs.build_commit == '' && inputs.upload_release == true && inputs.termux_only != true runs-on: ubuntu-24.04 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} - name: Render the placeholder env: GH_TOKEN: ${{ github.token }} @@ -1495,7 +1604,7 @@ jobs: builds-table: name: Render the release builds table needs: [validate, build-win32, build-darwin, build-linux, publish-win32-updater, publish-darwin-updater, termux-deb] - if: inputs.upload_release == true && inputs.release-phase == '' + if: inputs.build_commit == '' && inputs.upload_release == true && inputs.release-phase == '' runs-on: ubuntu-24.04 environment: release-signing steps: @@ -1521,6 +1630,47 @@ jobs: python3 scripts/render-builds-table.py \ --tag "$HERMES_PAYLOAD_TAG" --repo "$GITHUB_REPOSITORY" + commit-builds-summary: + name: Commit build summary (every binary, built or not) + needs: [validate, build-win32, build-darwin, build-linux, publish-win32-updater, termux-deb] + if: | + always() && inputs.build_commit != '' + && needs.validate.result == 'success' && needs.validate.outputs.sha != '' + runs-on: ubuntu-24.04 + environment: release-signing + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Runs from the admitted commit so the renderer matches the built bytes. + ref: ${{ needs.validate.outputs.sha }} + - name: Render the full expected-binary matrix + env: + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + RELEASE_COMMIT: ${{ needs.validate.outputs.sha }} + RELEASE_NEEDS: ${{ toJSON(needs) }} + # Render failed admitted builds too. Only receipt-listed objects + # receive download links. Rendering does not alter job results. + run: | + set -e + failed=$(python3 - <<'PY' + import json, os + needs = json.loads(os.environ["RELEASE_NEEDS"]) + failed = [name for name, info in needs.items() + if info.get("result") not in ("success", "skipped")] + print(",".join(sorted(failed))) + PY + ) + python3 scripts/render-builds-table.py \ + --summary-commit "$RELEASE_COMMIT" \ + --summary-out "$GITHUB_STEP_SUMMARY" \ + --summary-failed-legs "$failed" + candidate-manifest: name: Stage verified stable candidate artifacts needs: [validate, build-win32, build-darwin, publish-win32-updater, termux-deb] @@ -1673,6 +1823,7 @@ jobs: needs: [build-win32, build-darwin, build-linux, builds-table, publish-win32-updater, publish-darwin-updater] if: | always() + && inputs.build_commit == '' && inputs.upload_release == true && contains(inputs.tag, '-canary.') && needs.build-win32.result == 'success' diff --git a/apps/desktop/BUILDING.md b/apps/desktop/BUILDING.md index 3d2b3b249f..34a965f1c3 100644 --- a/apps/desktop/BUILDING.md +++ b/apps/desktop/BUILDING.md @@ -102,6 +102,49 @@ The Electron build bakes these paths into its stamp. Desktop startup does not inspect, create, or repair a PM payload. Non-bundled builds carry no placeholder payload. See [shared bundle builds](../../docs/shared-bundle-builds.md) for Termux reuse. +## Commit-only builds + +To preview a build for a pushed revision, run: + +```sh +python scripts/release.py --build-commit REV --remote origin +``` + +The command fetches the remote and resolves `REV` to a full commit SHA. +It prints the dispatch command without changing local branches, tags, or releases. +Add `--publish` to dispatch that build. This flag does not publish a release +in commit-build mode. + +The workflow must exist on the repository's default branch. Admission requires +a default-branch `workflow_dispatch` and repository write, maintain, or admin +permission for both the original actor and the actor who reruns it. +It rejects mixed tag, release-phase, channel-publication, and upgrade inputs. + +Builder jobs check out the admitted SHA. Their artifacts and completion receipts +go to `releases/commit/FULL_SHA/`, separate from tag archives and update channels. +The run summary lists Windows packages and both universal bundles, macOS DMG/ZIP +files, and the Termux package. Linux release legs remain disabled and are listed +as not built. Only receipt-listed artifacts that exist in storage get download +links. Missing receipts show the failed or incomplete leg. + +Commit builds require the signing credentials used by their release legs. +Store bundle envelopes remain unsigned for Partner Center, but these builds +never submit them. No GitHub release, updater feed, or APT channel is changed. +An identical upload retry can succeed. Different bytes at an existing commit +object key fail rather than replace that object. + +For a local native build, check out the exact SHA and run: + +```sh +python scripts/bundles/desktop.py --commit=FULL_SHA --variant=bundled +``` + +The builder uses that commit's project version. Sideload MSIX versions append +`.0`. Store package versions use the commit timestamp with the existing UTC +calendar policy, even when the app version starts with zero. +Commit-built stamps disable automatic release-channel updates. Local command and transport tests do not +replace signed-package installation and update acceptance on each native host. + ## Windows signing and App Installer The signing jobs provide `AZURE_SIGN_ENDPOINT`, `AZURE_SIGN_ACCOUNT`, diff --git a/apps/desktop/scripts/commit-bundles.windows.test.mjs b/apps/desktop/scripts/commit-bundles.windows.test.mjs new file mode 100644 index 0000000000..cd4a94e874 --- /dev/null +++ b/apps/desktop/scripts/commit-bundles.windows.test.mjs @@ -0,0 +1,148 @@ +import assert from 'node:assert/strict' +import { execFileSync, spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +import { test } from 'vitest' + +import { ensureWindowsBundleTools } from './windows-bundle-tools.mjs' + +const repo = path.resolve(import.meta.dirname, '../../..') +const sha256 = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex') + +function run(command, args, cwd, env) { + return execFileSync(command, args, { cwd, env, encoding: 'utf8', timeout: 60_000, stdio: ['ignore', 'pipe', 'pipe'] }) +} + +function fixture(kit) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'commit-msix-')) + const desktop = path.join(root, 'apps/desktop') + fs.mkdirSync(path.join(desktop, 'scripts'), { recursive: true }) + fs.mkdirSync(path.join(root, 'scripts')) + for (const file of ['stage-msixbundle.mjs', 'bundle-store-msixbundle.mjs', 'msix-shared.mjs', 'release-content-types.json']) { + fs.copyFileSync(path.join(repo, 'scripts', file), path.join(root, 'scripts', file)) + } + fs.copyFileSync(path.join(repo, 'apps/desktop/scripts/windows-bundle-tools.mjs'), path.join(desktop, 'scripts/windows-bundle-tools.mjs')) + fs.copyFileSync(path.join(repo, 'apps/desktop/product-identity.cjs'), path.join(desktop, 'product-identity.cjs')) + fs.writeFileSync(path.join(desktop, 'package.json'), JSON.stringify({ name: 'fixture', version: '0.21.1' })) + fs.writeFileSync(path.join(desktop, 'electron-builder.config.cjs'), `module.exports=${JSON.stringify({ directories: { buildResources: kit }, toolsets: { winCodeSign: { url: 'file://' + kit } } })}\n`) + fs.symlinkSync(path.join(repo, 'node_modules'), path.join(root, 'node_modules'), 'junction') + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => + !/^(AZURE_|CLOUDFLARE_|HERMES_|GITHUB_|GH_|NODE_OPTIONS$)/i.test(key))) + Object.assign(env, { HERMES_PAYLOAD_TAG: '', CI: '', GIT_CONFIG_GLOBAL: path.join(root, 'git-config'), + GIT_CONFIG_NOSYSTEM: '1', GIT_AUTHOR_NAME: 'Fixture', GIT_AUTHOR_EMAIL: 'fixture@example.invalid', + GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + GIT_AUTHOR_DATE: '2026-09-10T10:20:30Z', GIT_COMMITTER_DATE: '2026-09-10T10:20:30Z' }) + for (const args of [['init', '-q'], ['add', 'scripts', 'apps'], ['-c', 'commit.gpgsign=false', 'commit', '-qm', 'fixture']]) run('git', args, root, env) + const commit = run('git', ['rev-parse', 'HEAD'], root, env).trim() + env.HERMES_BUILD_COMMIT = commit + env.HERMES_PAYLOAD_VERSION = '0.21.1' + const release = path.join(desktop, 'release') + fs.mkdirSync(release) + return { root, desktop, env, commit, release } +} + +function identity(root, env, variant) { + return JSON.parse(run(process.execPath, ['--input-type=module', '-e', + "import{appIdentity}from'./scripts/msix-shared.mjs';console.log(JSON.stringify(appIdentity(process.cwd()+'/apps/desktop','')))"], + root, { ...env, HERMES_DESKTOP_VARIANT: variant })) +} + +function makePackage(makeappx, root, release, metadata, variant, arch, env) { + const content = path.join(root, `${variant}-${arch}`) + fs.mkdirSync(content) + fs.mkdirSync(path.join(content, 'assets')) + for (const name of ['StoreLogo.png', 'Square150x150Logo.png', 'Square44x44Logo.png']) { + fs.copyFileSync(path.join(repo, 'apps/desktop/assets/appx', name), path.join(content, 'assets', name)) + } + const name = metadata.identity.store ? metadata.identity.storeMsix.identityName : metadata.identity.msixAppIdWithOrg + const publisher = metadata.identity.store ? metadata.identity.storeMsix.publisher : 'CN=Fixture' + fs.writeFileSync(path.join(content, 'index.html'), 'Assembly fixtureNot an installed Hermes application.') + fs.writeFileSync(path.join(content, 'AppxManifest.xml'), ` + + + Assembly fixtureFixtureassets\\StoreLogo.png + + + +\n`) + const file = path.join(release, `${variant === 'store' ? 'Store-' : ''}${metadata.name}-${metadata.fileVersion}-win-${arch}.msix`) + run(makeappx, ['pack', '/o', '/d', content, '/p', file], root, env) + assert.ok(fs.statSync(file).size > 0) + return file +} + +test.runIf(process.platform === 'win32')('commit assembly preserves per-arch packages and produces two isolated SDK bundles', { timeout: 180_000 }, async () => { + const tools = await ensureWindowsBundleTools() + const { root, env, commit, release } = fixture(path.dirname(path.dirname(tools.makeappx))) + try { + const metadata = Object.fromEntries(['bundled', 'store'].map(variant => [variant, identity(root, env, variant)])) + const inputs = {} + for (const variant of ['bundled', 'store']) { + for (const arch of ['x64', 'arm64']) { + const file = makePackage(tools.makeappx, root, release, metadata[variant], variant, arch, env) + inputs[file] = sha256(file) + } + } + const scripts = { bundled: 'stage-msixbundle.mjs', store: 'bundle-store-msixbundle.mjs' } + for (const variant of ['bundled', 'store']) { + const args = ['--commit', commit, '--version', '0.21.1'] + if (variant === 'bundled') args.push('--variant', variant, '--no-upload') + const outputFile = path.join(root, 'store-output') + if (variant === 'store') args.push('--output-file', outputFile) + const result = spawnSync(process.execPath, [path.join(root, 'scripts', scripts[variant]), ...args], { cwd: root, env, encoding: 'utf8', timeout: 60_000 }) + assert.equal(result.status, 0, result.stdout + result.stderr) + const info = metadata[variant] + const bundle = path.join(release, `${variant === 'store' ? 'Store-' : ''}${info.name}-${info.version}-win.msixbundle`) + assert.ok(fs.statSync(bundle).size > 0) + if (variant === 'store') assert.equal(fs.readFileSync(outputFile, 'utf8').trim(), bundle) + const expanded = path.join(root, `expanded-${variant}`) + run(tools.makeappx, ['unbundle', '/o', '/p', bundle, '/d', expanded], root, env) + const xml = fs.readFileSync(path.join(expanded, 'AppxMetadata/AppxBundleManifest.xml'), 'utf8') + const identityTag = /]*>/.exec(xml)[0] + assert.ok(identityTag.includes(`Version="${info.version}"`), xml) + const bundledNames = [...xml.matchAll(/FileName="([^"]+\.msix)"/g)].map(match => match[1]).sort() + const expected = Object.keys(inputs).filter(file => path.basename(file).startsWith('Store-') === (variant === 'store')).map(file => path.basename(file)).sort() + assert.deepEqual(bundledNames, expected) + for (const name of expected) assert.equal(sha256(path.join(expanded, name)), inputs[path.join(release, name)]) + const digest = sha256(bundle) + const modified = fs.statSync(bundle).mtimeMs + const invalid = [ + [...args, '--candidate'], [...args, '--tag', 'v0.21.1'], [...args, '--unknown'], + ['--commit', commit.slice(0, 8), '--version', '0.21.1', ...(variant === 'bundled' ? ['--no-upload'] : [])], + ['--commit', commit, '--version', '1.65536.0', ...(variant === 'bundled' ? ['--no-upload'] : [])], + ] + if (variant === 'bundled') invalid.push(args.filter(value => value !== '--no-upload')) + for (const badArgs of invalid) { + const rejected = spawnSync(process.execPath, [path.join(root, 'scripts', scripts[variant]), ...badArgs], { cwd: root, env, encoding: 'utf8', timeout: 30_000 }) + assert.notEqual(rejected.status, 0, `${scripts[variant]} accepted ${badArgs.join(' ')}\n${rejected.stdout}${rejected.stderr}`) + assert.equal(sha256(bundle), digest) + assert.equal(fs.statSync(bundle).mtimeMs, modified) + } + const missing = path.join(release, expected[0]) + fs.renameSync(missing, `${missing}.held`) + try { + const refused = spawnSync(process.execPath, [path.join(root, 'scripts', scripts[variant]), ...args], { cwd: root, env, encoding: 'utf8', timeout: 30_000 }) + assert.notEqual(refused.status, 0) + assert.match(refused.stdout + refused.stderr, /need both per-arch/) + assert.equal(sha256(bundle), digest) + assert.equal(fs.statSync(bundle).mtimeMs, modified) + } finally { + fs.renameSync(`${missing}.held`, missing) + } + if (variant === 'store') { + run('git', ['tag', 'v0.21.1'], root, env) + const tagEnv = { ...env, HERMES_BUILD_COMMIT: '', HERMES_PAYLOAD_VERSION: '', HERMES_PAYLOAD_TAG: 'v0.21.1' } + const tagged = spawnSync(process.execPath, [path.join(root, 'scripts', scripts.store), '--tag', 'v0.21.1', '--output-file', outputFile], { cwd: root, env: tagEnv, encoding: 'utf8', timeout: 60_000 }) + assert.equal(tagged.status, 0, tagged.stdout + tagged.stderr) + assert.equal(fs.readFileSync(outputFile, 'utf8').trim(), bundle) + } + } + assert.equal(fs.readdirSync(release).some(name => name.endsWith('.appinstaller')), false) + for (const [file, digest] of Object.entries(inputs)) assert.equal(sha256(file), digest) + } finally { + fs.rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/scripts/store-package-version.test.mjs b/apps/desktop/scripts/store-package-version.test.mjs index 84639ae974..6598ef7847 100644 --- a/apps/desktop/scripts/store-package-version.test.mjs +++ b/apps/desktop/scripts/store-package-version.test.mjs @@ -3,7 +3,7 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' -import { afterEach, expect, test } from 'vitest' +import { afterEach, expect, test, vi } from 'vitest' import { appIdentity, storeManifestTemplate, storePackageVersion, storePackageVersionAt } from '../../../scripts/msix-shared.mjs' import { stageStoreManifest } from './before-build.mjs' import { AppInfo } from '../../../node_modules/app-builder-lib/dist/appInfo.js' @@ -67,3 +67,39 @@ test('Store calendar ordering survives minute, hour, day and year boundaries and expect(() => storePackageVersionAt(NaN)).toThrow() expect(() => storePackageVersion('v0.27.1-canary.20260231000000', '.')).toThrow('Invalid canary') }) + +test('commit builds use the commit time for Store identity without changing the app version', () => { + const { root, app } = fixture() + const commit = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim() + const timestamp = Number(execFileSync('git', ['log', '-1', '--format=%ct', commit], { cwd: root, encoding: 'utf8' }).trim()) + vi.stubEnv('HERMES_PAYLOAD_TAG', '') + vi.stubEnv('HERMES_BUILD_COMMIT', commit) + vi.stubEnv('HERMES_PAYLOAD_VERSION', '0.21.1') + vi.useFakeTimers() + try { + vi.setSystemTime(new Date('2030-01-01T00:00:00Z')) + const identity = appIdentity(app) + const xml = fs.readFileSync(stageStoreManifest(app, ''), 'utf8') + expect(identity.version).toBe(storePackageVersionAt(timestamp)) + expect(/]*Version="([^"]+)"/.exec(xml)[1]).toBe(identity.version) + expect(identity.fileVersion).toBe('0.21.1') + vi.setSystemTime(new Date('2031-01-01T00:00:00Z')) + expect(appIdentity(app).version).toBe(identity.version) + const prior = fs.readFileSync(path.join(app, 'build/store-msix-manifest.xml')) + for (const bad of ['short', 'a'.repeat(40)]) { + vi.stubEnv('HERMES_BUILD_COMMIT', bad) + expect(() => stageStoreManifest(app, '')).toThrow() + expect(fs.readFileSync(path.join(app, 'build/store-msix-manifest.xml'))).toEqual(prior) + } + vi.stubEnv('HERMES_BUILD_COMMIT', commit) + for (const version of ['01.2.3', '1.65536.0', '1.2.3-canary.123', '']) { + vi.stubEnv('HERMES_PAYLOAD_VERSION', version) + expect(() => appIdentity(app)).toThrow() + } + vi.stubEnv('HERMES_PAYLOAD_VERSION', '0.21.1') + expect(() => appIdentity(app, 'v0.21.1')).toThrow() + } finally { + vi.useRealTimers() + vi.unstubAllEnvs() + } +}) diff --git a/scripts/bundle-store-msixbundle.mjs b/scripts/bundle-store-msixbundle.mjs index 78545131b7..9df125bf84 100644 --- a/scripts/bundle-store-msixbundle.mjs +++ b/scripts/bundle-store-msixbundle.mjs @@ -19,24 +19,28 @@ import { execFileSync } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' import { appIdentity } from './msix-shared.mjs' import { ensureWindowsBundleTools } from '../apps/desktop/scripts/windows-bundle-tools.mjs' const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') -// node strips the first '--' (and an immediately-following option) for its -// own use; parse space-separated flag pairs, not --flag=value. -const args = process.argv.slice(2) -const flagValue = (name) => { - for (let i = 0; i < args.length - 1; i += 1) { - if (args[i] === name) return args[i + 1] - } - return undefined +const { values } = parseArgs({ options: { + tag: { type: 'string' }, commit: { type: 'string' }, version: { type: 'string' }, + 'output-file': { type: 'string' }, +} }) +const tag = values.tag || process.env.HERMES_PAYLOAD_TAG +const commitBuild = values.commit +if (commitBuild) { + if (tag) throw new Error('Commit builds cannot select a release tag') + process.env.HERMES_BUILD_COMMIT = commitBuild + process.env.HERMES_PAYLOAD_VERSION = values.version || '' +} else if (values.version !== undefined) { + throw new Error('--version requires --commit') } -const tag = flagValue('--tag') || process.env.HERMES_PAYLOAD_TAG -if (!tag) { - console.error('[bundle-store] --tag= is required') +if (!tag && !commitBuild) { + console.error('[bundle-store] --tag or --commit is required') process.exit(1) } if (process.platform !== 'win32') { @@ -82,6 +86,6 @@ execFileSync(makeappx, ['bundle', '/o', '/bv', version, '/d', staging, '/p', bun fs.rmSync(staging, { recursive: true, force: true }) // Download logs can share stdout. The explicit output file is the machine contract. -const outputFile = flagValue('--output-file') +const outputFile = values['output-file'] if (outputFile) fs.writeFileSync(outputFile, bundle, 'utf8') console.log(bundle) diff --git a/scripts/bundles/desktop.py b/scripts/bundles/desktop.py index 2999dce63d..31868318f8 100644 --- a/scripts/bundles/desktop.py +++ b/scripts/bundles/desktop.py @@ -54,14 +54,24 @@ def npm_command(node: str) -> list[str]: raise FileNotFoundError(f"npm CLI missing beside {npm}") -def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: +def build(repo: Path, tag: str | None, variant: str, builder_args: list[str], + commit_build: str | None = None) -> None: from pm.store import current_target + from scripts.releases.commit_build import require_commit, version_at repo = repo.resolve() - version = release_version(repo, tag) - commit = capture(["git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"], repo) - if capture(["git", "rev-parse", "HEAD"], repo) != commit: - raise ValueError("the build checkout must be at the release tag") + if commit_build: + commit = require_commit(commit_build) + if tag: + raise ValueError("Commit builds cannot also select a tag") + if capture(["git", "rev-parse", "HEAD"], repo) != commit: + raise ValueError("the build checkout must be at the commit being built") + version = version_at(repo, commit) + else: + version = release_version(repo, tag) + commit = capture(["git", "rev-parse", "--verify", f"refs/tags/{tag}^{{commit}}"], repo) + if capture(["git", "rev-parse", "HEAD"], repo) != commit: + raise ValueError("the build checkout must be at the release tag") node = shutil.which("node") if not node or not shutil.which("uv"): raise FileNotFoundError("Node and uv are required") @@ -70,7 +80,16 @@ def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: raise ValueError(f"uv must report its build triple (official uv 0.12+): {banner}") npm = npm_command(node) env = {**os.environ, "CI": "true", "PYTHONUTF8": "1", "GITHUB_SHA": commit, - "HERMES_DESKTOP_VARIANT": variant, "HERMES_PAYLOAD_TAG": tag} + "HERMES_DESKTOP_VARIANT": variant} + if commit_build: + env["HERMES_PAYLOAD_VERSION"] = version + env["HERMES_BUILD_COMMIT"] = commit + env.pop("HERMES_PAYLOAD_TAG", None) + env.pop("GITHUB_REF_NAME", None) + env.pop("GITHUB_HEAD_REF", None) + else: + env.pop("HERMES_BUILD_COMMIT", None) + env["HERMES_PAYLOAD_TAG"] = tag target = current_target() node_arch = capture([node, "-p", "process.arch"], repo) if node_arch != target.split("-")[1]: @@ -91,7 +110,7 @@ def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: else: run([*npm, "run", "build", "--workspace", "ui-tui"], cwd=repo, env=env) run([*npm, "run", "build", "--workspace", "web"], cwd=repo, env=env) - run([sys.executable, "-m", "pm.cli", "bundle", "--out", str(payload), "--ref", tag], cwd=repo, env=env) + run([sys.executable, "-m", "pm.cli", "bundle", "--out", str(payload), "--ref", commit], cwd=repo, env=env) manifest = json.loads((payload / "manifest.json").read_text(encoding="utf-8-sig")) plant_surfaces(payload / manifest["repo"], repo) relativize_links(payload) @@ -100,8 +119,12 @@ def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: # Windows file-version and MSIX build-number policy remains with its packager. version_args = [] if sys.platform == "win32": - script = "const w=require('./apps/desktop/scripts/windows-file-version.mjs');const m=require('./scripts/msix-shared.mjs');console.log(JSON.stringify({file:w.windowsFileVersion(process.argv[1]),build:process.argv[2]!=='store'&&process.argv[1].includes('-canary.')?m.canaryBuildMinutes(process.argv[1],process.cwd()):null}))" - metadata = json.loads(capture([node, "-e", script, tag, variant], repo)) + if commit_build: + # The plain version needs no canary build-number override. + metadata = {"file": None, "build": None} + else: + script = "const w=require('./apps/desktop/scripts/windows-file-version.mjs');const m=require('./scripts/msix-shared.mjs');console.log(JSON.stringify({file:w.windowsFileVersion(process.argv[1]),build:process.argv[2]!=='store'&&process.argv[1].includes('-canary.')?m.canaryBuildMinutes(process.argv[1],process.cwd()):null}))" + metadata = json.loads(capture([node, "-e", script, tag, variant], repo)) env.pop("BUILD_NUMBER", None) if metadata["build"] is not None and variant != "store": env["BUILD_NUMBER"] = str(metadata["build"]) @@ -114,12 +137,19 @@ def build(repo: Path, tag: str, variant: str, builder_args: list[str]) -> None: def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--tag", required=True) + parser.add_argument("--tag", required=False, + help="Release tag (vX.Y.Z / canary). Required unless --commit is given") + parser.add_argument("--commit", dest="commit_build", default=None, + help="Commit-only build: exact full 40-char SHA the checkout is at; " + "version comes from the target pyproject, no tag is referenced") parser.add_argument("--variant", choices=["bundled", "store", "light"], default="bundled") parser.add_argument("--repo", type=Path, default=ROOT) parser.add_argument("builder_args", nargs=argparse.REMAINDER) args = parser.parse_args() - build(args.repo, args.tag, args.variant, [v for v in args.builder_args if v != "--"]) + if bool(args.tag) == bool(args.commit_build): + parser.error("exactly one of --tag or --commit is required") + build(args.repo, args.tag, args.variant, [v for v in args.builder_args if v != "--"], + commit_build=args.commit_build) if __name__ == "__main__": diff --git a/scripts/msix-shared.mjs b/scripts/msix-shared.mjs index 1999870629..faf101db9d 100644 --- a/scripts/msix-shared.mjs +++ b/scripts/msix-shared.mjs @@ -239,6 +239,21 @@ export function appIdentity(desktopDir, tag = process.env.HERMES_PAYLOAD_TAG || const identity = require(path.join(desktopDir, 'product-identity.cjs')) const pkg = JSON.parse(fs.readFileSync(path.join(desktopDir, 'package.json'), 'utf8')) const repoRoot = path.resolve(desktopDir, '..', '..') + // Commit artifacts retain app semver but do not advance an update channel. + if (process.env.HERMES_BUILD_COMMIT) { + if (tag) throw new Error('Commit-only builds must not set HERMES_PAYLOAD_TAG') + const commit = process.env.HERMES_BUILD_COMMIT + if (!/^[a-f0-9]{40}$/.test(commit)) throw new Error('Commit builds require an exact full SHA') + const version = String(process.env.HERMES_PAYLOAD_VERSION || '') + if (!/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(version) + || version.split('.').some(part => Number(part) > 65535)) { + throw new Error('Commit builds require HERMES_PAYLOAD_VERSION=X.Y.Z with 16-bit fields') + } + const packageVersion = identity.store + ? storePackageVersionAt(gitTagCommitTime(repoRoot, commit)) + : `${version}.0` + return { identity, version: packageVersion, fileVersion: version, name: identity.appNamePascal } + } if (identity.store) { return { identity, version: storePackageVersion(String(tag), repoRoot), fileVersion: String(tag).slice(1), name: identity.appNamePascal } diff --git a/scripts/stage-msixbundle.mjs b/scripts/stage-msixbundle.mjs index 61a8fe97b5..62628140f9 100644 --- a/scripts/stage-msixbundle.mjs +++ b/scripts/stage-msixbundle.mjs @@ -27,6 +27,7 @@ import { execFileSync } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' +import { parseArgs } from 'node:util' import { appIdentity, buildAppInstaller } from './msix-shared.mjs' import { ensureWindowsBundleTools } from '../apps/desktop/scripts/windows-bundle-tools.mjs' @@ -34,17 +35,23 @@ import { ensureWindowsBundleTools } from '../apps/desktop/scripts/windows-bundle const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') -// node strips the first '--' (and an immediately-following option) for its -// own use; parse space-separated flag pairs, not --flag=value. -const args = process.argv.slice(2) -const flagValue = (name) => { - for (let i = 0; i < args.length - 1; i += 1) { - if (args[i] === name) return args[i + 1] - } - return undefined +const { values } = parseArgs({ options: { + tag: { type: 'string' }, commit: { type: 'string' }, version: { type: 'string' }, + variant: { type: 'string' }, 'no-upload': { type: 'boolean' }, candidate: { type: 'boolean' }, +} }) +const tag = values.tag +const commitBuild = values.commit || '' +const commitVersion = values.version || '' +const noUpload = values['no-upload'] === true +const candidate = values.candidate === true +const variant = values.variant || process.env.HERMES_DESKTOP_VARIANT || 'bundled' + +if (commitBuild && (tag || process.env.HERMES_PAYLOAD_TAG || candidate)) { + throw new Error('Commit builds cannot select a release tag or candidate mode') +} +if (!commitBuild && (values.version !== undefined || noUpload)) { + throw new Error('--version and --no-upload require --commit') } -const tag = flagValue('--tag') -const variant = flagValue('--variant') || process.env.HERMES_DESKTOP_VARIANT || 'bundled' // product-identity.cjs keys the app name off HERMES_DESKTOP_VARIANT — the // artifact filenames (HermesBundled-*-win-x64.msix) carry the bundled @@ -52,7 +59,26 @@ const variant = flagValue('--variant') || process.env.HERMES_DESKTOP_VARIANT || // fails. Set it before anything requires the identity. process.env.HERMES_DESKTOP_VARIANT = variant -if (!tag) { +if (commitBuild) { + if (!/^[a-f0-9]{40}$/.test(commitBuild)) { + console.error('[stage-msixbundle] --commit must be an exact full 40-hex SHA') + process.exit(1) + } + if (!/^\d+\.\d+\.\d+$/.test(commitVersion)) { + console.error('[stage-msixbundle] --version=X.Y.Z is required with --commit (the target pyproject version)') + process.exit(1) + } + if (!noUpload) { + console.error('[stage-msixbundle] commit mode must pass --no-upload (commit builds never write a feed)') + process.exit(1) + } + if (tag) { + console.error('[stage-msixbundle] --commit and --tag are mutually exclusive') + process.exit(1) + } + process.env.HERMES_BUILD_COMMIT = commitBuild + process.env.HERMES_PAYLOAD_VERSION = commitVersion +} else if (!tag) { console.error('[stage-msixbundle] --tag= is required') process.exit(1) } @@ -65,9 +91,8 @@ if (process.platform !== 'win32') { process.exit(1) } -const candidate = args.includes('--candidate') const canary = /-canary\./.test(tag) -if (!canary && !candidate) throw new Error('Stable bundles must use the staged stable-release workflow') +if (!commitBuild && !canary && !candidate) throw new Error('Stable bundles must use the staged stable-release workflow') const channel = canary ? 'canary' : 'stable' const channelDir = `releases/win32/${variant === 'light' ? 'light/' : ''}${channel}` @@ -160,6 +185,13 @@ if (candidate) { process.exit(0) } +// Commit-only mode stops here: the workflow hands the bundle to R2 through +// scripts.releases.handoff (schema-2 receipt) — never a feed dir. +if (commitBuild) { + console.log(`[stage-msixbundle] commit bundle ready (no upload): ${bundle}`) + process.exit(0) +} + // ── 2. .appinstaller + uploads ───────────────────────────────────────────── const baseUrl = String(process.env.CLOUDFLARE_R2_PUBLIC_URL || '').replace(/\/+$/, '') if (!baseUrl) { diff --git a/scripts/termux/build_deb.sh b/scripts/termux/build_deb.sh index e947b44519..86c036edb9 100644 --- a/scripts/termux/build_deb.sh +++ b/scripts/termux/build_deb.sh @@ -27,19 +27,15 @@ HERE="$(cd "$(dirname "$0")" && pwd)" # The container digest is a pm pin (the termux-docker package); read it # from the single lock beside every other third-party artifact pin. REPO_ROOT="$(cd "$HERE/../.." && pwd)" -DIGEST="$(cd "$REPO_ROOT" && python3 -c 'import sys; sys.path.insert(0, "."); from pm.lock import termux_docker_digest; print(termux_docker_digest())')" -[ -n "$DIGEST" ] || { printf 'termux-docker digest missing from pm/lock.json\n' >&2; exit 1; } -# The derived builder image (toolchain pre-baked) when CI provides it; -# the bare pinned base otherwise. Its tag IS this lock digest (short -# form), so a lock bump rolls the builder image with the base. -IMAGE="${TERMUX_BUILDER_IMAGE:-termux/termux-docker@$DIGEST}" + REPO="" TAG="" +COMMIT_MODE="" PAYLOAD="" OUT="" -usage() { printf 'usage: build_deb.sh --repo --tag --payload --out \n' >&2; exit 2; } +usage() { printf 'usage: build_deb.sh --repo (--tag | --commit ) --payload --out \n' >&2; exit 2; } log() { printf '\n==> %s\n' "$*"; } fail() { printf 'build_deb: FAILED: %s\n' "$*" >&2; exit 1; } @@ -47,26 +43,52 @@ while [ "$#" -gt 0 ]; do case "$1" in --repo) REPO="${2:?}"; shift 2 ;; --tag) TAG="${2:?}"; shift 2 ;; + --commit) COMMIT_MODE="${2:?}"; shift 2 ;; --payload) PAYLOAD="${2:?}"; shift 2 ;; --out) OUT="${2:?}"; shift 2 ;; *) usage ;; esac done -[ -n "$REPO" ] && [ -n "$TAG" ] && [ -n "$PAYLOAD" ] && [ -n "$OUT" ] || usage +[ -n "$REPO" ] && [ -n "$PAYLOAD" ] && [ -n "$OUT" ] || usage +{ [ -n "$TAG" ] || [ -n "$COMMIT_MODE" ]; } && { [ -z "$TAG" ] || [ -z "$COMMIT_MODE" ]; } || usage -for tool in python3 docker dpkg-deb jq; do +for tool in python3 git docker dpkg-deb jq; do command -v "$tool" >/dev/null || fail "missing tool: $tool" done +DIGEST="$(cd "$REPO_ROOT" && python3 -c 'from pm.lock import termux_docker_digest; print(termux_docker_digest())')" +[ -n "$DIGEST" ] || fail "termux-docker digest missing from pm/lock.json" +IMAGE="${TERMUX_BUILDER_IMAGE:-termux/termux-docker@$DIGEST}" + REPO_ABS="$(cd "$REPO" && pwd)" PAYLOAD_ABS="$(cd "$PAYLOAD" && pwd)" -OUT_ABS="$(mkdir -p "$OUT" && cd "$OUT" && pwd)" -# [0] Provenance: the tag must be real in the checkout; the payload must be -# the built tree of that checkout, not some other directory. The commit is -# captured ONCE here and reused for the install stamp below. -COMMIT="$(git -C "$REPO_ABS" rev-parse --verify "refs/tags/$TAG^{commit}")" \ - || fail "tag $TAG not found in $REPO_ABS" +# Resolve source identity before writing output or changing payload files. +# Commit mode requires the checkout HEAD and staged version to agree. +if [ -n "$COMMIT_MODE" ]; then + [[ "$COMMIT_MODE" =~ ^[a-f0-9]{40}$ ]] || fail "--commit requires an exact full 40-character SHA" + COMMIT="$(git -C "$REPO_ABS" rev-parse HEAD)" || fail "not a git checkout: $REPO_ABS" + [ "$COMMIT" = "$COMMIT_MODE" ] || fail "checkout HEAD $(echo "$COMMIT" | cut -c1-12) is not the requested commit" + PY_VERSION="$(python3 - "$REPO_ROOT" "$REPO_ABS" "$COMMIT" "$PAYLOAD_ABS/app/pyproject.toml" <<'PY' +import sys, tomllib +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from scripts.releases.commit_build import version_at +version = version_at(Path(sys.argv[2]), sys.argv[3]) +staged = tomllib.loads(Path(sys.argv[4]).read_text(encoding="utf-8"))["project"]["version"] +if staged != version: + raise ValueError("payload version does not match the admitted commit") +print(version) +PY + )" || fail "commit version validation failed" + DEB_VERSION="${PY_VERSION}+commit${COMMIT_MODE:0:12}" + export HERMES_PAYLOAD_TAG="" + export HERMES_BUILD_COMMIT="$COMMIT_MODE" +else + unset HERMES_BUILD_COMMIT + COMMIT="$(git -C "$REPO_ABS" rev-parse --verify "refs/tags/$TAG^{commit}")" \ + || fail "tag $TAG not found in $REPO_ABS" +fi for d in python node uv npm ffmpeg ripgrep runtime-libs app wheelhouse; do [ -d "$PAYLOAD_ABS/$d" ] || fail "payload missing $d/ -- run termux_build.sh + build_cpython.sh + build_node.sh first" done @@ -77,10 +99,14 @@ NODEBIN_REL="data/data/com.termux/files/usr/bin/node" PKG="hermes-agent" -# [1] Version derivation: pure function in deb_version.py, tested separately. -log "Deriving Debian version from tag $TAG" -DEB_VERSION="$(python3 "$HERE/deb_version.py" "$TAG")" || fail "version derivation failed for tag $TAG" +# [1] Version derivation: tag mode uses the pure function in deb_version.py +# (tested separately). Commit mode derives it from pyproject above. +if [ -z "$COMMIT_MODE" ]; then + log "Deriving Debian version from tag $TAG" + DEB_VERSION="$(python3 "$HERE/deb_version.py" "$TAG")" || fail "version derivation failed for tag $TAG" +fi log "Package version: $DEB_VERSION" +OUT_ABS="$(mkdir -p "$OUT" && cd "$OUT" && pwd)" # [2] Assemble the venv offline, INSIDE the pinned container: the staged # interpreter is bionic/arm64 and cannot run on this host. Completeness is @@ -159,10 +185,8 @@ printf 'apt\n' > "$PAYLOAD_ABS/app/.install_method" log "Writing trampolines" python3 "$HERE/launchers.py" --payload "$PAYLOAD_ABS" -# [4] Install stamp: provenance for the steward contract (distribution -# apt-termux -> update/uninstall refuse with pkg remediation). Written by the -# canonical writer (same one docker/nix/desktop use) so the schema stays -# identical across packagers; the tag rides in via HERMES_PAYLOAD_TAG. +# The shared stamp writer records the apt-termux update owner. +# Commit mode exports HERMES_BUILD_COMMIT and leaves the tag empty. log "Writing app/install-stamp.json" HERMES_PAYLOAD_TAG="$TAG" \ HERMES_DESKTOP_VARIANT=bundled \ diff --git a/scripts/termux/termux_build.sh b/scripts/termux/termux_build.sh index ab4ed69a92..15181801ed 100755 --- a/scripts/termux/termux_build.sh +++ b/scripts/termux/termux_build.sh @@ -146,17 +146,19 @@ fi # ===================== HOST HALF (glibc runner) ========================== REPO="" TAG="" +COMMIT_MODE="" OUT="" while [ "$#" -gt 0 ]; do case "$1" in --repo) REPO="${2:?}"; shift 2 ;; --tag) TAG="${2:?}"; shift 2 ;; + --commit) COMMIT_MODE="${2:?}"; shift 2 ;; --out) OUT="${2:?}"; shift 2 ;; - *) printf 'usage: termux_build.sh --repo --tag --out \n' >&2; exit 2 ;; + *) printf 'usage: termux_build.sh --repo (--tag | --commit ) --out \n' >&2; exit 2 ;; esac done -[ -n "$REPO" ] && [ -n "$TAG" ] && [ -n "$OUT" ] || { - printf 'usage: termux_build.sh --repo --tag --out \n' >&2; exit 2; } +[ -n "$REPO" ] && [ -n "$OUT" ] && { [ -n "$TAG" ] || [ -n "$COMMIT_MODE" ]; } && { [ -z "$TAG" ] || [ -z "$COMMIT_MODE" ]; } || { + printf 'usage: termux_build.sh --repo (--tag | --commit ) --out \n' >&2; exit 2; } for tool in uv git curl docker python3; do command -v "$tool" >/dev/null 2>&1 \ @@ -169,13 +171,21 @@ case "$ARCH" in *) fail "refusing to build on non-aarch64 host (uname -m: $ARCH)" ;; esac -# [b] FIRST: refuse mutable releases. -log "Verifying release tag $TAG exists on origin" -git -C "$REPO" ls-remote --exit-code --tags origin "$TAG" >/dev/null \ - || fail "tag $TAG not found on origin; refusing to build a mutable release" -if command -v gh >/dev/null 2>&1; then - gh release view "$TAG" --repo "$(git -C "$REPO" remote get-url origin | sed -e 's#.*github.com[:/]##' -e 's#\.git$##')" >/dev/null 2>&1 \ - || fail "release $TAG not found; refusing to build before the release exists" +# Check source identity before writing build output. +if [ -n "$COMMIT_MODE" ]; then + [[ "$COMMIT_MODE" =~ ^[a-f0-9]{40}$ ]] || fail "--commit requires an exact full 40-character SHA" + [ "$(git -C "$REPO" rev-parse HEAD)" = "$COMMIT_MODE" ] || fail "checkout does not match --commit" + log "Commit-only build of $COMMIT_MODE -- skipping the tag/release gates" + REF="$COMMIT_MODE" +else + log "Verifying release tag $TAG exists on origin" + git -C "$REPO" ls-remote --exit-code --tags origin "$TAG" >/dev/null \ + || fail "tag $TAG not found on origin; refusing to build a mutable release" + if command -v gh >/dev/null 2>&1; then + gh release view "$TAG" --repo "$(git -C "$REPO" remote get-url origin | sed -e 's#.*github.com[:/]##' -e 's#\.git$##')" >/dev/null 2>&1 \ + || fail "release $TAG not found; refusing to build before the release exists" + fi + REF="$TAG" fi REPO_ABS="$(cd "$REPO" && pwd)" @@ -188,8 +198,8 @@ rm -rf "$WORK/tree" mkdir -p "$WORK/tree" "$WHEELHOUSE" # [c] Stage the tag as a gitless tree. -log "Archiving $TAG into $WORK/tree" -python3 - "$REPO_ABS" "$TAG" "$WORK/tree" <<'PY' +log "Archiving $REF into $WORK/tree" +python3 - "$REPO_ABS" "$REF" "$WORK/tree" <<'PY' import sys from pathlib import Path sys.path.insert(0, sys.argv[1]) @@ -220,7 +230,7 @@ rm -f "$OUT_ABS/index.json" "$OUT_ABS/SHA256SUMS" "$WORK/resolved.txt" "$WORK/bu log "Resolving dependency graph from the tag's uv.lock" ( cd "$WORK/tree" && uv export --frozen --no-emit-project --extra acp \ --no-hashes --no-annotate --no-header -o "$WORK/req.txt" ) \ - || fail "uv export failed (frozen lock at $TAG)" + || fail "uv export failed (frozen lock at $REF)" # Host parsing needs packaging too. Use the release lock, not runner packages. PACKAGING_SPEC="$(python3 - "$WORK/tree/uv.lock" <<'PY' import sys, tomllib @@ -383,7 +393,9 @@ cp -a "$WORK/tree" "$OUT_ABS/app" # [h] Only a successful native build and both gates can publish cache proof. log "Emitting index.json and SHA256SUMS" -python3 "$HERE/wheelhouse_cache.py" write "${CACHE_ARGS[@]}" --tag "$TAG" \ +provenance=(--tag "$TAG") +if [ -n "$COMMIT_MODE" ]; then provenance=(--commit "$COMMIT_MODE"); fi +python3 "$HERE/wheelhouse_cache.py" write "${CACHE_ARGS[@]}" "${provenance[@]}" \ || fail "manifest emission failed" log "Wheelhouse complete: $WHEELHOUSE" diff --git a/scripts/termux/wheelhouse_cache.py b/scripts/termux/wheelhouse_cache.py index 8027175009..8f58e2e629 100644 --- a/scripts/termux/wheelhouse_cache.py +++ b/scripts/termux/wheelhouse_cache.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse import hashlib import json +import re from pathlib import Path @@ -82,13 +83,17 @@ def main() -> int: parser.add_argument("--builder", required=True) parser.add_argument("--platform-tag", required=True) parser.add_argument("--python-abi", required=True) - parser.add_argument("--tag", default="") + provenance = parser.add_mutually_exclusive_group() + provenance.add_argument("--tag", default="") + provenance.add_argument("--commit") args = parser.parse_args() + if args.commit is not None and not re.fullmatch(r"[a-f0-9]{40}", args.commit): + parser.error("--commit requires an exact full SHA") identity = build_identity(args.repo, args.builder, args.platform_tag, args.python_abi) if args.action == "check": return 0 if is_usable(args.payload, identity) else 1 write_manifest( - args.payload, identity, tag=args.tag, + args.payload, identity, **({"commit": args.commit} if args.commit else {"tag": args.tag}), platformTag=args.platform_tag, pythonAbi=args.python_abi, ) return 0 diff --git a/tests/ci/test_commit_build_staging.py b/tests/ci/test_commit_build_staging.py new file mode 100644 index 0000000000..0e8b4fcab2 --- /dev/null +++ b/tests/ci/test_commit_build_staging.py @@ -0,0 +1,140 @@ +"""Execute commit staging and summary steps against a disposable object store.""" +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +from urllib.request import urlopen + +from tests.ci.test_desktop_release_tag_admission import _BASH, _child_env, _workflow +from tests.scripts.test_release_r2 import r2_server # noqa: F401 + + +ROOT = Path(__file__).resolve().parents[2] + + +def step_script(job, name): + return next(step['run'] for step in _workflow()['jobs'][job]['steps'] if step.get('name') == name) + + +def shell_step(tmp_path, r2_server, job, name, env): + helper = tmp_path / 'bin' + helper.mkdir(exist_ok=True) + driver = helper / 'python-driver.py' + driver.write_text( + 'import runpy,sys\n' + f'sys.path.insert(0, {str(ROOT)!r})\n' + 'from scripts.releases import r2\n' + f'r2.s3_endpoint=lambda _: "http://127.0.0.1:{r2_server.server_port}"\n' + 'args=sys.argv[1:]\n' + 'assert args[:2] == ["-m", "scripts.releases.handoff"] or ' + 'args[:1] == ["scripts/render-builds-table.py"] or args == ["-"], args\n' + 'if args[:1] == ["-m"]:\n' + ' sys.argv=args[1:]\n' + ' runpy.run_module(args[1],run_name="__main__")\n' + 'elif args == ["-"]:\n' + ' exec(compile(sys.stdin.read(), "workflow-inline", "exec"))\n' + 'else:\n' + ' sys.argv=args\n' + f' runpy.run_path({str(ROOT / "scripts/render-builds-table.py")!r},run_name="__main__")\n', + encoding='utf-8', + ) + for name_ in ['python', 'python3']: + command = helper / name_ + command.write_text(f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(driver))} "$@"\n', + encoding='utf-8', newline='\n') + command.chmod(0o755) + script = tmp_path / 'step.sh' + script.write_text(step_script(job, name), encoding='utf-8', newline='\n') + environment = _child_env(**env) + environment['PATH'] = str(helper) + os.pathsep + environment['PATH'] + return subprocess.run([_BASH, '-e', '-o', 'pipefail', str(script)], cwd=tmp_path, + env=environment, capture_output=True, text=True, encoding='utf-8', timeout=60) + + +def test_commit_staging_and_summary_bind_every_produced_file_without_channels(tmp_path, r2_server): + sha = 'a' * 40 + base = f'http://127.0.0.1:{r2_server.server_port}/hermes-releases' + env = dict(HERMES_BUILD_COMMIT=sha, HERMES_PAYLOAD_TAG='', RELEASE_COMMIT=sha, + RELEASE_PHASE='', GITHUB_SHA='b' * 40, CLOUDFLARE_R2_PUBLIC_URL=base, + CLOUDFLARE_R2_ACCOUNT_ID='loopback', CLOUDFLARE_R2_ACCESS_KEY_ID='test-inert', + CLOUDFLARE_R2_SECRET_ACCESS_KEY='test-inert', CLOUDFLARE_R2_BUCKET='hermes-releases') + release = tmp_path / 'apps/desktop/release' + release.mkdir(parents=True) + producers = [ + ('build-win32', 'Stage Windows packages to R2', 'win32-x64', [ + 'HermesBundled-0.33.0-win-x64.msix', 'Store-HermesBundled-0.33.0-win-x64.msix']), + ('build-win32', 'Stage Windows packages to R2', 'win32-arm64', [ + 'HermesBundled-0.33.0-win-arm64.msix', 'Store-HermesBundled-0.33.0-win-arm64.msix']), + ('build-darwin', 'Stage macOS packages and feed inputs to R2', 'darwin-arm64', [ + 'HermesBundled-0.33.0-mac-arm64.dmg', 'HermesBundled-0.33.0-mac-arm64.zip', + 'HermesBundled-0.33.0-mac-arm64.zip.blockmap']), + ('build-darwin', 'Stage macOS packages and feed inputs to R2', 'darwin-x64', [ + 'HermesBundled-0.33.0-mac-x64.dmg', 'HermesBundled-0.33.0-mac-x64.zip', + 'HermesBundled-0.33.0-mac-x64.zip.blockmap']), + ('publish-win32-updater', 'Stage universal bundles to R2', 'windows-universal', [ + 'HermesBundled-0.33.0.0-win.msixbundle', 'Store-HermesBundled-0.33.0.0-win.msixbundle']), + ] + artifact_keys = set() + for job, name, target, names in producers: + for file in release.iterdir(): + file.unlink() + for filename in names: + (release / filename).write_bytes(f'transport fixture: {filename}'.encode()) + result = shell_step(tmp_path, r2_server, job, name, {**env, 'TARGET': target}) + assert result.returncode == 0, result.stdout + result.stderr + receipt_key = f'releases/commit/{sha}/handoff-{target}.json' + receipt = json.loads(r2_server.store[receipt_key][0]) + assert receipt['schema'] == 2 and receipt['commit'] == sha and 'tag' not in receipt + assert {row['path'] for row in receipt['files']} == set(names) + artifact_keys.update(f'releases/commit/{sha}/{filename}' for filename in names) + puts = [path for method, path, _ in r2_server.requests if method == 'PUT'] + assert puts[-1].endswith(receipt_key) + + termux_name = 'Stage the commit-build deb to R2' + before = dict(r2_server.store) + missing = shell_step(tmp_path, r2_server, 'termux-deb', termux_name, env) + assert missing.returncode != 0 + assert r2_server.store == before + deb = tmp_path / 'termux-build/deb/hermes-agent_0.33.0~commit.aaaaaaaaaaaa_aarch64.deb' + deb.parent.mkdir(parents=True) + deb.write_bytes(b'transport fixture, not a native Debian package') + staged = shell_step(tmp_path, r2_server, 'termux-deb', termux_name, env) + assert staged.returncode == 0, staged.stdout + staged.stderr + artifact_keys.add(f'releases/commit/{sha}/deb/{deb.name}') + receipt = json.loads(r2_server.store[f'releases/commit/{sha}/handoff-termux.json'][0]) + assert {row['path'] for row in receipt['files']} == {f'deb/{deb.name}'} + + summary = tmp_path / 'summary.md' + summary_env = {**env, 'GITHUB_STEP_SUMMARY': str(summary), 'RELEASE_NEEDS': json.dumps({ + 'validate': {'result': 'success'}, 'build-win32': {'result': 'success'}, + 'build-darwin': {'result': 'success'}, 'publish-win32-updater': {'result': 'success'}, + 'termux-deb': {'result': 'success'}, 'build-linux': {'result': 'success'}, + })} + result = shell_step(tmp_path, r2_server, 'commit-builds-summary', + 'Render the full expected-binary matrix', summary_env) + assert result.returncode == 0, result.stdout + result.stderr + text = summary.read_text(encoding='utf-8') + links = re.findall(r'\]\((http[^)]+)\)', text) + # Blockmaps are receipt inputs; every other staged product has a download row. + expected = {f'{base}/{key}' for key in artifact_keys if not key.endswith('.blockmap')} + assert set(links) == expected + assert len(links) == len(expected) + for url in links: + with urlopen(url, timeout=5) as response: + key = url.removeprefix(base + '/') + assert response.read() == r2_server.store[key][0] + assert all(key.startswith(f'releases/commit/{sha}/') for key in r2_server.store) + assert not any(method == 'DELETE' for method, _, _ in r2_server.requests) + + # The actual summary command remains useful after an admitted matrix failure. + r2_server.store.pop(f'releases/commit/{sha}/handoff-darwin-x64.json') + summary.unlink() + summary_env['RELEASE_NEEDS'] = json.dumps({'validate': {'result': 'success'}, + 'build-darwin': {'result': 'failure'}}) + incomplete = shell_step(tmp_path, r2_server, 'commit-builds-summary', + 'Render the full expected-binary matrix', summary_env) + assert incomplete.returncode == 0, incomplete.stdout + incomplete.stderr + assert 'failed: build-darwin' in summary.read_text(encoding='utf-8') diff --git a/tests/ci/test_desktop_release_commit_admission.py b/tests/ci/test_desktop_release_commit_admission.py new file mode 100644 index 0000000000..a16d1ca5b2 --- /dev/null +++ b/tests/ci/test_desktop_release_commit_admission.py @@ -0,0 +1,90 @@ +"""Commit-build admission rejects mixed inputs before repository code runs.""" +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys + +from tests.ci.test_desktop_release_tag_admission import _admission_script, _child_env, _git, _seed_repo, _BASH + + +def environment(clone, commit): + return _child_env( + TAG='', BUILD_COMMIT=commit, RELEASE_PHASE='', UPLOAD_RELEASE='false', + TERMUX_UPGRADE_FROM_TAG='', DEFAULT_BRANCH='main', GITHUB_REF='refs/heads/main', + GITHUB_EVENT_NAME='workflow_dispatch', GITHUB_REPOSITORY='fixture/repo', + GITHUB_WORKFLOW_REF='fixture/repo/.github/workflows/desktop-bundled-release.yml@refs/heads/main', + GITHUB_ACTOR='maintainer', GITHUB_TRIGGERING_ACTOR='maintainer', + GITHUB_OUTPUT=str(clone / 'outputs'), GH_TOKEN='fixture-token', + GIT_ALLOW_PROTOCOL='file', PYTHONUTF8='1', + ) + + +def run_admission(clone, env): + helper = clone / 'test-bin' + helper.mkdir(exist_ok=True) + (helper / 'python').write_text( + f'#!/usr/bin/env bash\nexec {shlex.quote(sys.executable)} "$@"\n', encoding='utf-8') + (helper / 'python').chmod(0o755) + script = clone / 'admission.sh' + script.write_text(_admission_script(), encoding='utf-8', newline='\n') + return subprocess.run([_BASH, '-e', '-o', 'pipefail', str(script)], cwd=clone, + env={**env, 'PATH': str(helper) + os.pathsep + env['PATH']}, + capture_output=True, text=True, encoding='utf-8', timeout=60) + + +def test_mixed_dispatch_is_refused_before_loading_repository_code(tmp_path): + _, clone = _seed_repo(tmp_path) + package = clone / 'scripts/releases' + package.mkdir(parents=True) + witness = clone / 'module-ran' + (package / 'commit_build.py').write_text( + f'from pathlib import Path\nPath({str(witness)!r}).write_text("executed")\n', encoding='utf-8') + (clone / 'outputs').write_text('prior=value\n', encoding='utf-8') + env = environment(clone, _git('rev-parse', 'HEAD', cwd=clone)) + for extra in ({'TAG': 'v1.2.3'}, {'RELEASE_PHASE': 'candidate'}, {'UPLOAD_RELEASE': 'true'}, + {'TERMUX_UPGRADE_FROM_TAG': 'v1.2.2'}, {'BUILD_COMMIT': 'abc123'}): + result = run_admission(clone, {**env, **extra}) + assert result.returncode != 0, result.stdout + result.stderr + assert not witness.exists(), 'rejected input executed the checkout admission module' + assert (clone / 'outputs').read_text(encoding='utf-8') == 'prior=value\n' + + +def test_trusted_dispatch_admits_a_pushed_feature_without_switching_checkout(tmp_path): + origin, clone = _seed_repo(tmp_path) + main = _git('rev-parse', 'HEAD', cwd=clone) + _git('checkout', '-qb', 'feature', cwd=origin) + (origin / 'pyproject.toml').write_text('[project]\nname="fixture"\nversion="3.2.1"\n', encoding='utf-8') + _git('add', 'pyproject.toml', cwd=origin) + _git('commit', '-qm', 'feature', cwd=origin) + commit = _git('rev-parse', 'HEAD', cwd=origin) + _git('fetch', 'origin', cwd=clone) + source = Path(__file__).resolve().parents[2] / 'scripts/releases' + shutil.copytree(source, clone / 'scripts/releases', ignore=shutil.ignore_patterns('__pycache__')) + helper = clone / 'test-bin' + helper.mkdir() + permission_log = clone / 'permission.jsonl' + code = ( + 'import json,sys\nfrom pathlib import Path\n' + 'assert sys.argv[1] == "api" and sys.argv[2].endswith("/permission")\n' + f'with Path({str(permission_log)!r}).open("a",encoding="utf-8") as stream: ' + 'stream.write(json.dumps(sys.argv[1:])+"\\n")\nprint("write")\n' + ) + if os.name == 'nt': + from scripts.bundles.mint_launchers import mint_one + mint_one(str(helper), sys.executable, code, {'name': 'gh', 'module': 'fixture', 'func': 'main'}) + else: + module = helper / 'gh.py' + module.write_text(code, encoding='utf-8') + (helper / 'gh').write_text(f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(module))} "$@"\n', encoding='utf-8') + (helper / 'gh').chmod(0o755) + result = run_admission(clone, environment(clone, commit)) + assert result.returncode == 0, result.stdout + result.stderr + outputs = dict(line.split('=', 1) for line in (clone / 'outputs').read_text(encoding='utf-8').splitlines()) + assert outputs == {'sha': commit, 'channel': 'commit', 'payload-version': '3.2.1'} + assert _git('rev-parse', 'HEAD', cwd=clone) == main + requests = [json.loads(line) for line in permission_log.read_text(encoding='utf-8').splitlines()] + assert requests and all(row[1] == 'repos/fixture/repo/collaborators/maintainer/permission' for row in requests) + assert not _git('tag', '--list', cwd=clone) diff --git a/tests/scripts/test_commit_bundle_entrypoints.py b/tests/scripts/test_commit_bundle_entrypoints.py new file mode 100644 index 0000000000..5b95cea9b5 --- /dev/null +++ b/tests/scripts/test_commit_bundle_entrypoints.py @@ -0,0 +1,98 @@ +"""Tagless packaging enters the real builder with an exact source identity.""" +from __future__ import annotations + +import os +import json +from pathlib import Path +import shutil +import subprocess +import sys + +import pytest + +from scripts.bundles import desktop + +from tests.ci.test_desktop_release_tag_admission import _BASH, _child_env, _git, _seed_repo + + +@pytest.mark.parametrize("variant", ["bundled", "store", "light"]) +def test_desktop_build_reaches_the_managed_payload_with_commit_ref(tmp_path, monkeypatch, variant): + _, repo = _seed_repo(tmp_path) + sha = _git("rev-parse", "HEAD", cwd=repo) + (repo / "pyproject.toml").write_text('[project]\nname="x"\nversion="9.9.9"\n', encoding='utf-8') + monkeypatch.setenv("HERMES_PAYLOAD_TAG", "v9.9.9") + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + monkeypatch.setenv("BUILD_NUMBER", "123") + monkeypatch.setattr(desktop.shutil, "which", lambda name: name) + monkeypatch.setattr(desktop, "npm_command", lambda node: [node, "npm-cli.js"]) + from pm.store import current_target + target_arch = current_target().split("-")[1] + def capture(argv, cwd): + if argv[0] == "git": + return _git(*argv[1:], cwd=cwd) + if argv == ["uv", "--version"]: + return "uv 0.12.0 aarch64-pc-windows-msvc" + if "process.arch" in argv: + return target_arch + return "26.7.0" + monkeypatch.setattr(desktop, "capture", capture) + (repo / "package-lock.json").write_text("{}", encoding="utf-8") + (repo / "node_modules").mkdir() + (repo / 'apps/desktop').mkdir(parents=True) + (repo / 'ui-tui/dist').mkdir(parents=True) + (repo / 'ui-tui/dist/entry.js').write_text('source fixture', encoding='utf-8') + (repo / 'hermes_cli/web_dist').mkdir(parents=True) + (repo / 'hermes_cli/web_dist/index.html').write_text('source fixture', encoding='utf-8') + calls = [] + def run(argv, *, cwd, env): + assert cwd.resolve().is_relative_to(repo.resolve()) + calls.append((argv, cwd, env.copy())) + assert env.get("HERMES_PAYLOAD_TAG", "") == "" + assert env["HERMES_BUILD_COMMIT"] == sha + assert env["GITHUB_SHA"] == sha + assert env["HERMES_PAYLOAD_VERSION"] == "0.1.2" + if "pm.cli" in argv: + assert argv[-2:] == ["--ref", sha] + payload = repo / 'apps/desktop/build/agent-payload' + (payload / 'hermes-agent').mkdir(parents=True) + (payload / 'manifest.json').write_text(json.dumps({'repo': 'hermes-agent', 'target': current_target()}), encoding='utf-8') + launcher_calls = [] + monkeypatch.setattr(desktop, 'stage_launchers', lambda payload, manifest: launcher_calls.append((payload, manifest))) + monkeypatch.setattr(desktop, "run", run) + desktop.build(repo, None, variant, ['--publish=never'], commit_build=sha) + assert any('pm.cli' in argv for argv, _, _ in calls) == (variant != 'light') + assert bool(launcher_calls) == (variant != 'light') + argv, cwd, env = calls[-1] + assert argv[:5] == ['node', 'npm-cli.js', 'run', 'builder', '--'] + assert '-c.extraMetadata.version=0.1.2' in argv + assert argv[-1] == '--publish=never' + assert cwd == repo / 'apps/desktop' + assert env['HERMES_DESKTOP_VARIANT'] == variant + if os.name == 'nt': + assert 'BUILD_NUMBER' not in env + before = len(calls) + for tag, commit in [('v0.1.2', sha), (None, 'b' * 40), (None, 'short')]: + with pytest.raises(ValueError): + desktop.build(repo, tag, variant, [], commit_build=commit) + assert len(calls) == before + + + +def test_termux_commit_args_reach_prerequisite_checks_without_mutation(tmp_path): + repo = Path(__file__).resolve().parents[2] + out = tmp_path / "must-not-be-written" + helper = tmp_path / "bin" + helper.mkdir() + # Empty prerequisite commands are not build substitutes: stop at the first + # actual prerequisite check, before any payload or output is created. + env = _child_env(PATH=str(helper), HERMES_PAYLOAD_TAG="") + scripts = [repo / "scripts/termux/termux_build.sh", repo / "scripts/termux/build_deb.sh"] + for script in scripts: + args = ["--repo", str(repo), "--commit", "a" * 40, "--out", str(out)] + if script.name == "build_deb.sh": + args += ["--payload", str(tmp_path / "absent")] + result = subprocess.run([_BASH, str(script), *args], env=env, cwd=tmp_path, + capture_output=True, text=True, timeout=30) + assert result.returncode == 1, result.stdout + result.stderr + assert "usage:" not in result.stderr + assert not out.exists() diff --git a/tests/scripts/test_termux_commit_identity.py b/tests/scripts/test_termux_commit_identity.py new file mode 100644 index 0000000000..eecb749680 --- /dev/null +++ b/tests/scripts/test_termux_commit_identity.py @@ -0,0 +1,61 @@ +"""Termux packaging refuses identity mistakes before modifying its payload.""" +import os +from pathlib import Path +import shlex +import subprocess +import sys + +from tests.ci.test_desktop_release_tag_admission import _BASH, _GIT, _child_env, _git, _seed_repo + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_deb_identity_refusal_leaves_payload_and_output_untouched(tmp_path): + _, repo = _seed_repo(tmp_path) + commit = _git('rev-parse', 'HEAD', cwd=repo) + payload = tmp_path / 'payload' + (payload / 'app').mkdir(parents=True) + (payload / 'app/pyproject.toml').write_bytes((repo / 'pyproject.toml').read_bytes()) + (payload / 'venv').mkdir() + witness = payload / 'venv/keep' + witness.write_bytes(b'prior dependency tree') + out = tmp_path / 'output' + helper = tmp_path / 'bin' + helper.mkdir() + boundary = tmp_path / 'native-boundary' + for name in ('docker', 'dpkg-deb', 'jq'): + tool = helper / name + tool.write_text( + f'#!/bin/sh\nprintf %s {shlex.quote(name)} > {shlex.quote(str(boundary))}\nexit 87\n', + encoding='utf-8', newline='\n') + tool.chmod(0o755) + python = helper / 'python3' + python.write_text(f'#!/bin/sh\nexec {shlex.quote(sys.executable)} "$@"\n', + encoding='utf-8', newline='\n') + python.chmod(0o755) + env = _child_env(HERMES_PAYLOAD_TAG='', HERMES_BUILD_COMMIT='', GIT_ALLOW_PROTOCOL='file', + PYTHONUTF8='1') + for name in ('MSYS_NO_PATHCONV', 'MSYS2_ARG_CONV_EXCL', 'GIT_DIR', 'GIT_WORK_TREE', 'PYTHONPATH', 'PYTHONHOME'): + env.pop(name, None) + env['PATH'] = os.pathsep.join([str(helper), str(Path(_GIT).parent), env.get('PATH', '')]) + args = ['--repo', str(repo), '--payload', str(payload), '--out', str(out)] + + def invoke(identity): + result = subprocess.run([_BASH, str(ROOT / 'scripts/termux/build_deb.sh'), *args, *identity], + cwd=tmp_path, env=env, capture_output=True, text=True, + encoding='utf-8', timeout=30) + assert result.returncode != 0, result.stdout + result.stderr + assert not boundary.exists(), 'identity refusal reached the native builder' + assert witness.read_bytes() == b'prior dependency tree' + assert not out.exists(), result.stdout + result.stderr + return result.stdout + result.stderr + + assert 'not the requested commit' in invoke(['--commit', 'a' * 40]) + assert 'exact full' in invoke(['--commit', 'short']) + assert 'usage:' in invoke(['--tag', 'v0.1.2', '--commit', commit]) + # The version in the archived payload must agree with the admitted commit. + (payload / 'app/pyproject.toml').write_text('[project]\nversion="9.9.9"\n', encoding='utf-8') + assert 'version' in invoke(['--commit', commit]).lower() + (payload / 'app/pyproject.toml').write_bytes((repo / 'pyproject.toml').read_bytes()) + assert 'payload missing python/' in invoke(['--commit', commit]) diff --git a/tests/test_termux_wheelhouse_cache.py b/tests/test_termux_wheelhouse_cache.py index 547d1c8887..8a5d174b97 100644 --- a/tests/test_termux_wheelhouse_cache.py +++ b/tests/test_termux_wheelhouse_cache.py @@ -3,6 +3,8 @@ from __future__ import annotations import json from pathlib import Path +import subprocess +import sys import pytest @@ -56,3 +58,29 @@ def test_cache_rejects_incomplete_or_inconsistent_manifests(tmp_path, damage): else: manifest_path.write_text("{broken", encoding="utf-8") assert not wheelhouse_cache.is_usable(payload, identity) + + +def test_cache_cli_keeps_commit_and_tag_provenance_distinct(tmp_path): + payload, _ = cache_tree(tmp_path) + repo = Path(__file__).resolve().parents[1] + args = [sys.executable, str(repo / 'scripts/termux/wheelhouse_cache.py'), 'write', + '--payload', str(payload), '--repo', str(repo), '--builder', 'fixture-image', + '--platform-tag', 'android_24_arm64_v8a', '--python-abi', 'cp314'] + commit = 'a' * 40 + result = subprocess.run([*args, '--commit', commit], cwd=tmp_path, + capture_output=True, text=True, encoding='utf-8', timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + manifest_path = payload / 'index.json' + manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + assert manifest['commit'] == commit and 'tag' not in manifest + assert wheelhouse_cache.is_usable(payload, manifest['inputs']) + before = manifest_path.read_bytes() + for flags in (['--commit', 'short'], ['--commit', commit, '--tag', 'v1.2.3']): + refused = subprocess.run([*args, *flags], cwd=tmp_path, capture_output=True, timeout=30) + assert refused.returncode != 0 + assert manifest_path.read_bytes() == before + tagged = subprocess.run([*args, '--tag', 'v1.2.3'], cwd=tmp_path, + capture_output=True, text=True, encoding='utf-8', timeout=30) + assert tagged.returncode == 0, tagged.stdout + tagged.stderr + manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + assert manifest['tag'] == 'v1.2.3' and 'commit' not in manifest