diff --git a/.github/workflows/bootstrap-installer.yml b/.github/workflows/bootstrap-installer.yml index 4723308784..a99565f501 100644 --- a/.github/workflows/bootstrap-installer.yml +++ b/.github/workflows/bootstrap-installer.yml @@ -18,8 +18,8 @@ permissions: contents: read concurrency: - group: bootstrap-installer-${{ github.ref }} - cancel-in-progress: true + group: bootstrap-installer-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: posix: diff --git a/.github/workflows/canary-release.yml b/.github/workflows/canary-release.yml index c98d1d3dea..d73b11dca2 100644 --- a/.github/workflows/canary-release.yml +++ b/.github/workflows/canary-release.yml @@ -76,12 +76,14 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: python3 scripts/release.py --prune-canaries --publish --remote origin - - name: Install locked feed tooling - run: npm ci --workspaces=false --ignore-scripts --no-audit --no-fund + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: '0.12.3' + enable-cache: false - name: Prune R2 canary objects env: CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} - run: node scripts/r2-release.mjs prune-canaries --keep-days 14 + run: uv run --no-project --with PyYAML==6.0.3 python -m scripts.releases.r2 prune-canaries --keep-days 14 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3ed8ae55f5..65fa880a9a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,6 +16,19 @@ name: CI on: workflow_dispatch: + inputs: + release: + description: 'Stable-release candidate run: force every applicability lane and make the aggregate gate strict (skipped required lanes fail).' + required: false + type: boolean + default: false + workflow_call: + inputs: + release: + description: 'Stable-release candidate run: force every applicability lane and make the aggregate gate strict (skipped required lanes fail).' + required: false + type: boolean + default: false pull_request: push: branches: [main] @@ -26,35 +39,47 @@ permissions: actions: read # needed by osv-scanner (SARIF upload) security-events: write # needed by osv-scanner (SARIF upload) +# cancel-in-progress only ever applies to PR events. A push, a dispatch, and +# above all a stable-release workflow_call run are never cancelled by a later +# commit or a rerun of the same branch — the caller's own concurrency uses +# github.run_id, so even a parent rerun cannot kill this child mid-flight. +# Release (workflow_call with inputs.release) never cancels and never gets +# cancelled: its group is github.run_id. PRs collapse per-PR; pushes use the +# ref as before. concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ci-${{ inputs.release == true && github.run_id || (github.event_name == 'pull_request' && github.event.pull_request.number || github.ref) }} + cancel-in-progress: ${{ inputs.release != true && github.event_name == 'pull_request' }} jobs: # ───────────────────────────────────────────────────────────────────── # detect: run the classifier once. Every downstream job reads its outputs # to decide whether to run. On push/dispatch the classifier fails open # (all lanes true) so post-merge validation is never weakened. + # + # A release run (inputs.release) additionally forces every lane true via + # the `release-forced-*` outputs: a stable candidate must run the FULL + # pipeline, not the lanes its diff would touch — the diff of a release + # tag is not a meaningful applicability signal. # ───────────────────────────────────────────────────────────────────── detect: name: Detect affected areas runs-on: ubuntu-latest timeout-minutes: 1 outputs: - python: ${{ steps.classify.outputs.python }} - python_prod: ${{ steps.classify.outputs.python_prod }} - frontend: ${{ steps.classify.outputs.frontend }} - site: ${{ steps.classify.outputs.site }} - scan: ${{ steps.classify.outputs.scan }} - deps: ${{ steps.classify.outputs.deps }} - uv_lock: ${{ steps.classify.outputs.uv_lock }} - npm_lock: ${{ steps.classify.outputs.npm_lock }} - installer: ${{ steps.classify.outputs.installer }} - bootstrap: ${{ steps.classify.outputs.bootstrap }} - desktop_updater: ${{ steps.classify.outputs.desktop_updater }} - rust: ${{ steps.classify.outputs.rust }} - docker_meta: ${{ steps.classify.outputs.docker_meta }} - mcp_catalog: ${{ steps.classify.outputs.mcp_catalog }} + python: ${{ steps.gate-lanes.outputs.python }} + python_prod: ${{ steps.gate-lanes.outputs.python_prod }} + frontend: ${{ steps.gate-lanes.outputs.frontend }} + site: ${{ steps.gate-lanes.outputs.site }} + scan: ${{ steps.gate-lanes.outputs.scan }} + deps: ${{ steps.gate-lanes.outputs.deps }} + uv_lock: ${{ steps.gate-lanes.outputs.uv_lock }} + npm_lock: ${{ steps.gate-lanes.outputs.npm_lock }} + installer: ${{ steps.gate-lanes.outputs.installer }} + bootstrap: ${{ steps.gate-lanes.outputs.bootstrap }} + desktop_updater: ${{ steps.gate-lanes.outputs.desktop_updater }} + rust: ${{ steps.gate-lanes.outputs.rust }} + docker_meta: ${{ steps.gate-lanes.outputs.docker_meta }} + mcp_catalog: ${{ steps.gate-lanes.outputs.mcp_catalog }} ci_review: ${{ steps.classify.outputs.ci_review }} ci_review_files: ${{ steps.classify.outputs.ci_review_files }} event_name: ${{ github.event_name }} @@ -65,6 +90,25 @@ jobs: uses: ./.github/actions/detect-changes with: github-token: ${{ github.token }} + - name: Force all lanes on release + # Overwrite the classifier outputs with 'true' when inputs.release. + # ci_review stays raw: it only ever feeds the PR review-label gate. + id: gate-lanes + env: + RELEASE: ${{ inputs.release }} + CLASSIFIED: ${{ toJSON(steps.classify.outputs) }} + run: | + python3 - <<'PY' + import json, os + values = json.loads(os.environ['CLASSIFIED']) + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: + for lane, value in values.items(): + if os.environ['RELEASE'] == 'true' and value in ('true', 'false'): + value = 'true' + if '\n' in value or '\r' in value: + continue + output.write(f'{lane}={value}\n') + PY # ───────────────────────────────────────────────────────────────────── # Lane-gated sub-workflows. Each runs in parallel after detect finishes. @@ -240,8 +284,14 @@ jobs: # ───────────────────────────────────────────────────────────────────── # Gate: runs after everything. ``if: always()`` ensures it reports a - # status even when some deps were skipped. Only actual ``failure`` - # results cause it to fail; ``skipped`` is treated as success. + # status even when some deps were skipped. + # + # Non-release (PR/push): failure fails, skipped counts as success. + # Release (inputs.release): strict — every required job must be + # `success`. A skipped required lane fails the gate; only the PR-only + # jobs (history-check, lockfile-diff, supply-chain, review-labels) and + # the deferred Desktop E2E (e2e-desktop) may skip. The OSV scan's + # findings are advisory, but its execution is required. # # Branch protection should require ONLY this check. # @@ -264,6 +314,7 @@ jobs: - history-check - contributor-check - uv-lockfile + - infographic-check - case-collision-check - lazy-deps-guard - lockfile-diff @@ -283,29 +334,21 @@ jobs: outputs: needs-json: ${{ steps.evaluate.outputs.needs-json }} steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} - name: Evaluate job results id: evaluate + # Shared with the stable orchestrator (scripts/ci/required_results.py + # is imported there). NEEDS is the toJSON(needs) context; RELEASE + # switches the gate into strict mode for release runs. env: NEEDS: ${{ toJSON(needs) }} + RELEASE: ${{ inputs.release }} run: | - echo "$NEEDS" | python3 -c " - import json, sys - needs = json.load(sys.stdin) - # Emit compact {job_name: result} for the comment assembler. - compact = {name: info['result'] for name, info in needs.items()} - print(f'needs-json={json.dumps(compact)}') - with open('$GITHUB_OUTPUT', 'a') as f: - f.write(f'needs-json={json.dumps(compact)}\n') - failed = [name for name, info in needs.items() if info['result'] == 'failure'] - for name, info in sorted(needs.items()): - result = info['result'] - icon = '✅' if result in ('success', 'skipped') else '❌' - print(f'{icon} {name}: {result}') - if failed: - print(f'::error::{len(failed)} job(s) failed: {\", \".join(failed)}') - sys.exit(1) - print('All checks passed (or were skipped)') - " + args=() + if [ "$RELEASE" = true ]; then args+=(--release); fi + printf '%s' "$NEEDS" | python3 scripts/ci/required_results.py "${args[@]}" # ───────────────────────────────────────────────────────────────────── # CI timing report: collect per-job/step durations from the GitHub API, diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 3b033ff4e2..716b8c2534 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -24,7 +24,7 @@ name: Desktop Bundled Release # (delete-then-replace, newest always wins). Gated on MS_STORE_PRODUCT_ID # (+ MS_STORE_CANARY_FLIGHT_ID for the canary arm). # publish-darwin-updater → macOS electron-updater feed -# (needs BOTH darwin legs): r2-release.mjs finalize merges the per-arch +# (needs BOTH darwin legs): scripts.releases.r2 finalize merges the per-arch # ymls into releases/darwin//-mac.yml — the feed # pointer is written LAST, and the job's concurrency group serializes # same-channel publications. @@ -53,8 +53,8 @@ name: Desktop Bundled Release # R2 secrets (repo-level or the release-signing environment): the R2 # account id + an R2 API token (S3-compatible) with read/write on the # release bucket; CLOUDFLARE_R2_BUCKET and CLOUDFLARE_R2_PUBLIC_URL are -# non-secret vars. scripts/r2-release.mjs derives the S3 endpoint from -# the account id and needs only node — no npm ci, no extra deps. +# non-secret vars. scripts/releases/r2.py derives the S3 endpoint from +# the account id. Upload/list operations need only Python; feed operations use PyYAML. # # Windows Store submission (publish-win32-store): MSStore CLI via # microsoft/microsoft-store-apppublisher. Credentials live in the @@ -70,6 +70,24 @@ name: Desktop Bundled Release # ingestion (see sign-msix.mjs). on: + workflow_call: + inputs: + tag: + required: true + type: string + release-phase: + required: true + type: string + manifest-sha256: + default: '' + type: string + outputs: + manifest-url: + value: ${{ jobs.candidate-manifest.outputs.manifest-url }} + description: Immutable candidate manifest + manifest-sha256: + value: ${{ jobs.candidate-manifest.outputs.manifest-sha256 }} + description: Digest of the candidate manifest workflow_dispatch: inputs: tag: @@ -99,7 +117,7 @@ permissions: id-token: write concurrency: - group: desktop-bundled-release-${{ inputs.tag }} + group: desktop-bundled-release-${{ inputs.tag }}-${{ inputs.release-phase || 'canary' }} cancel-in-progress: false jobs: @@ -134,7 +152,17 @@ jobs: id: admission env: TAG: ${{ inputs.tag }} + RELEASE_PHASE: ${{ inputs.release-phase }} run: | + case "$RELEASE_PHASE" in + candidate|publish|promote) + [[ "$TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { echo "::error::not a release tag: $TAG"; exit 1; } + [ "$GITHUB_REF" = "refs/tags/$TAG" ] || exit 1 + [ "$(git rev-parse HEAD)" = "$GITHUB_SHA" ] || exit 1 + ;; + '') [[ "$TAG" == *-canary.* ]] || { echo 'Use Stable Release for stable tags' >&2; exit 1; } ;; + *) echo 'Unknown release phase' >&2; exit 1 ;; + esac case "$TAG" in v[0-9]*.[0-9]*.[0-9]*-canary.20[0-9][0-9][0-9][0-9][0-9][0-9]*) echo "canary tag: $TAG" @@ -177,7 +205,7 @@ jobs: # Store submission) is gated on THIS job, never on mac/linux. build-win32: name: bundled ${{ matrix.target.label }} - if: inputs.termux_only != true + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') needs: validate runs-on: ${{ matrix.target.runner }} environment: release-signing @@ -449,6 +477,22 @@ jobs: node apps/desktop/scripts/audit-bundle-arch.mjs \ --arch="${MATRIX_LABEL##*-}" --root=apps/desktop/release + - name: Record stable Windows candidate + if: inputs.release-phase == 'candidate' + shell: bash + env: + TARGET: ${{ matrix.target.label }} + run: | + python -m scripts.bundles.release_artifacts record --platform windows --arch "${TARGET##*-}" \ + --root apps/desktop/release --tag "$HERMES_PAYLOAD_TAG" --out "$RUNNER_TEMP/$TARGET.tar" + - name: Retain stable Windows candidate + if: inputs.release-phase == 'candidate' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: stable-candidate-${{ matrix.target.label }} + path: ${{ runner.temp }}/${{ matrix.target.label }}.tar + if-no-files-found: error + - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -479,14 +523,14 @@ jobs: case "$f" in */Store-*) continue ;; # archived by the Store loop below esac - node scripts/r2-release.mjs put \ + python -m scripts.releases.r2 put \ --tag "$HERMES_PAYLOAD_TAG" --key "$(basename "$f")" --file "$f" done # The Store-submission MSIX (built by the win legs) goes to the tag # archive too — hidden from the builds table and never a feed dir. for f in apps/desktop/release/Store-*.msix; do [ -f "$f" ] || continue - node scripts/r2-release.mjs put \ + python -m scripts.releases.r2 put \ --tag "$HERMES_PAYLOAD_TAG" --key "$(basename "$f")" --file "$f" done @@ -503,7 +547,7 @@ jobs: # pointer last, whole channel green before anything publishes. build-darwin: name: bundled ${{ matrix.target.label }} - if: inputs.termux_only != true + if: inputs.termux_only != true && (inputs.release-phase == '' || inputs.release-phase == 'candidate') needs: validate runs-on: ${{ matrix.target.runner }} environment: release-signing @@ -665,7 +709,7 @@ jobs: # job would publish. A non-publishing run (upload_release=false, # e.g. forks) builds unsigned on purpose. - name: Require signing credentials when publishing - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' shell: bash env: CSC_LINK: ${{ secrets.CSC_LINK }} @@ -691,7 +735,7 @@ jobs: # notarytool takes a FILE PATH for --key; raw .p8 content in argv # dies with `Invalid option: ***`. The build step must NOT re-declare # APPLE_API_KEY in its env: step env shadows GITHUB_ENV. - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' shell: bash env: APPLE_API_KEY_P8: ${{ secrets.APPLE_API_KEY_P8 }} @@ -743,7 +787,7 @@ jobs: # The backstop against a silent unsigned publish: assess the packed # app against the real Gatekeeper policy (requires a Developer ID # signature AND a stapled notarization ticket to pass offline). - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' shell: bash run: | shopt -s nullglob @@ -760,7 +804,7 @@ jobs: done - name: Rename the feed yml per arch - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' # electron-builder writes the channel feed yml (stable-mac.yml / # canary-mac.yml) with the SAME name on both legs; prefix the arch # so the publish job's merge-multiple download keeps both and @@ -777,8 +821,23 @@ jobs: test -s "$f" mv "$f" "apps/desktop/release/${arch}-${channel}-mac.yml" + - name: Record stable macOS candidate + if: inputs.release-phase == 'candidate' + env: + TARGET: ${{ matrix.target.label }} + run: | + python3 -m scripts.bundles.release_artifacts record --platform macos --arch "${TARGET##*-}" \ + --root apps/desktop/release --tag "$HERMES_PAYLOAD_TAG" --out "$RUNNER_TEMP/$TARGET.tar" + - name: Retain stable macOS candidate + if: inputs.release-phase == 'candidate' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: stable-candidate-${{ matrix.target.label }} + path: ${{ runner.temp }}/${{ matrix.target.label }}.tar + if-no-files-found: error + - name: Upload feed metadata - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: hermes-bundled-${{ matrix.target.label }}-${{ inputs.tag }} @@ -788,7 +847,7 @@ jobs: if-no-files-found: error - name: Retain non-publishing build artifacts - if: inputs.upload_release != true + if: inputs.upload_release != true && inputs.release-phase != 'candidate' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: hermes-bundled-${{ matrix.target.label }}-${{ inputs.tag }} @@ -814,7 +873,7 @@ jobs: echo "::error::no darwin release artifacts found"; exit 1 fi for f in "${files[@]}"; do - node scripts/r2-release.mjs put \ + python -m scripts.releases.r2 put \ --tag "$HERMES_PAYLOAD_TAG" --key "$(basename "$f")" --file "$f" done @@ -837,9 +896,9 @@ jobs: # ── Windows updater channels (REAL — gated on build-win32 only) ────────── publish-win32-updater: - name: Publish the win32 App Installer feeds + name: Assemble Windows bundle and optionally publish canary feed needs: [validate, build-win32] - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' runs-on: windows-2025 environment: release-signing timeout-minutes: 45 @@ -947,6 +1006,7 @@ jobs: - name: Bundle + stage the MSIX feeds shell: bash env: + RELEASE_PHASE: ${{ inputs.release-phase }} AZURE_TOKEN_CREDENTIALS: prod AZURE_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ vars.AZURE_TENANT_ID }} @@ -954,7 +1014,20 @@ jobs: # Out-of-store feed (bundled variant): universal .msixbundle + # .appinstaller per channel; plus re-upload the Store-submission # .msix to the tag archive (never a feed dir). - node scripts/stage-msixbundle.mjs --tag "$HERMES_PAYLOAD_TAG" --variant bundled + args=() + if [ "$RELEASE_PHASE" = candidate ]; then args+=(--candidate); fi + node scripts/stage-msixbundle.mjs --tag "$HERMES_PAYLOAD_TAG" --variant bundled "${args[@]}" + if [ "$RELEASE_PHASE" = candidate ]; then + node scripts/bundle-store-msixbundle.mjs --tag "$HERMES_PAYLOAD_TAG" + fi + + - name: Retain stable universal bundles + if: inputs.release-phase == 'candidate' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: stable-candidate-universal + path: apps/desktop/release/*.msixbundle + if-no-files-found: error # ── Windows Store submission (REAL — PARALLEL with publish-win32-updater) ─ # Bundles the two Store-*.msix into one universal Store .msixbundle and @@ -1074,7 +1147,7 @@ jobs: # The per-arch Store-*.msix are already in the immutable archive # (uploaded by the build legs); keep the assembled universal bundle # there too as the record of exactly what was submitted. - node scripts/r2-release.mjs put \ + python -m scripts.releases.r2 put \ --tag "$HERMES_PAYLOAD_TAG" \ --key "$(basename "${{ steps.storebundle.outputs.bundle }}")" \ --file "${{ steps.storebundle.outputs.bundle }}" @@ -1119,7 +1192,7 @@ jobs: # ── macOS updater channel (REAL) ─────────────────────────────────────────── # Merges the per-arch feed ymls (arm64-stable-mac.yml / x64-stable-mac.yml # …) into releases/darwin//-mac.yml via - # scripts/r2-release.mjs finalize. The binaries were staged by the build + # scripts/releases/r2.py finalize. The binaries were staged by the build # legs (releases/tag//); the feed POINTER is written here, last, only # after BOTH darwin legs are green — a failed leg can never publish a # partial channel. The concurrency group is scoped to the channel so two @@ -1149,8 +1222,10 @@ jobs: # Privileged job: pin to the SHA validate admitted, not the tag. ref: ${{ needs.validate.outputs.sha }} - - name: Install locked feed tooling - run: npm ci --workspaces=false --ignore-scripts --no-audit --no-fund + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: '0.12.3' + enable-cache: false - name: Download both darwin legs' feed ymls uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 @@ -1184,12 +1259,12 @@ jobs: # (feed-side); the pointer upload is the last write of the run. shell: bash run: | - node scripts/r2-release.mjs finalize --tag "$HERMES_PAYLOAD_TAG" --dir staged + uv run --no-project --with PyYAML==6.0.3 python -m scripts.releases.r2 finalize --tag "$HERMES_PAYLOAD_TAG" --dir staged termux-deb: name: Build + publish the termux .deb (aarch64) needs: [validate] - if: inputs.upload_release == true + if: inputs.upload_release == true || inputs.release-phase == 'candidate' runs-on: ubuntu-24.04-arm environment: release-signing timeout-minutes: 90 @@ -1455,6 +1530,7 @@ jobs: env: CHANNEL: ${{ steps.channel.outputs.channel }} PREVIOUS_RUN: ${{ inputs.termux_upgrade_from_run }} + RELEASE_PHASE: ${{ inputs.release-phase }} # Passphrase-protected signing keys only; unset for bare keys. TERMUX_APT_GPG_PASSPHRASE: ${{ secrets.TERMUX_APT_GPG_PASSPHRASE }} run: | @@ -1487,15 +1563,20 @@ jobs: "termux/termux-docker@$digest" bash /tmp/check-apt.sh "hermes-$CHANNEL" "$version" fi + if [ "$RELEASE_PHASE" = candidate ]; then + python3 -m scripts.bundles.release_artifacts record --platform termux --arch aarch64 \ + --root termux-build --tag "$HERMES_PAYLOAD_TAG" --out "$RUNNER_TEMP/termux.tar" + exit 0 + fi # --key-is-full is MANDATORY on every feed-dir upload: without it - # r2-release.mjs re-prefixes the key into the tag archive + # scripts.releases.r2 re-prefixes the key into the tag archive # (releases/tag//) and the APT feed never lands at # releases/termux//. shopt -s globstar nullglob # cd into the staged repo for the glob, but resolve the r2 client # absolutely first -- after the cd, scripts/ is no longer in the - # cwd and a bare `node scripts/r2-release.mjs` would ENOENT. - R2="$GITHUB_WORKSPACE/scripts/r2-release.mjs" + # cwd and a bare `python -m scripts.releases.r2` would ENOENT. + export PYTHONPATH="$GITHUB_WORKSPACE" cd termux-build/apt files=(pool/**/*.deb "dists/hermes-$CHANNEL"/main/binary-aarch64/by-hash/*/* key.asc "dists/hermes-$CHANNEL/main/binary-aarch64/Packages" @@ -1507,7 +1588,7 @@ jobs: # index and package has been uploaded and read back successfully. for f in "${files[@]}"; do test -f "$f" - node "$R2" put \ + python3 -m scripts.releases.r2 put \ --tag "$HERMES_PAYLOAD_TAG" \ --key "releases/termux/$CHANNEL/$f" \ --file "$f" \ @@ -1523,6 +1604,14 @@ jobs: -v "$GITHUB_WORKSPACE/scripts/termux/check_apt.sh:/tmp/check-apt.sh:ro" \ -v "$GITHUB_WORKSPACE/scripts/termux/validate_installed.py:/tmp/validate_installed.py:ro" \ "termux/termux-docker@$digest" bash /tmp/check-apt.sh "hermes-$CHANNEL" "$version" "$public_repo" + - name: Retain stable Termux candidate + if: inputs.release-phase == 'candidate' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: stable-candidate-termux + path: ${{ runner.temp }}/termux.tar + if-no-files-found: error + builds-pending: name: Mark the builds table as in progress if: inputs.upload_release == true && inputs.termux_only != true @@ -1546,7 +1635,7 @@ jobs: builds-table: name: Render the release builds table needs: [validate, build-win32, build-darwin, build-linux, publish-win32-updater, publish-darwin-updater, termux-deb] - if: inputs.upload_release == true + if: inputs.upload_release == true && inputs.release-phase == '' runs-on: ubuntu-24.04 environment: release-signing steps: @@ -1572,6 +1661,167 @@ jobs: python3 scripts/render-builds-table.py \ --tag "$HERMES_PAYLOAD_TAG" --repo "$GITHUB_REPOSITORY" + candidate-manifest: + name: Stage verified stable candidate artifacts + needs: [validate, build-win32, build-darwin, publish-win32-updater, termux-deb] + if: inputs.release-phase == 'candidate' + runs-on: ubuntu-latest-32-core + environment: release-signing + timeout-minutes: 90 + outputs: + manifest-url: ${{ steps.manifest.outputs.manifest-url }} + manifest-sha256: ${{ steps.manifest.outputs.manifest-sha256 }} + env: + RELEASE_TAG: ${{ inputs.tag }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: stable-candidate-* + path: candidates + - id: manifest + run: python -m scripts.bundles.release_artifacts assemble --root candidates --bundle-dir candidates/stable-candidate-universal --out release-candidates.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: stable-release-candidates + path: release-candidates.json + if-no-files-found: error + + stable-publish: + name: Verify published candidate files + needs: validate + if: inputs.release-phase == 'publish' + runs-on: ubuntu-latest-32-core + environment: release-signing + timeout-minutes: 90 + env: + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_MANIFEST_SHA256: ${{ inputs.manifest-sha256 }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: stable-release-candidates + - run: python -m scripts.bundles.release_artifacts publish --manifest release-candidates.json --root verified + + stable-store: + name: Submit the verified Store package + needs: [validate, stable-publish] + if: inputs.release-phase == 'publish' + runs-on: windows-2025 + environment: release-signing + timeout-minutes: 60 + env: + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_MANIFEST_SHA256: ${{ inputs.manifest-sha256 }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + MS_STORE_TENANT_ID: ${{ secrets.MS_STORE_TENANT_ID }} + MS_STORE_SELLER_ID: ${{ secrets.MS_STORE_SELLER_ID }} + MS_STORE_CLIENT_ID: ${{ secrets.MS_STORE_CLIENT_ID }} + MS_STORE_CLIENT_SECRET: ${{ secrets.MS_STORE_CLIENT_SECRET }} + MS_STORE_PRODUCT_ID: ${{ vars.MS_STORE_PRODUCT_ID }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: stable-release-candidates + - name: Retrieve the exact Store candidate + shell: bash + run: python -m scripts.bundles.release_artifacts materialize --manifest release-candidates.json --root verified --store-only + - uses: microsoft/microsoft-store-apppublisher@cc9910a8d59f2eb55cbb83df0a3800cf3b5300e0 # v1.4 + - name: Submit without rebuilding + shell: bash + run: | + test -n "$MS_STORE_PRODUCT_ID" + msstore reconfigure --tenantId "$MS_STORE_TENANT_ID" --sellerId "$MS_STORE_SELLER_ID" --clientId "$MS_STORE_CLIENT_ID" --clientSecret "$MS_STORE_CLIENT_SECRET" + files=(verified/Store-*.msixbundle) + test "${#files[@]}" -eq 1 + test -f "${files[0]}" + msstore publish "${files[0]}" -id "$MS_STORE_PRODUCT_ID" + + stable-promote: + name: Promote verified stable bundle channels + needs: validate + if: inputs.release-phase == 'promote' + runs-on: ubuntu-latest-32-core + environment: release-signing + timeout-minutes: 90 + env: + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_MANIFEST_SHA256: ${{ inputs.manifest-sha256 }} + GH_TOKEN: ${{ github.token }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: '0.12.3' + enable-cache: false + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: stable-release-candidates + - run: uv run --no-project --with PyYAML==6.0.3 python -m scripts.bundles.release_artifacts promote --manifest release-candidates.json --root verified + - run: python scripts/render-builds-table.py --tag "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" + + stable-phase-result: + name: Stable bundle phase completed + if: always() && inputs.release-phase != '' + needs: [validate, build-win32, build-darwin, publish-win32-updater, termux-deb, candidate-manifest, stable-publish, stable-store, stable-promote] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + - name: Require every phase job + env: + RELEASE_NEEDS: ${{ toJSON(needs) }} + RELEASE_PHASE: ${{ inputs.release-phase }} + run: | + python - <<'PY' + import json, os + from scripts.releases.stable import require_success + phases = { + 'candidate': ['validate', 'build-win32', 'build-darwin', 'publish-win32-updater', 'termux-deb', 'candidate-manifest'], + 'publish': ['validate', 'stable-publish', 'stable-store'], + 'promote': ['validate', 'stable-promote'], + } + require_success(json.loads(os.environ['RELEASE_NEEDS']), phases[os.environ['RELEASE_PHASE']]) + PY + publish-canary: name: Publish the canary release needs: [build-win32, build-darwin, build-linux, builds-table, publish-win32-updater, publish-darwin-updater] diff --git a/.github/workflows/docker-lint.yml b/.github/workflows/docker-lint.yml index 89b80fa10e..05297c93b7 100644 --- a/.github/workflows/docker-lint.yml +++ b/.github/workflows/docker-lint.yml @@ -17,8 +17,8 @@ permissions: contents: read concurrency: - group: docker-lint-${{ github.ref }} - cancel-in-progress: true + group: docker-lint-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: hadolint: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2aca250939..5e787b0945 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,18 +2,34 @@ name: Docker Build, Test, and Publish on: # This workflow owns its own triggers. ci.yml does not call it. - # A reusable-workflow call eeps the caller run in progress for that full time. - # GitHub refuses ``gh run rerun`` on a run that is still in progress. + # A reusable-workflow call keeps the caller run in progress for that full time. + # GitHub refuses ``gh run rerun`` on a run that is still in progress. # Thus one slow advisory job blocked every rerun of the fast required jobs. A separate # run reruns and cancels independently. # # Trusted main pushes resolve the environment-scoped Docker Hub secrets in # this same workflow, never across a workflow boundary. + # + # The ``release: published`` trigger was REMOVED on purpose: a GitHub release + # event must never rebuild or rewrite the stable Docker channel. Versioned + # tags, candidate tags and the stable/latest aliases are published only + # through the staged release path (workflow_call below), which the parent + # stable-release workflow gates behind full CI + package acceptance. pull_request: push: branches: [main] - release: - types: [published] + workflow_call: + inputs: + release-phase: + description: "Stable-release phase: 'test', 'publish' or 'promote'. Empty keeps the standalone triggers." + required: false + type: string + default: '' + tag: + description: "Exact stable tag (vX.Y.Z) under release. The caller is dispatched on this tag, so github.sha is the release candidate." + required: false + type: string + default: '' permissions: contents: read @@ -21,21 +37,50 @@ permissions: # Concurrency: push/release runs are NEVER cancelled so every merge gets # its own image. PR runs reuse a PR-scoped group with # cancel-in-progress: true so rapid pushes to the same PR collapse to -# the latest commit. +# the latest commit. Release runs include the run_id: several reusable calls +# (test/publish/promote) of this workflow live inside ONE parent run, and a +# shared group would cancel the parent mid-release. concurrency: - group: docker-${{ github.event.pull_request.number || github.ref }} + group: docker-${{ github.event.pull_request.number || github.ref }}-${{ inputs.release-phase || 'standalone' }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: IMAGE_NAME: nousresearch/hermes-agent jobs: + # Resolve the release phase. Release modes run only via workflow_call from + # the parent stable-release workflow on the tagged candidate commit. + mode: + name: Resolve release phase + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + phase: ${{ steps.resolve.outputs.phase }} + release: ${{ steps.resolve.outputs.release }} + steps: + - id: resolve + env: + PHASE: ${{ inputs.release-phase }} + run: | + set -euo pipefail + case "$PHASE" in + '') echo "phase=standalone" >> "$GITHUB_OUTPUT"; echo "release=false" >> "$GITHUB_OUTPUT" ;; + test) echo "phase=test" >> "$GITHUB_OUTPUT"; echo "release=true" >> "$GITHUB_OUTPUT" ;; + publish|promote) + echo "phase=$PHASE" >> "$GITHUB_OUTPUT"; echo "release=true" >> "$GITHUB_OUTPUT" ;; + *) echo "::error::Invalid release-phase input: $PHASE"; exit 1 ;; + esac + # Classify the PR's changed files. ci.yml used to gate the docker call on # its own ``detect`` outputs; now that this workflow triggers itself, it # runs the same composite action. On push and release the classifier fails # open (every lane true), so post-merge validation is never weakened. + # Release phases skip classification entirely: the parent already ran full + # CI and the tag dispatch has no meaningful PR diff to classify. detect: name: Detect affected areas + needs: [mode] + if: needs.mode.outputs.release != 'true' runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -65,12 +110,21 @@ jobs: echo "build=false" >> "$GITHUB_OUTPUT" fi - # Build and test the image for each architecture. This job runs PR code, - # so it must remain secret-free. Publishing happens in the separate, - # protected publish job after these tests pass. + # Build and test the image for each architecture. This job runs PR code, so + # in standalone mode it must remain secret-free. Publishing happens in the + # separate, protected publish paths after these tests pass. + # + # Runs ONLY in standalone mode and in the release 'test' phase: it builds + # the SAME Dockerfile and runs the SAME real docker-integration tests, then + # (test phase) saves the tested per-arch image archive + sha256 as run + # artifacts. No registry push, no signing secrets. The publish phase never + # rebuilds — it downloads these exact artifacts. build: - needs: [detect] - if: github.repository == 'NousResearch/hermes-agent' && needs.detect.outputs.build == 'true' + name: Build and test image (${{ matrix.arch }}) + needs: [mode, detect] + if: >- + !cancelled() && needs.mode.result == 'success' && + ((needs.mode.outputs.release != 'true' && github.repository == 'NousResearch/hermes-agent' && needs.detect.outputs.build == 'true') || needs.mode.outputs.phase == 'test') strategy: fail-fast: false matrix: @@ -93,6 +147,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Release calls are dispatched on the stable tag: pin to the exact + # candidate commit, never a mutable branch ref. + ref: ${{ needs.mode.outputs.release == 'true' && github.sha || '' }} - name: Write install stamp run: python3 scripts/write_install_stamp.py --output install-stamp.json --distribution docker --update-mechanism external --source ci @@ -141,6 +199,8 @@ jobs: # it between jobs via ``docker save``/``upload-artifact`` is slower # than the build itself. Reusing the existing daemon state is the # cheapest path to coverage on every PR that touches docker code. + # (The release path DOES pay that cost — see the save steps below — + # because publish must push the exact tested bytes, not a rebuild.) # --------------------------------------------------------------------- - name: Install uv (for docker tests) uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 @@ -179,13 +239,63 @@ jobs: # count to the core count. HERMES_TEST_WORKERS=$(nproc) scripts/run_tests.sh tests/docker/ + # --------------------------------------------------------------------- + # Release 'test' phase only: hand the EXACT tested bytes to the publish + # phase via actions artifacts. No registry push, no credentials here. + # --------------------------------------------------------------------- + - name: Save tested image archive (release test) + if: needs.mode.outputs.phase == 'test' + env: + ARCH: ${{ matrix.arch }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + mkdir -p /tmp/image-artifacts + docker image inspect "${IMAGE_NAME}:test" > /tmp/image-artifacts/image-inspect.json + python3 - <<'PY' + import json, os + from pathlib import Path + image = json.loads(Path('/tmp/image-artifacts/image-inspect.json').read_text())[0] + if image['Architecture'] != os.environ['ARCH']: + raise SystemExit('Docker image architecture mismatch') + Path('/tmp/image-artifacts/identity.json').write_text(json.dumps({ + 'tag': os.environ['RELEASE_TAG'], 'commit': os.environ['GITHUB_SHA'], + 'arch': os.environ['ARCH'], 'imageId': image['Id']}), encoding='utf-8') + PY + docker save --output "/tmp/image-artifacts/image-${ARCH}.tar" "${IMAGE_NAME}:test" + ( + cd /tmp/image-artifacts + sha256sum "image-${ARCH}.tar" > "image-${ARCH}.tar.sha256" + ) + + - name: Upload tested image archive (release test) + if: needs.mode.outputs.phase == 'test' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: docker-test-image-${{ matrix.arch }}-${{ inputs.tag }} + path: | + /tmp/image-artifacts/image-${{ matrix.arch }}.tar + /tmp/image-artifacts/image-${{ matrix.arch }}.tar.sha256 + /tmp/image-artifacts/identity.json + if-no-files-found: error + retention-days: 7 + compression-level: 0 + # --------------------------------------------------------------------------- - # Rebuild and push each architecture only after the unprivileged build/test - # matrix passes. This job is the sole Docker Hub credential boundary. + # Standalone publish: rebuild and push each architecture only after the + # unprivileged build/test matrix passes, on trusted main pushes. + # This job is the sole Docker Hub credential boundary for merges. + # + # NOTE: main pushes tag :main ONLY. :latest is a user-facing stable alias + # now; it moves only in the release 'promote' phase after the parent's + # global release gate, so a main push can never advance the stable channel. # --------------------------------------------------------------------------- publish: - if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') - needs: [build] + if: >- + needs.mode.outputs.release != 'true' && + github.repository == 'NousResearch/hermes-agent' && + github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [mode, build] environment: container-publish strategy: fail-fast: false @@ -264,13 +374,16 @@ jobs: # This is a registry-side operation — no building, no layer re-push — # so it runs in ~30 seconds. # - # On main pushes: tags both :main and :latest. - # On releases: tags :. + # Main pushes tag :main only. :latest is reserved for the release 'promote' + # phase (see release-promote below). # --------------------------------------------------------------------------- merge: - if: github.repository == 'NousResearch/hermes-agent' && (github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'release') + if: >- + needs.mode.outputs.release != 'true' && + github.repository == 'NousResearch/hermes-agent' && + github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest - needs: [publish] + needs: [mode, publish] timeout-minutes: 10 environment: container-publish steps: @@ -302,18 +415,13 @@ jobs: working-directory: /tmp/digests env: IMAGE_NAME: ${{ env.IMAGE_NAME }} - RELEASE_TAG: ${{ github.event.release.tag_name }} run: | set -euo pipefail args=() for digest_file in *; do args+=("${IMAGE_NAME}@sha256:${digest_file}") done - if [ "${{ github.event_name }}" = "release" ]; then - tags=(-t "${IMAGE_NAME}:${RELEASE_TAG}") - else - tags=(-t "${IMAGE_NAME}:main" -t "${IMAGE_NAME}:latest") - fi + tags=(-t "${IMAGE_NAME}:main") # Retry: Docker Hub API + just-pushed digest eventual consistency # can transiently fail the create; the operation is idempotent. for i in 1 2 3; do @@ -331,10 +439,360 @@ jobs: - name: Inspect image env: IMAGE_NAME: ${{ env.IMAGE_NAME }} - RELEASE_TAG: ${{ github.event.release.tag_name }} + run: docker buildx imagetools inspect "${IMAGE_NAME}:main" + + # =========================================================================== + # Staged stable-release path (workflow_call from stable-release.yml). + # The caller is dispatched on the exact stable TAG, so github.sha in every + # reusable call is the release candidate commit. + # =========================================================================== + + # Release 'publish' phase: load the EXACT tested image archives uploaded by + # the 'test' phase of this SAME workflow run, verify their hashes, push + # per-arch and NEVER rebuild. No owner gate: on a fork the Docker Hub + # login/push fails loudly (missing credentials) instead of faking green. + release-publish: + name: Publish tested Docker image (${{ matrix.arch }}) + if: needs.mode.outputs.phase == 'publish' + needs: [mode] + environment: container-publish + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + - arch: arm64 + runs-on: ubuntu-latest-32-core + timeout-minutes: 45 + env: + ARCH: ${{ matrix.arch }} + RELEASE_TAG: ${{ inputs.tag }} + steps: + - name: Checkout release code (helper scripts only, no build) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + + - name: Download tested image archive from the test phase (same run) + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: docker-test-image-${{ matrix.arch }}-${{ inputs.tag }} + path: /tmp/image-artifacts + + - name: Verify tested archive hash (published bytes == tested bytes) run: | - if [ "${{ github.event_name }}" = "release" ]; then - docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_TAG}" - else - docker buildx imagetools inspect "${IMAGE_NAME}:main" - fi + set -euo pipefail + cd /tmp/image-artifacts + echo " expected: $(cat "image-${ARCH}.tar.sha256")" + sha256sum --check "image-${ARCH}.tar.sha256" + + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Load tested image + run: | + set -euo pipefail + docker load --input /tmp/image-artifacts/image-${ARCH}.tar + docker image inspect "${IMAGE_NAME}:test" > /tmp/image-artifacts/loaded.json + python3 - <<'PY' + import json, os + from pathlib import Path + identity = json.loads(Path('/tmp/image-artifacts/identity.json').read_text()) + loaded = json.loads(Path('/tmp/image-artifacts/loaded.json').read_text())[0] + expected = (os.environ['RELEASE_TAG'], os.environ['GITHUB_SHA'], os.environ['ARCH']) + if (identity['tag'], identity['commit'], identity['arch']) != expected: + raise SystemExit('Tested Docker archive identity mismatch') + if (loaded['Id'], loaded['Architecture']) != (identity['imageId'], identity['arch']): + raise SystemExit('Loaded Docker image differs from tested image') + PY + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push tested image with per-arch release tag + run: | + set -euo pipefail + docker tag "${IMAGE_NAME}:test" "${IMAGE_NAME}:${RELEASE_TAG}-${ARCH}" + docker push "${IMAGE_NAME}:${RELEASE_TAG}-${ARCH}" + + - name: Record pushed per-arch digest + run: | + set -euo pipefail + mkdir -p /tmp/digests + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_TAG}-${ARCH}" \ + --format '{{json .Manifest.Digest}}' | tr -d '"')" + case "$digest" in + sha256:*) ;; + *) echo "::error::Unexpected digest format: $digest"; exit 1 ;; + esac + printf '%s' "$digest" > "/tmp/digests/${ARCH}.digest" + cat "/tmp/digests/${ARCH}.digest" + + - name: Upload per-arch digest artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: docker-publish-digest-${{ matrix.arch }}-${{ inputs.tag }} + path: /tmp/digests/${{ matrix.arch }}.digest + if-no-files-found: error + retention-days: 7 + + # Assemble the versioned multi-arch manifest list from the pushed per-arch + # digests and emit the reference/digest manifest artifact consumed by the + # promote phase. Registry-side only; nothing is rebuilt. + release-publish-manifest: + name: Assemble versioned manifest and digest receipt + if: needs.mode.outputs.phase == 'publish' + needs: [mode, release-publish] + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: container-publish + env: + RELEASE_TAG: ${{ inputs.tag }} + steps: + - name: Checkout release code (helper scripts only) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + + - name: Download per-arch digests + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + path: /tmp/digests + pattern: docker-publish-digest-*-${{ inputs.tag }} + merge-multiple: true + + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Create versioned manifest list + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + cd /tmp/digests + test -f amd64.digest && test -f arm64.digest + args=() + for arch in amd64 arm64; do + args+=("${IMAGE_NAME}@$(cat "${arch}.digest")") + done + for i in 1 2 3; do + if docker buildx imagetools create \ + -t "${IMAGE_NAME}:${RELEASE_TAG}" \ + "${args[@]}"; then + break + fi + if [ "$i" = 3 ]; then + echo "::error::imagetools create failed after 3 attempts" + exit 1 + fi + sleep 20 + done + + - name: Record manifest-list digest + id: list + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + digest="$(docker buildx imagetools inspect "${IMAGE_NAME}:${RELEASE_TAG}" \ + --format '{{json .Manifest.Digest}}' | tr -d '"')" + case "$digest" in + sha256:*) ;; + *) echo "::error::Unexpected manifest-list digest: $digest"; exit 1 ;; + esac + echo "digest=$digest" >> "$GITHUB_OUTPUT" + + - name: Emit release manifest artifact + id: manifest + run: | + set -euo pipefail + mkdir -p /tmp/manifest + python3 -m scripts.releases.docker manifest \ + --tag "$RELEASE_TAG" \ + --commit "$GITHUB_SHA" \ + --digest-amd64 "$(sed 's/^sha256://' /tmp/digests/amd64.digest)" \ + --digest-arm64 "$(sed 's/^sha256://' /tmp/digests/arm64.digest)" \ + > /tmp/manifest/manifest.json + python3 - "$RELEASE_TAG" "${{ steps.list.outputs.digest }}" <<'EOF' + import json, sys + manifest = json.load(open("/tmp/manifest/manifest.json")) + manifest["list-digest"] = sys.argv[2] + manifest["image"] = "nousresearch/hermes-agent" + manifest["tags"] = [sys.argv[1]] + json.dump(manifest, open("/tmp/manifest/manifest.json", "w"), indent=2) + EOF + cat /tmp/manifest/manifest.json + + - name: Verify manifest identity + run: | + set -euo pipefail + python3 -m scripts.releases.docker verify \ + --tag "$RELEASE_TAG" --commit "$GITHUB_SHA" /tmp/manifest/manifest.json + + - name: Upload release manifest artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: docker-publish-manifest-${{ inputs.tag }} + path: /tmp/manifest/manifest.json + if-no-files-found: error + retention-days: 7 + + # Strict phase gate: in every release phase, the jobs that phase requires + # must actually have run and succeeded. `if: always()` keeps this job in + # the graph even when earlier jobs were skipped, so a skipped/failed + # publisher turns this red instead of letting the phase go green. + release-phase-gate: + name: Docker phase requirements met + if: always() && inputs.release-phase != '' + needs: [mode, build, release-publish, release-publish-manifest, release-promote] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Validate phase job results + env: + PHASE: ${{ needs.mode.outputs.phase }} + BUILD: ${{ needs.build.result }} + RELEASE_PUBLISH: ${{ needs.release-publish.result }} + RELEASE_MANIFEST: ${{ needs.release-publish-manifest.result }} + RELEASE_PROMOTE: ${{ needs.release-promote.result }} + MODE: ${{ needs.mode.result }} + run: | + set -euo pipefail + test "$MODE" = success + case "$PHASE" in + test) + failures=() + [ "$BUILD" = success ] || failures+=("build=$BUILD") + [ "${#failures[@]}" -eq 0 ] || { printf '::error::%s\n' "${failures[@]}"; exit 1; } + ;; + publish) + failures=() + [ "$RELEASE_PUBLISH" = success ] || failures+=("release-publish=$RELEASE_PUBLISH") + [ "$RELEASE_MANIFEST" = success ] || failures+=("release-publish-manifest=$RELEASE_MANIFEST") + [ "${#failures[@]}" -eq 0 ] || { printf '::error::%s\n' "${failures[@]}"; exit 1; } + ;; + promote) + test "$RELEASE_PROMOTE" = success + ;; + *) echo "::error::Unknown phase $PHASE"; exit 1 ;; + esac + + # Release 'promote' phase: runs ONLY after the parent's global release gate + # (all bundle/acceptance/publication jobs succeeded). Repoints the + # user-facing stable aliases (stable AND latest) at the exact tested + # manifest-list digest from the publish phase, then reads the aliases back + # from the registry and fails if they do not resolve to that digest. + # This is the ONLY place in this workflow where stable/latest can move. + release-promote: + name: Promote stable Docker aliases + if: needs.mode.outputs.phase == 'promote' + needs: [mode] + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: container-publish + env: + RELEASE_TAG: ${{ inputs.tag }} + steps: + - name: Checkout release code (helper scripts only) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + + - name: Download release manifest from the publish phase (same run) + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: docker-publish-manifest-${{ inputs.tag }} + path: /tmp/manifest + + - name: Verify tested manifest identity + run: python3 -m scripts.releases.docker verify \ + --tag "$RELEASE_TAG" --commit "$GITHUB_SHA" /tmp/manifest/manifest.json + + - name: Set up Docker Buildx + id: buildx + continue-on-error: true + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Set up Docker Buildx (retry) + if: steps.buildx.outcome == 'failure' + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to Docker Hub + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Promote stable and latest to the tested digest + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + LIST_DIGEST="$(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" + for i in 1 2 3; do + if docker buildx imagetools create \ + -t "${IMAGE_NAME}:stable" \ + -t "${IMAGE_NAME}:latest" \ + "${IMAGE_NAME}@${LIST_DIGEST}"; then + break + fi + if [ "$i" = 3 ]; then + echo "::error::imagetools create (promote) failed after 3 attempts" + exit 1 + fi + sleep 20 + done + + - name: Read back and verify the stable aliases + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + EXPECTED="$(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" + sleep 10 # eventual consistency of just-created aliases + for alias in stable latest; do + for i in 1 2 3; do + got="$(docker buildx imagetools inspect "${IMAGE_NAME}:${alias}" \ + --format '{{json .Manifest.Digest}}' | tr -d '"')" && break + [ "$i" = 3 ] && { echo "::error::inspect ${alias} failed 3 times"; exit 1; } + sleep 20 + done + if [ "$got" != "$EXPECTED" ]; then + echo "::error::Alias ${alias} resolves to ${got}, expected ${EXPECTED}" + exit 1 + fi + echo "${IMAGE_NAME}:${alias} -> ${got} (verified)" + done + + - name: Promote receipt + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + run: | + set -euo pipefail + { + echo "image: ${IMAGE_NAME}" + echo "tag: ${RELEASE_TAG}" + echo "digest: $(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" + echo "aliases: stable,latest" + } | tee /tmp/manifest/promote-receipt.txt diff --git a/.github/workflows/install-e2e-macos-run.yml b/.github/workflows/install-e2e-macos-run.yml index 15ed09b645..75c6d6526d 100644 --- a/.github/workflows/install-e2e-macos-run.yml +++ b/.github/workflows/install-e2e-macos-run.yml @@ -52,6 +52,10 @@ on: required: false type: string default: '' + bundle-manifest-sha256: + description: Pinned transition manifest digest for stable acceptance + type: string + default: '' bundle-arch: description: 'Architecture of the bundled pair: arm64 (macos-15) or x64 (macos-15-intel).' required: false @@ -101,6 +105,7 @@ jobs: # installer/updater talk to. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ github.sha }} fetch-depth: 0 - name: Start screen recording @@ -184,6 +189,8 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} - name: Install driver dependencies run: npm ci --ignore-scripts --no-audit --no-fund @@ -201,6 +208,7 @@ jobs: --manifest-url "$BUNDLE_MANIFEST" --arch "$BUNDLE_ARCH" env: BUNDLE_MANIFEST: ${{ inputs.bundle-manifest-url }} + BUNDLE_MANIFEST_SHA256: ${{ inputs.bundle-manifest-sha256 }} BUNDLE_ARCH: ${{ inputs.bundle-arch }} HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs @@ -211,6 +219,7 @@ jobs: --manifest-url "$BUNDLE_MANIFEST" --arch "$BUNDLE_ARCH" env: BUNDLE_MANIFEST: ${{ inputs.bundle-manifest-url }} + BUNDLE_MANIFEST_SHA256: ${{ inputs.bundle-manifest-sha256 }} BUNDLE_ARCH: ${{ inputs.bundle-arch }} HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs diff --git a/.github/workflows/install-e2e-run.yml b/.github/workflows/install-e2e-run.yml index 8a0f41ea70..f61e0e9ec4 100644 --- a/.github/workflows/install-e2e-run.yml +++ b/.github/workflows/install-e2e-run.yml @@ -94,6 +94,7 @@ jobs: # in that clone. A shallow clone cannot serve either need. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ github.sha }} fetch-depth: 0 # One recording mechanism on every OS (Xvfb gives headless linux a diff --git a/.github/workflows/install-e2e-windows-run.yml b/.github/workflows/install-e2e-windows-run.yml index 534a6e35e0..8139c2c4ee 100644 --- a/.github/workflows/install-e2e-windows-run.yml +++ b/.github/workflows/install-e2e-windows-run.yml @@ -80,6 +80,10 @@ on: bundle-manifest-url: type: string default: '' + bundle-manifest-sha256: + description: Pinned transition manifest digest for stable acceptance + type: string + default: '' bundle-arch: type: string default: x64 @@ -95,6 +99,8 @@ jobs: timeout-minutes: ${{ inputs.timeout-minutes }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} - name: Install driver dependencies and verify helpers shell: pwsh run: | @@ -111,6 +117,7 @@ jobs: shell: pwsh env: BUNDLE_MANIFEST: ${{ inputs.bundle-manifest-url }} + BUNDLE_MANIFEST_SHA256: ${{ inputs.bundle-manifest-sha256 }} BUNDLE_ARCH: ${{ inputs.bundle-arch }} run: | powershell -NoProfile -ExecutionPolicy Bypass -File tests/install/windows-bundled-e2e.ps1 -ManifestUrl $env:BUNDLE_MANIFEST -Arch $env:BUNDLE_ARCH @@ -166,6 +173,7 @@ jobs: # in that clone. A shallow checkout cannot serve either need. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + ref: ${{ github.sha }} fetch-depth: 0 # One recording mechanism on every OS: the composite action installs diff --git a/.github/workflows/install-e2e.yml b/.github/workflows/install-e2e.yml index b299c86c49..c659e616d7 100644 --- a/.github/workflows/install-e2e.yml +++ b/.github/workflows/install-e2e.yml @@ -70,6 +70,51 @@ on: required: false type: string default: '' + workflow_call: + # Reusable entry for the stable-release orchestrator. Same input names as + # workflow_dispatch; on a release run the caller dispatches on the exact + # candidate tag, so github.sha is the candidate commit throughout. + # + # route: which combinations to run (same choices as dispatch). The + # bundled routes require the matching bundle manifests, exactly like a + # manual dispatch — a source (non-bundle) release run passes route=all + # with empty manifests and gets the source install/update matrix only. + inputs: + release: + description: 'Stable-release candidate run (informational; jobs behave as route=all with empty manifests).' + required: false + type: boolean + default: false + route: + description: 'Which combinations to run. all = every OS; both/update/installer = the linux legs; windows-desktop = the windows legs; macos-desktop = the macos legs.' + required: false + type: string + default: all + windows-bundle-manifest: + description: 'HTTPS manifest pinning signed Windows OLD/NEW msixbundles; NEW must match this ref.' + required: false + type: string + default: '' + macos-bundle-manifest: + description: 'HTTPS manifest pinning signed macOS OLD/NEW app ZIPs; NEW must match this ref.' + required: false + type: string + default: '' + tag-count: + description: 'How many release tags to sample (newest, oldest, and a spread between).' + required: false + type: string + default: '3' + install-ref: + description: 'Optional exact release tag for a focused reproduction; overrides tag-count.' + required: false + type: string + default: '' + exclude-ref: + description: 'Exact release tag to EXCLUDE from the sampled OLD baselines (e.g. the candidate itself — testing an update from it would be a no-change test).' + required: false + type: string + default: '' schedule: # Every 12 hours, off the hour to avoid the top-of-hour runner crunch. - cron: '20 7,19 * * *' @@ -82,9 +127,12 @@ on: permissions: contents: read +# A workflow_call run (stable release) is never cancelled: its group uses +# github.run_id so a parent rerun cannot kill this child mid-flight. The +# scheduled/push runs keep ref-scoped collapse. concurrency: - group: install-e2e-${{ github.ref }} - cancel-in-progress: true + group: install-e2e-${{ inputs.release == true && github.run_id || github.ref }} + cancel-in-progress: ${{ inputs.release != true }} jobs: # Which released versions do we test updating FROM? Resolved once, @@ -105,24 +153,36 @@ jobs: with: filter: blob:none fetch-tags: true - sparse-checkout: scripts/sandbox/pick-release-tags.sh + sparse-checkout: | + scripts/sandbox/pick-release-tags.sh + scripts/releases/pick_tags.py sparse-checkout-cone-mode: false - id: pick env: - # Dispatch inputs never touch shell syntax directly: TAG_COUNT - # arrives via the environment and is validated decimal-only (bash - # arithmetic reads a leading zero as octal). GitHub's 256-job cap - # applies to each per-OS matrix separately; at 10 tags the largest - # is windows at 180 (first over the cap at 15 tags = 270). + # Inputs never touch shell syntax directly: TAG_COUNT arrives via + # the environment and is validated decimal-only (bash arithmetic + # reads a leading zero as octal). GitHub's 256-job cap applies to + # each per-OS matrix separately; at 10 tags the largest is windows + # at 180 (first over the cap at 15 tags = 270). + # + # EXCLUDE_REF removes one tag from the sampled OLD baselines: on a + # stable release the candidate tag itself is the newest release + # tag, and sampling it as an OLD version would produce a no-change + # update leg — the candidate must never update from itself. TAG_COUNT: ${{ inputs.tag-count || 2 }} INSTALL_REF: ${{ inputs.install-ref }} + EXCLUDE_REF: ${{ inputs.exclude-ref }} + RELEASE_MODE: ${{ inputs.release }} run: | set -euo pipefail [[ "$TAG_COUNT" =~ ^(10|[1-9])$ ]] || { echo "tag-count must be 1-10, got: $TAG_COUNT" >&2; exit 1; } if [ -n "$INSTALL_REF" ]; then [[ "$INSTALL_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || { echo 'install-ref must be an exact release tag' >&2; exit 1; } + [ "$INSTALL_REF" != "$EXCLUDE_REF" ] || { echo "Candidate cannot be its own baseline" >&2; exit 1; } git rev-parse --verify "refs/tags/$INSTALL_REF^{commit}" >/dev/null tags="$(jq -cn --arg ref "$INSTALL_REF" '[$ref]')" + elif [ "$RELEASE_MODE" = true ]; then + tags="$(python3 -m scripts.releases.pick_tags --count "$TAG_COUNT" --exclude-ref "$EXCLUDE_REF")" else tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")" fi @@ -298,6 +358,50 @@ jobs: retention-days: 14 if-no-files-found: error + # Strict source-matrix gate (release runs). needs collapses each matrix to + # one aggregate, so the three source parents are the required check here. + # Every source leg must succeed — a skip is a failure EXCEPT when it is the + # native, declared capability skip (a starting tag predating the desktop + # app reports tag-has-desktop=false and its GUI leg legitimately cannot + # run); those are honest skips, counted and listed. A no-legs-at-all + # generation (empty matrix) also fails: a release run must actually test + # the source install/update paths. + source-gate: + name: Source matrix gate + if: always() && inputs.release == true + needs: [generate-matrix, linux, windows, macos] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Evaluate source legs + env: + LINUX: ${{ needs.linux.result }} + WINDOWS: ${{ needs.windows.result }} + MACOS: ${{ needs.macos.result }} + LINUX_MATRIX: ${{ needs.generate-matrix.outputs.linux }} + WINDOWS_MATRIX: ${{ needs.generate-matrix.outputs.windows }} + MACOS_MATRIX: ${{ needs.generate-matrix.outputs.macos }} + run: | + set -euo pipefail + fail=0 + for os_name in LINUX WINDOWS MACOS; do + result="${!os_name}" + matrix_var="${os_name}_MATRIX" + matrix="${!matrix_var}" + legs=$(printf '%s' "$matrix" | jq 'if type == "object" then ([.include // []] | add // []) else . end | length') + echo "$os_name: result=$result legs=$legs" + if [ "$result" != "success" ] || [ "$legs" -eq 0 ]; then + echo "$os_name requires executed source coverage" + echo "::error::$os_name source matrix result=$result (legs=$legs)" + fail=1 + fi + done + if [ "$fail" -ne 0 ]; then + echo "::error::source install/update matrix did not fully succeed; a release cannot proceed on a partial or skipped matrix" + exit 1 + fi + echo "Source install/update matrix: all OS aggregates succeeded" + # The outcome, human-readable: the plan chart again, with each cell # replaced by how that leg actually concluded. Per-leg conclusions are # NOT reachable through `needs` (a matrix job's result collapses to one diff --git a/.github/workflows/installer-tests.yml b/.github/workflows/installer-tests.yml index 109db526ff..75b5668b19 100644 --- a/.github/workflows/installer-tests.yml +++ b/.github/workflows/installer-tests.yml @@ -13,8 +13,8 @@ permissions: contents: read concurrency: - group: installer-tests-${{ github.ref }} - cancel-in-progress: true + group: installer-tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: powershell: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6b4023ebe5..cc380fc6ce 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,8 +23,8 @@ permissions: contents: read concurrency: - group: lint-${{ github.ref }} - cancel-in-progress: true + group: lint-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: lint-diff: diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index ccd47d1b09..6a6f12917d 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -14,16 +14,24 @@ on: pull_request: push: branches: [main] + workflow_call: + inputs: + release: + description: 'Stable-release candidate run: force the flake-check lane regardless of the classifier.' + required: false + type: boolean + default: false permissions: contents: read -# PR runs collapse to the newest commit. A push to main is never cancelled: -# each one saves the store cache that later PRs restore from, so cancelling a -# merge would leave the next PR to build from nothing. +# A workflow_call run (stable release) is never cancelled: its group uses +# github.run_id so a parent rerun cannot kill this child mid-flight, and +# cancel-in-progress is false there — each run saves the store cache that +# later PRs restore from. concurrency: - group: nix-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: nix-${{ inputs.release == true && github.run_id || github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ inputs.release != true && github.event_name == 'pull_request' }} jobs: # A `paths:` filter cannot gate this workflow correctly. The flake packages @@ -39,6 +47,10 @@ jobs: nix: ${{ steps.classify.outputs.nix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # github.sha is the exact candidate commit on a stable tag-dispatched + # caller and the head SHA on every other event. + ref: ${{ github.sha }} - name: Detect affected areas id: classify @@ -49,7 +61,9 @@ jobs: flake-check: name: nix flake check needs: [detect] - if: needs.detect.outputs.nix == 'true' + # Release runs force the lane: a stable candidate must build the flake + # no matter what its diff touches. + if: needs.detect.outputs.nix == 'true' || inputs.release == true # The build compiles the package and its whole dependency closure, so this # takes minutes and not seconds when the cache misses. `nix flake check` # builds 21 checks, and --max-jobs defaults to the core count. It uses the @@ -59,6 +73,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # github.sha is the exact candidate commit on a stable tag-dispatched + # caller and the head SHA on every other event. + ref: ${{ github.sha }} - name: Install Nix uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31.11.0 diff --git a/.github/workflows/pm-bundle.yml b/.github/workflows/pm-bundle.yml index 5f0bf75b57..0455482bb2 100644 --- a/.github/workflows/pm-bundle.yml +++ b/.github/workflows/pm-bundle.yml @@ -15,9 +15,21 @@ on: workflow_dispatch: inputs: ref: - description: 'Git ref to bundle (default: the triggering ref)' + description: 'Git ref to bundle (default: the triggering commit, github.sha)' required: false type: string + workflow_call: + inputs: + release: + description: 'Stable-release candidate run (ignored by the jobs; uniform callable surface).' + required: false + type: boolean + default: false + ref: + description: 'Git ref to bundle (default: the triggering commit, github.sha)' + required: false + type: string + default: '' pull_request: paths: - 'pm/**' @@ -29,9 +41,11 @@ on: permissions: contents: read +# A workflow_call run (stable release) is never cancelled: its group uses +# github.run_id so a parent rerun cannot kill this child mid-flight. concurrency: - group: pm-bundle-${{ github.ref }} - cancel-in-progress: true + group: pm-bundle-${{ inputs.release == true && github.run_id || github.ref }} + cancel-in-progress: ${{ inputs.release != true }} jobs: bundle: @@ -57,7 +71,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ inputs.ref || github.ref }} + # Default to the exact candidate commit (github.sha); inputs.ref + # remains an escape hatch for a focused manual reproduction. + ref: ${{ inputs.ref || github.sha }} fetch-tags: true # The host uv only bootstraps a python to run pm itself; every diff --git a/.github/workflows/rust-tests.yml b/.github/workflows/rust-tests.yml index 5c91e1a352..f211530fd9 100644 --- a/.github/workflows/rust-tests.yml +++ b/.github/workflows/rust-tests.yml @@ -21,8 +21,8 @@ permissions: contents: read concurrency: - group: rust-tests-${{ github.ref }} - cancel-in-progress: true + group: rust-tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: bootstrap-installer: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml new file mode 100644 index 0000000000..4eb2831ef5 --- /dev/null +++ b/.github/workflows/stable-release.yml @@ -0,0 +1,286 @@ +name: Stable Release +run-name: Stable release ${{ inputs.tag }} + +# Dispatch on the tag so reusable workflows and github.sha identify the same tree. +on: + workflow_dispatch: + inputs: + tag: + description: Exact stable tag, matching the selected workflow ref + required: true + type: string + baseline-manifest: + description: Optional HTTPS manifest of the previous published stable packages + default: '' + type: string + +permissions: + contents: read + +concurrency: + group: stable-release + cancel-in-progress: false + +jobs: + admit: + runs-on: ubuntu-24.04 + outputs: + tag: ${{ steps.admit.outputs.tag }} + commit: ${{ steps.admit.outputs.commit }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - id: admit + run: python -m scripts.releases.stable admit + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + + ci: + name: Full CI pipeline + needs: admit + permissions: + contents: read + pull-requests: write + actions: read + security-events: write + uses: ./.github/workflows/ci.yaml + with: + release: true + + docker: + name: Docker build and tests + needs: [admit, ci] + uses: ./.github/workflows/docker.yml + with: + release-phase: test + tag: ${{ needs.admit.outputs.tag }} + + nix: + needs: [ci, docker] + uses: ./.github/workflows/nix.yml + with: + release: true + + pm-bundle: + needs: [ci, docker] + uses: ./.github/workflows/pm-bundle.yml + with: + release: true + ref: ${{ github.sha }} + + termux-checks: + needs: [ci, docker] + permissions: + contents: read + actions: read + uses: ./.github/workflows/termux-verify.yml + with: + release: true + + windows-live: + needs: [ci, docker] + uses: ./.github/workflows/windows-venv-e2e.yml + with: + release: true + + install-e2e: + name: Install and update E2E + needs: [ci, docker] + permissions: + contents: read + actions: read + uses: ./.github/workflows/install-e2e.yml + with: + release: true + route: all + tag-count: '3' + exclude-ref: ${{ inputs.tag }} + + candidates: + name: Build signed release candidates + needs: [admit, ci, docker] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + release-phase: candidate + + transitions: + name: Pin actual OLD and NEW signed packages + needs: [admit, candidates] + runs-on: ubuntu-24.04 + environment: release-signing + outputs: + windows: ${{ steps.plan.outputs.windows }} + macos: ${{ steps.plan.outputs.macos }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - id: plan + run: python -m scripts.releases.stable transitions + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.admit.outputs.tag }} + CANDIDATE_MANIFEST_URL: ${{ needs.candidates.outputs.manifest-url }} + CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} + BASELINE_MANIFEST_URL: ${{ inputs.baseline-manifest }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} + + windows-packaged: + name: Windows signed-package acceptance + needs: transitions + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.transitions.outputs.windows) }} + uses: ./.github/workflows/install-e2e-windows-run.yml + with: + install-method: packaged-app + update-method: open-app-update + install-ref: ${{ matrix.old }} + leg-id: stable-${{ matrix.id }} + bundle-manifest-url: ${{ matrix.manifest }} + bundle-manifest-sha256: ${{ matrix.manifest_sha256 }} + bundle-arch: ${{ matrix.arch }} + + macos-packaged: + name: macOS signed-package acceptance + needs: transitions + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.transitions.outputs.macos) }} + uses: ./.github/workflows/install-e2e-macos-run.yml + with: + install-method: packaged-app + update-method: open-app-update + install-ref: ${{ matrix.old }} + leg-id: stable-${{ matrix.id }} + bundle-manifest-url: ${{ matrix.manifest }} + bundle-manifest-sha256: ${{ matrix.manifest_sha256 }} + bundle-arch: ${{ matrix.arch }} + + acceptance: + name: All release acceptance checks pass + if: always() + needs: [admit, ci, docker, nix, pm-bundle, termux-checks, windows-live, install-e2e, candidates, transitions, windows-packaged, macos-packaged] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - run: python -m scripts.releases.stable gate admit ci docker nix pm-bundle termux-checks windows-live install-e2e candidates transitions windows-packaged macos-packaged + env: + RELEASE_NEEDS: ${{ toJSON(needs) }} + + publish-docker: + name: Publish tested Docker image + needs: [admit, acceptance] + uses: ./.github/workflows/docker.yml + with: + release-phase: publish + tag: ${{ needs.admit.outputs.tag }} + + publish-bundles: + name: Publish tested bundle artifacts + needs: [admit, acceptance, candidates] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + release-phase: publish + manifest-sha256: ${{ needs.candidates.outputs.manifest-sha256 }} + + publication: + name: All artifact publication succeeded + if: always() + needs: [acceptance, publish-docker, publish-bundles] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - run: python -m scripts.releases.stable gate acceptance publish-docker publish-bundles + env: + RELEASE_NEEDS: ${{ toJSON(needs) }} + + promote-docker: + name: Advance stable Docker channel + needs: [admit, publication] + uses: ./.github/workflows/docker.yml + with: + release-phase: promote + tag: ${{ needs.admit.outputs.tag }} + + promote-bundles: + name: Advance stable bundle channels + needs: [admit, publication, candidates] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + release-phase: promote + manifest-sha256: ${{ needs.candidates.outputs.manifest-sha256 }} + + complete: + name: Stable release is green + if: always() + needs: [admit, ci, docker, acceptance, candidates, publication, promote-docker, promote-bundles] + runs-on: ubuntu-24.04 + environment: release-signing + permissions: + contents: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - run: python -m scripts.releases.stable gate admit ci docker acceptance candidates publication promote-docker promote-bundles + env: + RELEASE_NEEDS: ${{ toJSON(needs) }} + - name: Record accepted stable packages and publish release + run: python -m scripts.releases.stable complete + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_MANIFEST_URL: ${{ needs.candidates.outputs.manifest-url }} + CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} diff --git a/.github/workflows/termux-verify.yml b/.github/workflows/termux-verify.yml index 6830ba0bb8..2ef7d88fd2 100644 --- a/.github/workflows/termux-verify.yml +++ b/.github/workflows/termux-verify.yml @@ -9,14 +9,23 @@ on: - 'tests/test_termux*' - '.github/workflows/termux-verify.yml' workflow_dispatch: + workflow_call: + inputs: + release: + description: 'Stable-release candidate run (ignored by the jobs; uniform callable surface).' + required: false + type: boolean + default: false permissions: contents: read actions: read +# A workflow_call run (stable release) is never cancelled: its group uses +# github.run_id so a parent rerun cannot kill this child mid-flight. concurrency: - group: termux-verify-${{ github.ref }} - cancel-in-progress: true + group: termux-verify-${{ inputs.release == true && github.run_id || github.ref }} + cancel-in-progress: ${{ inputs.release != true }} jobs: contracts: @@ -26,6 +35,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + # github.sha is the exact candidate commit on a stable tag-dispatched + # caller and the head SHA on every other event. + ref: ${{ github.sha }} persist-credentials: false - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: @@ -54,6 +66,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + # github.sha is the exact candidate commit on a stable tag-dispatched + # caller and the head SHA on every other event. + ref: ${{ github.sha }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -73,6 +88,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: + # github.sha is the exact candidate commit on a stable tag-dispatched + # caller and the head SHA on every other event. + ref: ${{ github.sha }} persist-credentials: false - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: diff --git a/.github/workflows/tests-os.yml b/.github/workflows/tests-os.yml index 0dd8812ffa..9004ea29ce 100644 --- a/.github/workflows/tests-os.yml +++ b/.github/workflows/tests-os.yml @@ -46,8 +46,8 @@ permissions: contents: read concurrency: - group: tests-os-${{ github.ref }} - cancel-in-progress: true + group: tests-os-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: os-tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a0dca54ad0..71879519b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,8 +8,8 @@ permissions: # Cancel in-progress runs for the same ref concurrency: - group: tests-${{ github.ref }} - cancel-in-progress: true + group: tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: test: diff --git a/.github/workflows/uv-lockfile-check.yml b/.github/workflows/uv-lockfile-check.yml index d258ad731d..e2c1487aab 100644 --- a/.github/workflows/uv-lockfile-check.yml +++ b/.github/workflows/uv-lockfile-check.yml @@ -54,8 +54,8 @@ permissions: contents: read concurrency: - group: uv-lockfile-check-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: uv-lockfile-check-${{ github.ref_type == 'tag' && github.run_id || github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: check: diff --git a/.github/workflows/windows-venv-e2e.yml b/.github/workflows/windows-venv-e2e.yml index a09b2f2d45..867a6721b2 100644 --- a/.github/workflows/windows-venv-e2e.yml +++ b/.github/workflows/windows-venv-e2e.yml @@ -16,13 +16,24 @@ on: push: branches: - "wine2e/**" + workflow_call: + inputs: + release: + description: 'Stable-release candidate run. Unused by the jobs; the caller passes it for a uniform callable surface.' + required: false + type: boolean + default: false permissions: contents: read +# A workflow_call run (stable release) shares the caller's commit; its group +# includes github.run_id so a rerun of the parent gets a fresh group instead +# of cancelling this child mid-flight. wine2e push runs keep ref-scoped +# collapse as before. concurrency: - group: windows-venv-e2e-${{ github.ref }} - cancel-in-progress: true + group: windows-venv-e2e-${{ inputs.release == true && github.run_id || github.ref }} + cancel-in-progress: ${{ inputs.release != true }} jobs: venv-holder-e2e: @@ -32,6 +43,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # The stable caller dispatches on the candidate tag; github.sha is + # the exact candidate commit there and the head SHA everywhere else. + ref: ${{ github.sha }} - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 @@ -53,28 +68,27 @@ jobs: command: uv sync --locked --python 3.11 --extra dev --extra messaging - name: Run venv-holder live E2E + # Canonical runner (scripts/run_tests.sh) — never bare pytest: it + # enforces credential hygiene, TZ, temp HERMES_HOME and subprocess + # isolation. The live files are host-marked windows-only, so the + # runner collects them on this runner instead of skipping them. shell: bash run: | set -uo pipefail - uv run --no-sync python -m pytest \ + bash scripts/run_tests.sh \ tests/hermes_cli/test_venv_holder_windows_live.py \ tests/hermes_cli/test_taskkill_identity_windows_live.py \ tests/hermes_cli/test_git_trampoline_windows_live.py \ - "tests/hermes_cli/test_managed_uv.py::TestWindowsRuntimeSelfLock" \ - -o addopts= -v -p no:cacheprovider + tests/hermes_cli/test_runtime_repair.py::TestWindowsRuntimeSelfLock - name: Run Telegram CLOSE-WAIT reconnect live E2E (#87057) shell: bash run: | set -uo pipefail - uv run --no-sync python -m pytest \ - tests/gateway/test_telegram_closewait_windows_live.py \ - -o addopts= -v -p no:cacheprovider + bash scripts/run_tests.sh tests/gateway/test_telegram_closewait_windows_live.py - name: Run background-executor spawn parity live E2E (#70716) shell: bash run: | set -uo pipefail - uv run --no-sync python -m pytest \ - tests/tools/test_process_registry_windows_live.py \ - -o addopts= -v -p no:cacheprovider + bash scripts/run_tests.sh tests/tools/test_process_registry_windows_live.py diff --git a/apps/desktop/update-feed.cjs b/apps/desktop/update-feed.cjs index ae1b6db014..be1e0b160c 100644 --- a/apps/desktop/update-feed.cjs +++ b/apps/desktop/update-feed.cjs @@ -1,10 +1,7 @@ 'use strict' -// apps/desktop/update-feed.cjs — the ONE source of truth for the Darwin -// (macOS) electron-updater feed layout, shared by the desktop runtime -// (generic provider base URL) and the release pipeline -// (scripts/r2-release.mjs finalize). Pure CJS: requireable from both the -// app bundle and ESM scripts via createRequire. No dependencies. +// Read the feed facts shared by the desktop runtime and Python publisher. +// update-feed.json owns the paths; neither consumer derives a second copy. // // darwinFeed('stable') → { directory: 'releases/darwin/stable', // channel: 'stable', @@ -19,18 +16,13 @@ // '/' + feed.fileName; the producer publishes the manifest at exactly that // key. There is no placeholder default URL — the caller supplies the base. -const CHANNELS = ['stable', 'canary'] +const feeds = require('./update-feed.json') function darwinFeed(channel, light = false) { - if (!CHANNELS.includes(channel)) { + if (!Object.hasOwn(feeds, channel)) { throw new TypeError(`darwinFeed: unknown channel ${JSON.stringify(channel)} (expected stable|canary)`) } - return { - directory: light ? `releases/darwin/light/${channel}` : `releases/darwin/${channel}`, - channel, - fileName: `${channel}-mac.yml`, - allowPrerelease: channel === 'canary', - } + return { ...feeds[channel][light ? 'light' : 'bundled'] } } module.exports = { darwinFeed } diff --git a/apps/desktop/update-feed.json b/apps/desktop/update-feed.json new file mode 100644 index 0000000000..4e6bf53eb8 --- /dev/null +++ b/apps/desktop/update-feed.json @@ -0,0 +1,10 @@ +{ + "stable": { + "bundled": {"directory": "releases/darwin/stable", "channel": "stable", "fileName": "stable-mac.yml", "allowPrerelease": false}, + "light": {"directory": "releases/darwin/light/stable", "channel": "stable", "fileName": "stable-mac.yml", "allowPrerelease": false} + }, + "canary": { + "bundled": {"directory": "releases/darwin/canary", "channel": "canary", "fileName": "canary-mac.yml", "allowPrerelease": true}, + "light": {"directory": "releases/darwin/light/canary", "channel": "canary", "fileName": "canary-mac.yml", "allowPrerelease": true} + } +} diff --git a/docs/macos-bundle-updates.md b/docs/macos-bundle-updates.md index 0e2511ecd9..0627e72a83 100644 --- a/docs/macos-bundle-updates.md +++ b/docs/macos-bundle-updates.md @@ -6,7 +6,8 @@ Windows App Installer and Store ownership are unchanged. ## Feed contract -`apps/desktop/update-feed.cjs` defines each channel directory and filename. +`apps/desktop/update-feed.json` defines each channel directory and filename. +The desktop CJS adapter and Python publisher read these same facts. The builder writes that URL into `app-update.yml`. The client uses this file unless `updates.desktop_feed_base_url` supplies an explicit bucket-base URL. @@ -18,7 +19,7 @@ unless `updates.desktop_feed_base_url` supplies an explicit bucket-base URL. The current workflow builds the bundled variant, on ARM64 and Intel runners. Light has separate client/feed routing but no release matrix leg in this change. -`r2-release.mjs finalize` requires one metadata file for each architecture, +`python -m scripts.releases.r2 finalize` requires one metadata file for each architecture, named `arm64-CHANNEL-mac.yml` and `x64-CHANNEL-mac.yml`. It rejects wrong versions, variants, architectures, hashes and inconsistent legacy path fields. Each referenced ZIP/DMG is streamed back and checked against its SHA-512 and diff --git a/docs/stable-releases.md b/docs/stable-releases.md new file mode 100644 index 0000000000..e911753713 --- /dev/null +++ b/docs/stable-releases.md @@ -0,0 +1,85 @@ +# Stable release admission and promotion + +`Stable Release` is the release gate. A successful desktop builder alone is +not a successful stable release. Canary builds retain their separate workflow. + +## Order + +1. Admit an exact `vMAJOR.MINOR.PATCH` tag and non-prerelease draft. +2. Run the whole `ci.yaml` pipeline in release mode. +3. Build and test the same Docker images as the Docker workflow. +4. Run Nix, native PM bundles, install/update E2E, Termux and Windows live + process tests. Build, sign and notarize the candidate packages. +5. Test upgrades to the actual signed Windows/macOS packages on both + architectures. The baseline must be a published stable release. +6. Publish the tested Docker and bundle artifacts. Do not rebuild for publication. +7. After every required check and artifact publication succeeds, advance the + Docker, App Installer, macOS and APT stable channels. +8. Verify promotion, record the accepted package manifest, then publish the + GitHub release. Only this final job reports the stable release green. + +Versioned candidate uploads are staging, not stable promotion. Failed, +cancelled, missing and unexpectedly skipped requirements block the gate. +Public channel updates never run merely because one architecture built. + +Docker Hub, R2, APT and the Store do not support one cross-service transaction. +All prerequisites finish before promotion starts, but a failure during final +promotion can leave some services advanced and others unchanged. Such a run +stays red. Inspect its per-service results before retrying; do not rebuild or +replace the tested candidate to recover a pointer update. + +## Run a release + +Use `scripts/release.py --bump patch --publish --remote ` to create +its version commit, stable tag and draft. Stable dispatch selects +`stable-release.yml` at that tag. The workflow rejects a different ref/SHA, +a moved tag, a tag outside repository main, or a project-version mismatch. +Canary dispatch still uses the default branch for its workflow/cache scope. + +To resume an existing draft, dispatch `Stable Release` with the exact tag as +both the workflow ref and the `tag` input. Keep tags immutable. The local +workflow calls and their checkouts use the tag's commit, not moving main. + +## Signed-package baseline + +The last successful stable release records +`releases/stable/release-candidates.json` on the configured R2 public origin. +It identifies actual Windows universal MSIX bundles, macOS ZIPs and package +provenance. The next run combines those records with its candidate manifest +and uses the existing native bundled-update drivers. + +For an existing stable release that predates this metadata, supply +`baseline-manifest` as an HTTPS URL to an equivalent manifest of its actual +published packages. The baseline tag must be a published stable release, +package identities must agree, and versions must increase. Missing baseline +artifacts are a blocker, not permission to fabricate or skip acceptance. +See [the bundled update contract](../tests/install/BUNDLED_UPDATES.md). + +## Explicit exclusions and policy + +- Desktop Playwright E2E (`e2e-desktop.yml`) is deferred at the owner's request + because it is flaky. It is reported as deferred, not passed. Stabilize it + and prove repeatable CI runs before adding it to this gate. +- Install/update E2E and native signed-package acceptance are **not** deferred. +- PR-only history, label and diff review checks do not apply to a stable tag. + All applicable source CI jobs still run, regardless of changed paths. +- OSV vulnerability findings retain their existing advisory policy. Required + scanner execution failures are failures, not advisory findings. +- Disabled Linux desktop packaging is not claimed as shipped. Native Linux + PM bundles, Docker, Nix and install/update checks remain required. +- Housekeeping, autofix, comment and skills-index/deploy workflows are not + release acceptance suites. + +## Implementation ownership + +Actions owns job ordering, runner selection, permissions and environments. +Python owns shared release admission, manifests, artifact hashes, publication +and channel promotion under `scripts/releases/` and `scripts/bundles/`. +Electron-builder configuration/hooks and native Windows/macOS adapters remain +in JavaScript or PowerShell. These adapters consume release facts rather than +reimplementing the release gate. Gate jobs use only Python's standard library; +they do not install the application or the JS workspace to report a verdict. + +Signing and publication credentials stay in their protected job environments. +The source CI call does not inherit deployment secrets. Configure the existing +release-signing and container-publish environments before running this pipeline. diff --git a/scripts/bundles/release_artifacts.py b/scripts/bundles/release_artifacts.py new file mode 100644 index 0000000000..be40a832d2 --- /dev/null +++ b/scripts/bundles/release_artifacts.py @@ -0,0 +1,266 @@ +"""Record, stage and promote the exact native release artifacts.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import plistlib +import re +import subprocess +import tarfile +import tempfile +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +from scripts.releases.stable import validate_candidates + + +def sha256_file(file: Path) -> str: + with file.open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def stamp_matches(stamp: dict, tag: str, commit: str) -> None: + if stamp.get("commit") != commit or stamp.get("tag") != tag: + raise ValueError("Built package provenance does not match the release") + + +def single(items): + items = list(items) + if len(items) != 1: + raise ValueError(f"Expected exactly one artifact, found {items}") + return items[0] + + +def record(platform: str, arch: str, root: Path, tag: str, commit: str, out: Path) -> None: + """Read identities from the built packages, never from the workflow matrix.""" + row = {"platform": platform, "arch": arch, "tag": tag, "commit": commit} + if platform == "windows": + package = single(p for p in root.glob(f"*-win-{arch}.msix") if not p.name.startswith("Store-")) + with zipfile.ZipFile(package) as archive: + manifest = ET.fromstring(archive.read("AppxManifest.xml")) + identity = manifest.find("{*}Identity") + application = single(manifest.findall("{*}Applications/{*}Application")) + stamp_name = single(n for n in archive.namelist() if n.replace("\\", "/").endswith("/resources/install-stamp.json")) + stamp_matches(json.loads(archive.read(stamp_name)), tag, commit) + if identity.attrib["ProcessorArchitecture"].lower() != arch: + raise ValueError("MSIX architecture differs from release target") + row.update(identity=identity.attrib["Name"], publisher=identity.attrib["Publisher"], + applicationId=application.attrib["Id"], version=identity.attrib["Version"]) + files = list(root.glob(f"*-win-{arch}.msix")) + elif platform == "macos": + package = single(root.glob(f"*-mac-{arch}.zip")) + app = single(root.glob("mac*/*.app")) + subprocess.run(["codesign", "--verify", "--strict", str(app)], check=True) + signature = subprocess.run(["codesign", "-dv", "--verbose=4", str(app)], check=True, capture_output=True, text=True, encoding="utf-8") + team = re.search(r"^TeamIdentifier=([A-Z0-9]{10})$", signature.stderr, re.M) + if not team: + raise ValueError("Signed app has no Developer ID team") + with zipfile.ZipFile(package) as archive: + info = plistlib.loads(archive.read(single(n for n in archive.namelist() if re.fullmatch(r"[^/]+\.app/Contents/Info.plist", n)))) + stamp = json.loads(archive.read(single(n for n in archive.namelist() if re.fullmatch(r"[^/]+\.app/Contents/Resources/install-stamp.json", n)))) + stamp_matches(stamp, tag, commit) + row.update(identity=info["CFBundleIdentifier"], teamId=team.group(1), version=info["CFBundleShortVersionString"], filename=package.name) + if row["version"] != tag[1:]: + raise ValueError("App version differs from release tag") + files = [p for p in root.iterdir() if p.is_file() and (f"-mac-{arch}." in p.name or p.name == f"{arch}-stable-mac.yml")] + elif platform == "termux": + package = single((root / "deb").glob("*.deb")) + fields = subprocess.check_output(["dpkg-deb", "--field", str(package), "Package", "Version", "Architecture"], text=True, encoding="utf-8") + parsed = dict(line.split(": ", 1) for line in fields.splitlines()) + row.update(identity=parsed["Package"], version=parsed["Version"], filename=package.name) + if parsed["Architecture"] != "aarch64": + raise ValueError("Wrong Termux package architecture") + with tempfile.TemporaryDirectory() as temp: + subprocess.run(["dpkg-deb", "--extract", str(package), temp], check=True) + stamps = list(Path(temp).rglob("install-stamp.json")) + if not any((data := json.loads(p.read_text(encoding="utf-8"))).get("commit") == commit and data.get("tag") == tag for p in stamps): + raise ValueError("Termux package has no matching provenance") + files = [package] + else: + raise ValueError("Unknown platform") + out.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(out, "w") as archive: + for file in files: + archive.add(file, arcname=file.name) + if platform == "termux": + archive.add(root / "apt", arcname="apt") + with tempfile.TemporaryDirectory() as temp: + metadata = Path(temp) / f"metadata-{platform}-{arch}.json" + metadata.write_text(json.dumps(row), encoding="utf-8") + archive.add(metadata, arcname=metadata.name) + + +def unpack(directory: Path, target: Path) -> None: + target.mkdir(parents=True, exist_ok=True) + for file in directory.rglob("*.tar"): + with tarfile.open(file) as archive: + archive.extractall(target, filter="data") + + +def assemble(directory: Path, bundle: Path, tag: str, commit: str, public_base: str, out: Path) -> dict: + """Collect native outputs and retain hashes of every file publication will use.""" + from scripts.releases.r2 import put + + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + unpack(directory, root) + rows = [json.loads(p.read_text(encoding="utf-8")) for p in root.glob("metadata-*.json")] + for item in bundle.iterdir(): + if item.is_file() and item.suffix == ".msixbundle": + import shutil + shutil.copy2(item, root / item.name) + universal = single(p for p in root.glob("*.msixbundle") if not p.name.startswith("Store-")) + windows = [r for r in rows if r["platform"] == "windows"] + if sorted(r["arch"] for r in windows) != ["arm64", "x64"]: + raise ValueError("Windows metadata must include both architectures") + for field in ("identity", "publisher", "version", "applicationId"): + if len({r[field] for r in windows}) != 1: + raise ValueError(f"Windows packages disagree on {field}") + with zipfile.ZipFile(universal) as archive: + manifest = ET.fromstring(archive.read("AppxMetadata/AppxBundleManifest.xml")) + identity = manifest.find("{*}Identity") + for attr, field in (("Name", "identity"), ("Publisher", "publisher"), ("Version", "version")): + if identity.attrib[attr] != windows[0][field]: + raise ValueError("Universal bundle identity does not match its packages") + files = [] + for file in sorted(root.rglob("*")): + if not file.is_file() or file.name.startswith("metadata-"): + continue + relative = file.relative_to(root).as_posix() + key = f"releases/tag/{tag}/{relative}" + put(tag=tag, key=key, file=file, key_is_full=True, immutable=True) + files.append({"path": relative, "sha256": sha256_file(file), "url": f"{public_base.rstrip('/')}/{key}"}) + by_name = {item["path"]: item for item in files} + packages = [] + for row in rows: + stamp_matches(row, tag, commit) + filename = universal.name if row["platform"] == "windows" else row["filename"] + item = by_name[filename] + packages.append({k: v for k, v in {**row, "artifact": {"url": item["url"], "sha256": item["sha256"]}}.items() if k != "filename"}) + result = {"schema": 1, "tag": tag, "commit": commit, "packages": packages, "files": files} + validate_candidates(result, tag, commit, public_base) + if not any(row["platform"] == "termux" for row in packages): + raise ValueError("Missing Termux candidate") + out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + put(tag=tag, key="release-candidates.json", file=out, immutable=True) + return result + + +def materialize(manifest: dict, root: Path, *, public_base: str, store_only: bool = False) -> None: + validate_candidates(manifest, manifest["tag"], manifest["commit"], public_base) + files = manifest.get("files", []) + if not files or len({item["path"] for item in files}) != len(files): + raise ValueError("Missing or duplicate candidate file receipts") + by_url = {item["url"]: item["sha256"] for item in files} + if any(by_url.get(row["artifact"]["url"]) != row["artifact"]["sha256"] for row in manifest["packages"]): + raise ValueError("Package receipts differ from candidate file receipts") + selected = [item for item in files if item["path"].startswith("Store-") and item["path"].endswith(".msixbundle")] if store_only else files + if store_only and len(selected) != 1: + raise ValueError("Expected one Store candidate") + for item in selected: + relative = Path(item["path"]) + if relative.is_absolute() or ".." in relative.parts or any(c in item["path"] for c in "\\:%?#") or item["path"].startswith("/") or "//" in item["path"]: + raise ValueError("Invalid artifact path") + expected = f"{public_base.rstrip('/')}/releases/tag/{manifest['tag']}/{relative.as_posix()}" + if item["url"] != expected or not re.fullmatch(r"[a-f0-9]{64}", item["sha256"]): + raise ValueError("Invalid artifact URL or digest") + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + with urllib.request.urlopen(item["url"], timeout=600) as response, target.open("wb") as file: + if not response.geturl().startswith("https://"): + raise ValueError("Artifact redirected outside HTTPS") + while chunk := response.read(1024 * 1024): + digest.update(chunk) + file.write(chunk) + if digest.hexdigest() != item["sha256"]: + raise ValueError(f"Artifact digest mismatch: {relative}") + if not manifest.get("files"): + raise ValueError("Missing candidate file receipts") + + +def publish(manifest: dict, root: Path, public_base: str) -> None: + """Verify versioned uploads and stage APT content-addressed files, not indexes.""" + from scripts.releases.r2 import put + + materialize(manifest, root, public_base=public_base) + for file in (root / "apt").rglob("*"): + rel = file.relative_to(root / "apt").as_posix() + if file.is_file() and (rel.startswith("pool/") or "/by-hash/" in rel): + put(tag=manifest["tag"], key=f"releases/termux/stable/{rel}", file=file, key_is_full=True, immutable=True) + + +def promote(manifest: dict, root: Path, public_base: str) -> None: + from scripts.releases.r2 import put, finalize + + materialize(manifest, root, public_base=public_base) + windows = next(r for r in manifest["packages"] if r["platform"] == "windows") + uri = f"{public_base.rstrip('/')}/releases/win32/stable/stable.appinstaller" + ns = "http://schemas.microsoft.com/appx/appinstaller/2017/2" + ET.register_namespace("", ns) + descriptor = ET.Element(f"{{{ns}}}AppInstaller", {"Uri": uri, "Version": windows["version"]}) + ET.SubElement(descriptor, f"{{{ns}}}MainBundle", {"Name": windows["identity"], "Publisher": windows["publisher"], "Version": windows["version"], "Uri": windows["artifact"]["url"]}) + settings = ET.SubElement(descriptor, f"{{{ns}}}UpdateSettings") + ET.SubElement(settings, f"{{{ns}}}OnLaunch", {"HoursBetweenUpdateChecks": "12"}) + appinstaller = root / "stable.appinstaller" + ET.ElementTree(descriptor).write(appinstaller, encoding="utf-8", xml_declaration=True) + apt = root / "apt" + indexes = sorted(p for p in apt.rglob("*") if p.is_file() and not p.relative_to(apt).as_posix().startswith("pool/") and "/by-hash/" not in p.relative_to(apt).as_posix()) + indexes.sort(key=lambda p: p.name == "InRelease") + if not any(p.name == "InRelease" for p in indexes): + raise ValueError("Missing signed APT index") + # Every package and index must exist before the first channel write. + finalize(tag=manifest["tag"], dir=root) + def publish_pointer(key, file): + put(tag=manifest["tag"], key=key, file=file, key_is_full=True) + digest = hashlib.sha256() + with urllib.request.urlopen(f"{public_base.rstrip('/')}/{key}", timeout=60) as response: + while chunk := response.read(1024 * 1024): + digest.update(chunk) + if digest.hexdigest() != sha256_file(file): + raise ValueError(f"Channel read-back differs: {key}") + + publish_pointer("releases/win32/stable/stable.appinstaller", appinstaller) + for file in indexes: + publish_pointer(f"releases/termux/stable/{file.relative_to(apt).as_posix()}", file) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["record", "assemble", "publish", "promote", "materialize"]) + parser.add_argument("--platform", choices=["windows", "macos", "termux"]) + parser.add_argument("--arch") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--bundle-dir", type=Path) + parser.add_argument("--out", type=Path) + parser.add_argument("--tag", default=os.environ.get("RELEASE_TAG")) + parser.add_argument("--commit", default=os.environ.get("GITHUB_SHA")) + parser.add_argument("--public-base", default=os.environ.get("CLOUDFLARE_R2_PUBLIC_URL")) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--store-only", action="store_true") + args = parser.parse_args() + if args.command == "record": + record(args.platform, args.arch, args.root, args.tag, args.commit, args.out) + elif args.command == "assemble": + assemble(args.root, args.bundle_dir, args.tag, args.commit, args.public_base, args.out) + if os.environ.get("GITHUB_OUTPUT"): + with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as file: + file.write(f"manifest-url={args.public_base.rstrip('/')}/releases/tag/{args.tag}/release-candidates.json\nmanifest-sha256={sha256_file(args.out)}\n") + else: + expected_digest = os.environ.get("CANDIDATE_MANIFEST_SHA256", "") + if not re.fullmatch(r"[a-f0-9]{64}", expected_digest) or sha256_file(args.manifest) != expected_digest: + raise ValueError("Candidate manifest differs from accepted candidate") + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + validate_candidates(manifest, args.tag, args.commit, args.public_base) + if args.command == "materialize": + materialize(manifest, args.root, public_base=args.public_base, store_only=args.store_only) + else: + {"publish": publish, "promote": promote}[args.command](manifest, args.root, args.public_base) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci/required_results.py b/scripts/ci/required_results.py new file mode 100644 index 0000000000..8c6877dfd5 --- /dev/null +++ b/scripts/ci/required_results.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Strict CI aggregate gate (shared by ci.yaml and the stable orchestrator). + +Behavior tests: tests/ci/test_required_results.py + +Input: the JSON ``toJSON(needs)`` of the all-checks-pass job on stdin. +Any non-``success`` result fails the gate. ``skipped`` additionally fails in +release mode unless the job is in :data:`EXCLUDED_JOBS` (PR-only jobs that +cannot run on a tag event, plus the deferred Desktop E2E). The OSV scan is +advisory in its findings only — its execution is required. + + echo "$NEEDS" | python3 scripts/ci/required_results.py [--release] +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any + +# Jobs that can never run on a release (push/tag) event, plus the deferred +# Desktop E2E. These are the only skips a strict run tolerates. +PR_ONLY_JOBS = ("history-check", "lockfile-diff", "supply-chain", "review-labels") +DEFERRED_JOBS = ("e2e-desktop",) +EXCLUDED_JOBS = frozenset((*PR_ONLY_JOBS, *DEFERRED_JOBS)) + +NEEDS_JSON_OUTPUT = "needs-json" + + +def evaluate_gate( + needs: dict[str, dict[str, Any]] | None, + release: bool = False, +) -> dict[str, Any]: + """Verdict for a ``needs`` context; see the module docstring. + + Returns ``ok``, plus sorted ``failed`` (every non-success, non-allowed + entry) and ``allowed_skips``. + """ + failed: list[str] = [] + allowed_skips: list[str] = [] + entries = needs or {} + if not entries: + failed.append("") + for name, info in entries.items(): + result = (info or {}).get("result") + if result == "success": + continue + if result == "skipped" and (not release or name in EXCLUDED_JOBS): + allowed_skips.append(name) + continue + failed.append(name) + return { + "ok": not failed, + "failed": sorted(failed), + "allowed_skips": sorted(allowed_skips), + } + + +def compact_results(needs: dict[str, dict[str, Any]] | None) -> dict[str, str]: + """{job_name: result} for every entry, for the PR comment assembler.""" + return {name: (info or {}).get("result", "") for name, info in (needs or {}).items()} + + +def render_report(needs: dict[str, dict[str, Any]] | None, verdict: dict[str, Any]) -> list[str]: + """Per-job lines plus ::error:: annotations for the failed set.""" + lines: list[str] = [] + for name in sorted(needs or {}): + result = (needs[name] or {}).get("result", "unknown") + icon = {"success": "✅", "skipped": "⏭️"}.get(result, "❌") + lines.append(f"{icon} {name}: {result}") + if verdict["failed"]: + failed = verdict["failed"] + lines.append(f"::error::{len(failed)} job(s) failed: {', '.join(failed)}") + return lines + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--release", + action="store_true", + help="Strict mode: a skipped excluded job is the only tolerated skip.", + ) + args = parser.parse_args(argv) + + needs = json.load(sys.stdin) + verdict = evaluate_gate(needs, release=args.release) + compact = compact_results(needs) + + output_file = os.environ.get("GITHUB_OUTPUT") + if output_file: + with open(output_file, "a", encoding="utf-8") as fh: + fh.write(f"{NEEDS_JSON_OUTPUT}={json.dumps(compact)}\n") + + print(f"{NEEDS_JSON_OUTPUT}={json.dumps(compact)}") + for line in render_report(needs, verdict): + print(line) + if verdict["ok"]: + print("All checks passed" if args.release else "All checks passed (or were skipped)") + return 0 if verdict["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/darwin-feed.mjs b/scripts/darwin-feed.mjs deleted file mode 100644 index c8e2597e76..0000000000 --- a/scripts/darwin-feed.mjs +++ /dev/null @@ -1,90 +0,0 @@ -import { isDeepStrictEqual } from 'node:util' -import yaml from 'js-yaml' -import semver from 'semver' -import feedContract from '../apps/desktop/update-feed.cjs' - -const { darwinFeed } = feedContract -const arches = ['arm64', 'x64'] -const hashPattern = /^[A-Za-z0-9+/]{86}==$/ - -export function parseMacFeed(text) { - const feed = yaml.load(text) - if (!feed || typeof feed !== 'object' || !semver.valid(feed.version) || !Array.isArray(feed.files) || !feed.files.length) { - throw new Error('Invalid macOS update feed') - } - for (const file of feed.files) { - if (typeof file.url !== 'string' || !hashPattern.test(file.sha512) || !Number.isSafeInteger(file.size) || file.size <= 0) { - throw new Error('Invalid macOS artifact metadata') - } - } - if (feed.path && !feed.files.some(file => file.url === feed.path && file.sha512 === feed.sha512)) { - throw new Error('Legacy feed path/hash disagrees with files') - } - return feed -} - -export function macFeedReferences(text) { - const feed = parseMacFeed(text) - const references = [] - for (const file of feed.files) { - // Only our immutable artifact namespace is eligible for publication or pruning. - if (!/^\/releases\/tag\/v[0-9A-Za-z.+-]+\/[0-9A-Za-z._+-]+\.(zip|dmg)$/.test(file.url)) { - throw new Error(`Invalid macOS artifact path: ${file.url}`) - } - const key = file.url.slice(1) - references.push(key, `${key}.blockmap`) - } - return references -} - -export function mergeMacFeeds(legs, tag, light = false) { - const version = tag?.startsWith('v') ? tag.slice(1) : '' - if (!semver.valid(version) || !/^v\d+\.\d+\.\d+(?:-canary\.\d{14})?$/.test(tag)) { - throw new Error('Invalid macOS release tag') - } - const selection = darwinFeed(version.includes('-canary.') ? 'canary' : 'stable', light) - const expected = arches.map(arch => `${arch}-${selection.fileName}`) - if (!isDeepStrictEqual(Object.keys(legs).sort(), [...expected].sort())) { - throw new Error('Expected exactly one ARM64 and one x64 macOS feed') - } - const files = new Map() - let first - for (const [index, name] of expected.entries()) { - const leg = parseMacFeed(legs[name]) - if (leg.version !== version) { throw new Error(`Feed version does not match ${tag}`) } - const prefix = `${light ? 'HermesLight' : 'HermesBundled'}-${version}-mac-${arches[index]}` - if (!leg.files.some(file => file.url === `${prefix}.zip`)) { throw new Error(`Missing native ZIP for ${arches[index]}`) } - for (const file of leg.files) { - if (![`${prefix}.zip`, `${prefix}.dmg`].includes(file.url)) { throw new Error(`Wrong variant or architecture: ${file.url}`) } - const prior = files.get(file.url) - const rewritten = { ...file, url: `/releases/tag/${tag}/${file.url}` } - if (prior && !isDeepStrictEqual(prior, rewritten)) { throw new Error(`Conflicting artifact: ${file.url}`) } - files.set(file.url, rewritten) - } - first ??= leg - } - const merged = { ...first, files: [...files.values()] } - if (merged.path) { merged.path = `/releases/tag/${tag}/${merged.path}` } - const text = yaml.dump(merged, { lineWidth: -1, noRefs: true }) - macFeedReferences(text) - return { key: `${selection.directory}/${selection.fileName}`, text, files: merged.files, version } -} - -/** The transport verifies immutable bytes and conditionally replaces one pointer. */ -export async function publishMacFeed(plan, transport) { - const live = await transport.read(plan.key) - if (live) { - const oldFeed = parseMacFeed(live.text) - const nextFeed = parseMacFeed(plan.text) - const order = semver.compare(plan.version, oldFeed.version) - if (order < 0) { throw new Error('Refusing to move the macOS feed backward') } - if (order === 0) { - if (!isDeepStrictEqual(oldFeed, nextFeed)) { throw new Error('Refusing to replace published version with different artifacts') } - return - } - } - for (const file of plan.files) { await transport.verify(file.url.slice(1), file) } - await transport.write(plan.key, plan.text, live?.etag ?? null) - const published = await transport.read(plan.key) - if (!published || published.text !== plan.text) { throw new Error('macOS feed readback differs from publication') } -} diff --git a/scripts/msix-shared.mjs b/scripts/msix-shared.mjs index e21fd24760..7c4260c950 100644 --- a/scripts/msix-shared.mjs +++ b/scripts/msix-shared.mjs @@ -25,21 +25,7 @@ export const OUT_OF_STORE_PUBLISHER = // Installer (it would download as octet-stream instead). Everything else // stays octet-stream (R2's default) unchanged. Keys match by filename suffix, // case-insensitively. -const CONTENT_TYPES = { - '.appinstaller': 'application/appinstaller', - '.msixbundle': 'application/msixbundle', - '.msix': 'application/msix', - // Termux APT repo artifacts (uploaded under releases/termux//). - // InRelease/Release/Packages are extensionless; match by exact basename - // too so apt gets text/plain instead of octet-stream. - '.deb': 'application/vnd.debian.binary-package', - '.gz': 'application/gzip', - '.asc': 'text/plain', - 'release.gpg': 'application/pgp-signature', - inrelease: 'text/plain', - release: 'text/plain', - packages: 'text/plain' -} +const CONTENT_TYPES = require('./release-content-types.json') /** * The Content-Type to store for a staged release artifact, if any. diff --git a/scripts/r2-release.mjs b/scripts/r2-release.mjs deleted file mode 100644 index 2dbcb5e075..0000000000 --- a/scripts/r2-release.mjs +++ /dev/null @@ -1,643 +0,0 @@ -#!/usr/bin/env node -// scripts/r2-release.mjs — zero-dependency Cloudflare R2 (S3 API) client for -// release artifacts. Runs on every release runner (node is guaranteed there -// via pm/lock.json; the notes + prune jobs need no npm ci), so the only -// dependency is node's own fetch + crypto. SigV4 signed exactly per the -// botocore-generated vectors in tests-js/r2-release.test.mjs. -// -// node scripts/r2-release.mjs put --tag vX.Y.Z --key --file -// node scripts/r2-release.mjs finalize --tag vX.Y.Z --dir -// node scripts/r2-release.mjs list [--prefix

] -// node scripts/r2-release.mjs prune-canaries --keep-days 14 [--dry-run] -// -// Env (all required except where noted): -// CLOUDFLARE_R2_ACCOUNT_ID → S3 endpoint https://.r2.cloudflarestorage.com -// CLOUDFLARE_R2_ACCESS_KEY_ID R2 API token (S3-compatible) -// CLOUDFLARE_R2_SECRET_ACCESS_KEY -// CLOUDFLARE_R2_BUCKET -// -// Bucket layout (matches app-updater.ts's D1-settled arms): -// releases/tag// immutable per-release staging/archive -// releases/win32//.appinstaller App Installer feed -// releases/win32//*.msixbundle (produced by the -// publish-win32-updater job) -// releases/darwin//-mac.yml electron-updater feed -// dmg/zip/blockmap artifacts stay in releases/tag//. -// where is stable | canary (from the tag: -canary. → canary). -// The publish-win32-updater job merges the win32 legs' staging into the -// win32 feed; r2 finalize publishes the validated Darwin channel feed. - -import { createHash, createHmac } from 'node:crypto' -import fs from 'node:fs' -import path from 'node:path' -import { fileURLToPath } from 'node:url' - -import { contentTypeFor } from './msix-shared.mjs' - -// Re-exported for the r2 test (the Content-Type mapping is shared with the -// stage job; msix-shared.mjs is the single source). -export { contentTypeFor } from './msix-shared.mjs' - -const REGION = 'auto' -const SERVICE = 's3' -const EMPTY_SHA = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' - -// --------------------------------------------------------------------------- -// SigV4 (pure; the test file pins these against botocore-generated vectors) -// --------------------------------------------------------------------------- - -/** RFC3986 encode: everything except unreserved [A-Za-z0-9-_.~]. */ -export function rfc3986Encode(value) { - return encodeURIComponent(value).replace(/[!'()*]/g, (c) => - '%' + c.charCodeAt(0).toString(16).toUpperCase(), - ) -} - -/** Canonical query string: params sorted by encoded key (then encoded value). */ -export function canonicalQuery(params) { - return Object.entries(params) - .map(([k, v]) => [rfc3986Encode(k), rfc3986Encode(String(v))]) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([k, v]) => `${k}=${v}`) - .join('&') -} - -/** - * Canonical request for one S3-style request. - * `headers` is the exact header set that will be sent (host, x-amz-date, - * x-amz-content-sha256); canonicalization lowercases + sorts them. - */ -export function canonicalRequest(method, path, query, headers, payloadHash) { - const names = Object.keys(headers).map((n) => n.toLowerCase()).sort() - // Header values are read case-insensitively: keys may be mixed-case - // ('Content-Type'), but SigV4 canonicalizes the NAME to lowercase, so - // `headers[lowerName]` would miss the value. Find the original-key match. - const valueFor = (name) => { - const key = Object.keys(headers).find((k) => k.toLowerCase() === name) - return key == null ? undefined : headers[key] - } - const canonicalHeaders = names - .map((n) => `${n}:${String(valueFor(n)).trim().replace(/\s+/g, ' ')}`) - .join('\n') - return [ - method, - path, - query, - canonicalHeaders, - '', - names.join(';'), - payloadHash, - ].join('\n') -} - -/** The AWS4 string-to-sign for a canonical request. */ -export function stringToSign(canonical, date, scope) { - return ['AWS4-HMAC-SHA256', date, scope, createHash('sha256').update(canonical).digest('hex')].join('\n') -} - -function hmac(key, msg) { - return createHmac('sha256', key).update(msg).digest() -} - -/** Signature for a string-to-sign, given secret key + credential scope parts. */ -export function signature(stringToSignText, secretKey, date, region, service) { - const kDate = hmac(`AWS4${secretKey}`, date) - const kRegion = hmac(kDate, region) - const kService = hmac(kRegion, service) - const kSigning = hmac(kService, 'aws4_request') - return hmac(kSigning, stringToSignText).toString('hex') -} - -/** - * Full AWS4-HMAC-SHA256 Authorization header value for one request. - * `now` is 'YYYYMMDDTHHMMSSZ' (injectable for tests); scope date is its - * first 8 chars. `payloadHash` is hex sha256 of the body (or the empty - * string hash for bodyless requests). - */ -export function authHeader({ method, host, path, query, headers, payloadHash, accessKeyId, secretKey, now, region = REGION, service = SERVICE }) { - const date = now.slice(0, 8) - const canonical = canonicalRequest(method, path, query, headers, payloadHash) - const sts = stringToSign(canonical, now, `${date}/${region}/${service}/aws4_request`) - const sig = signature(sts, secretKey, date, region, service) - const signedHeaders = Object.keys(headers).map((n) => n.toLowerCase()).sort().join(';') - return ( - `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${date}/${region}/${service}/aws4_request, ` + - `SignedHeaders=${signedHeaders}, Signature=${sig}` - ) -} - -// --------------------------------------------------------------------------- -// R2 request plumbing -// --------------------------------------------------------------------------- - -export function s3Endpoint(accountId) { - return `https://${accountId}.r2.cloudflarestorage.com` -} - -/** Encode an object key into the URI path, segment by segment. */ -export function encodeKeyPath(key) { - return key.split('/').map(rfc3986Encode).join('/') -} - -function requiredEnv(name) { - const value = process.env[name] - if (!value) { - console.error(`::error::missing env ${name} — see the header comment in scripts/r2-release.mjs`) - process.exit(2) - } - return value -} - -function r2Headers(method, host, path, query, bodyHash, now, creds, contentType, contentLength, extraHeaders) { - const headers = { - host, - 'x-amz-date': now, - 'x-amz-content-sha256': bodyHash, - } - // Optional Content-Type for binary artifacts (App Installer / MSIX). It is - // added BEFORE the Authorization header is computed, so it lands in the - // SigV4 canonical headers + SignedHeaders exactly like host / x-amz-*. - if (contentType) headers['Content-Type'] = contentType - // Stream bodies (files >2GiB) need an explicit Content-Length — R2 - // rejects a bodyless-length PUT with 411 MissingContentLength. - if (contentLength != null) headers['Content-Length'] = String(contentLength) - // Extra request headers (e.g. Range) join BEFORE signing so they are - // covered by SignedHeaders like every other header we send. - if (extraHeaders) Object.assign(headers, extraHeaders) - headers.authorization = authHeader({ - method, - host, - path, - query, - headers, - payloadHash: bodyHash, - accessKeyId: creds.accessKeyId, - secretKey: creds.secretKey, - now, - }) - return headers -} - -async function signedFetch(method, url, { body, bodyHash, creds, now, contentType, contentLength, extraHeaders }) { - const { host, pathname, search } = new URL(url) - const query = search.replace(/^\?/, '') - const headers = r2Headers(method, host, pathname, query, bodyHash, now, creds, contentType, contentLength, extraHeaders) - const res = await fetch(url, { - method, - headers, - body: body ?? undefined, - // Stream bodies (files >2GiB) require the duplex option on Node's fetch. - ...(body != null && typeof body.pipe === 'function' ? { duplex: 'half' } : {}), - }) - const text = await res.text() - if (!res.ok) { - console.error(`::error::R2 ${method} ${pathname} -> ${res.status}`) - if (text) console.error(text.slice(0, 2000)) - } - return { res, text } -} - -/** Signed ranged GET: HEAD responses lose content-length through some - * proxies; Content-Range on a 1-byte GET is the reliable size oracle. */ -async function signedFetchRange(method, url, rangeHeaders, creds, now) { - return signedFetch(method, url, { bodyHash: EMPTY_SHA, creds, now, extraHeaders: rangeHeaders }) -} - -async function retry(fn, tries = 3) { - let lastError - for (let attempt = 1; attempt <= tries; attempt++) { - try { - return await fn() - } catch (err) { - lastError = err - if (attempt < tries) await new Promise((r) => setTimeout(r, 1000 * attempt)) - } - } - throw lastError -} - -// --------------------------------------------------------------------------- -// Layout helpers (pure) -// --------------------------------------------------------------------------- - -/** 'stable' for a stable tag, 'canary' for a -canary. tag. */ -export function channelForTag(tag) { - return /-canary\.20\d{6}(?:\d{6})?$/.test(tag) ? 'canary' : 'stable' -} - -// Content-Type for MSIX / App Installer artifacts lives in msix-shared.mjs -// (single source — the same suffixes must drive the stage job's uploads). - -/** Immutable per-release staging key. */ -export function stagingKeyFor(tag, filename) { - return `releases/tag/${tag}/${filename}` -} - -/** The feed directory key for a platform arm + channel, e.g. releases/win32/stable/. */ -export function feedDirFor(platform, channel) { - return `releases/${platform}/${channel}` -} - -// --------------------------------------------------------------------------- -// Commands -// --------------------------------------------------------------------------- - -/** APT indexes are mutable; by-hash indexes and versioned packages are not. */ -export function cacheControlFor(key) { - if (key.startsWith('releases/darwin/') && key.endsWith('-mac.yml')) return 'no-store' - if (!key.startsWith('releases/termux/')) return undefined - return key.includes('/by-hash/') || key.includes('/pool/') - ? 'public, max-age=31536000, immutable' - : 'no-store' -} - -async function putObject(creds, base, bucket, key, payload, now, contentType, conditions = {}) { - // `payload` is either a small in-memory Buffer (feed manifests from - // finalize) or a FILE PATH (binaries via `put`). The msixbundle is - // ~2.7GB so path payloads stream from disk (fs.readFileSync throws - // ERR_FS_FILE_TOO_LARGE past 2GiB); buffers upload directly. - const isPath = typeof payload === 'string' - const size = isPath ? (await fs.promises.stat(payload)).size : payload.length - const bodyHash = isPath - ? await new Promise((resolve, reject) => { - const hash = createHash('sha256') - const stream = fs.createReadStream(payload) - stream.on('data', (c) => hash.update(c)) - stream.on('end', () => resolve(hash.digest('hex'))) - stream.on('error', reject) - }) - : createHash('sha256').update(payload).digest('hex') - const url = `${base}/${bucket}/${encodeKeyPath(key)}` - await retry(async () => { - // Path payloads get a fresh stream per attempt: a consumed - // ReadableStream cannot be replayed for the retry ("body object - // should not be disturbed"). - const body = isPath ? fs.createReadStream(payload) : payload - const cacheControl = cacheControlFor(key) - const extraHeaders = { ...conditions, ...(cacheControl ? { 'Cache-Control': cacheControl } : {}) } - const { res, text } = await signedFetch('PUT', url, { body, bodyHash, contentLength: size, creds, now, contentType, extraHeaders }) - if (res.status === 412 && isPath && conditions['If-None-Match'] === '*') { - await verifyRemoteArtifact(url, creds, now, size, bodyHash, 'sha256', 'hex') - return - } - if (!res.ok) throw new Error(`PUT ${key} -> ${res.status}${text ? `: ${text.slice(0, 300)}` : ''}`) - }) - // HEAD can come back without content-length (intermediaries strip it on - // HEAD more readily than on ranged GETs). Fall back to a 1-byte ranged - // GET, whose Content-Range carries the authoritative total size. - const { res: headRes } = await retry(async () => { - const r = await signedFetch('HEAD', url, { bodyHash: EMPTY_SHA, creds, now }) - if (!r.res.ok) throw new Error(`HEAD ${key} -> ${r.res.status}`) - return r - }) - let remoteSize = headRes.headers.get('content-length') - if (remoteSize === null) { - const rangeHeaders = { range: 'bytes=0-0' } - const range = await retry(async () => { - const r = await signedFetchRange('GET', url, rangeHeaders, creds, now) - if (!r.res.ok) throw new Error(`GET range ${key} -> ${r.res.status}`) - return r - }) - const contentRange = range.res.headers.get('content-range') - const total = contentRange ? contentRange.match(/bytes 0-0\/(\d+)/) : null - if (!total) { - console.error(`::error::R2 ${key}: could not determine remote size (HEAD had no content-length, ranged GET had no content-range: ${contentRange})`) - process.exit(1) - } - remoteSize = total[1] - } - if (String(remoteSize) !== String(size)) { - console.error(`::error::R2 HEAD ${key}: size mismatch (remote ${remoteSize}, local ${size})`) - process.exit(1) - } - console.log(`✓ r2: ${key} (${size} bytes)`) -} - -export async function cmdPut({ tag, key, file, keyIsFull = false }) { - const accountId = requiredEnv('CLOUDFLARE_R2_ACCOUNT_ID') - const accessKeyId = requiredEnv('CLOUDFLARE_R2_ACCESS_KEY_ID') - const secretKey = requiredEnv('CLOUDFLARE_R2_SECRET_ACCESS_KEY') - const bucket = requiredEnv('CLOUDFLARE_R2_BUCKET') - const creds = { accessKeyId, secretKey } - const base = s3Endpoint(accountId) - - const now = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') - const keyPath = keyIsFull ? key : stagingKeyFor(tag, key) - const immutableMac = keyPath.startsWith('releases/tag/') && /-mac-(arm64|x64)\.(zip|dmg)(\.blockmap)?$/.test(keyPath) - await putObject(creds, base, bucket, keyPath, file, now, contentTypeFor(key), immutableMac ? { 'If-None-Match': '*' } : {}) -} - -async function verifyRemoteArtifact(urlValue, creds, now, expectedSize, digest, algorithm = 'sha512', encoding = 'base64') { - const url = new URL(urlValue) - const headers = r2Headers('GET', url.host, url.pathname, '', EMPTY_SHA, now, creds) - const response = await fetch(url, { headers, signal: AbortSignal.timeout(600_000) }) - if (!response.ok || !response.body) throw new Error(`Cannot verify ${url.pathname}: ${response.status}`) - const hash = createHash(algorithm) - let size = 0 - for await (const chunk of response.body) { hash.update(chunk); size += chunk.length } - if (size !== expectedSize || hash.digest(encoding) !== digest) throw new Error(`Artifact checksum mismatch: ${url.pathname}`) -} - -/** Validate both native legs, verify their bytes, then replace the feed pointer. */ -export async function cmdFinalize({ tag, dir, variant }) { - const { mergeMacFeeds, publishMacFeed } = await import('./darwin-feed.mjs') - if (variant && variant !== 'light') throw new Error('Unknown macOS variant') - const accountId = requiredEnv('CLOUDFLARE_R2_ACCOUNT_ID') - const creds = { - accessKeyId: requiredEnv('CLOUDFLARE_R2_ACCESS_KEY_ID'), - secretKey: requiredEnv('CLOUDFLARE_R2_SECRET_ACCESS_KEY') - } - const bucket = requiredEnv('CLOUDFLARE_R2_BUCKET') - const base = s3Endpoint(accountId) - const now = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') - const legs = Object.fromEntries(fs.readdirSync(dir).filter(name => name.endsWith('-mac.yml')) - .map(name => [name, fs.readFileSync(path.join(dir, name), 'utf8')])) - const plan = mergeMacFeeds(legs, tag, variant === 'light') - await publishMacFeed(plan, { - read: async key => { - const url = `${base}/${bucket}/${encodeKeyPath(key)}` - const { res, text } = await signedFetch('GET', url, { bodyHash: EMPTY_SHA, creds, now }) - if (res.status === 404) return null - if (!res.ok) throw new Error(`GET ${key} -> ${res.status}`) - const etag = res.headers.get('etag') - if (!etag) throw new Error(`No ETag for ${key}`) - return { text, etag } - }, - verify: async (key, file) => { - await verifyRemoteArtifact(`${base}/${bucket}/${encodeKeyPath(key)}`, creds, now, file.size, file.sha512) - }, - write: (key, text, etag) => putObject(creds, base, bucket, key, Buffer.from(text), now, - 'application/yaml', etag ? { 'If-Match': etag } : { 'If-None-Match': '*' }) - }) - console.log(`✓ r2: finalized ${tag} → ${plan.key}`) -} - -/** Parse a ListObjectsV2 XML body into { keys, lastModified, truncated, nextToken }. */ -export function parseListXml(xml) { - const unescape = (s) => - s.replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&') - const keys = [...xml.matchAll(/([^<]+)<\/Key>/g)].map((m) => unescape(m[1])) - const lastModified = {} - for (const m of xml.matchAll(/([\s\S]*?)<\/Contents>/g)) { - const key = m[1].match(/([^<]+)<\/Key>/) - const lm = m[1].match(/([^<]+)<\/LastModified>/) - if (key && lm) lastModified[unescape(key[1])] = Date.parse(lm[1]) - } - const truncated = /true<\/IsTruncated>/.test(xml) - const tokenMatch = xml.match(/([^<]+)<\/NextContinuationToken>/) - return { keys, lastModified, truncated, nextToken: tokenMatch ? unescape(tokenMatch[1]) : null } -} - -/** GET one object's body, or null when it does not exist / cannot be read. */ -async function getObject(creds, base, bucket, key, now) { - const url = `${base}/${bucket}/${encodeKeyPath(key)}` - const { res, text } = await signedFetch('GET', url, { bodyHash: EMPTY_SHA, creds, now }) - return res.ok ? text : null -} - -async function listObjects(prefix = '') { - const accountId = requiredEnv('CLOUDFLARE_R2_ACCOUNT_ID') - const accessKeyId = requiredEnv('CLOUDFLARE_R2_ACCESS_KEY_ID') - const secretKey = requiredEnv('CLOUDFLARE_R2_SECRET_ACCESS_KEY') - const bucket = requiredEnv('CLOUDFLARE_R2_BUCKET') - const creds = { accessKeyId, secretKey } - const base = s3Endpoint(accountId) - - const keys = [] - const lastModified = {} - let token = null - for (;;) { - const params = { 'list-type': '2', 'max-keys': '1000' } - if (prefix) params.prefix = prefix - if (token) params['continuation-token'] = token - const query = canonicalQuery(params) - const now = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') - const url = `${base}/${bucket}?${query}` - const { res, text } = await signedFetch('GET', url, { bodyHash: EMPTY_SHA, creds, now }) - if (!res.ok) process.exit(1) - const parsed = parseListXml(text) - keys.push(...parsed.keys) - Object.assign(lastModified, parsed.lastModified) - if (!parsed.truncated || !parsed.nextToken) break - token = parsed.nextToken - } - return { keys, lastModified } -} - -async function cmdList({ prefix }) { - const { keys } = await listObjects(prefix) - for (const key of keys) console.log(key) -} - -/** Keys whose own canary date (YYYYMMDD in the name) is before `cutoff`. */ -export function canaryDoomedKeys(keys, cutoff) { - return keys.filter((key) => { - const m = key.match(/-canary\.(\d{8})/) - return m && m[1] < cutoff - }) -} - -/** Feed publish order: the immutable .msixbundle FIRST, the pointer LAST. */ -export function publishFeedUploads(plan, upload) { - upload(`${plan.channelDir}/${plan.bundleFilename}`, plan.bundleFile) - upload(`${plan.channelDir}/${plan.appinstallerName}`, plan.appinstallerFile) -} - -/** - * Bundle basenames a .appinstaller manifest still references, parsed from the - * KNOWN generated shape (MainPackage/MainBundle Uri attributes only — never - * any Uri=" in the document). An unrecognized/empty manifest returns [], and - * the pruner treats [] as "block this directory" (fail closed). - */ -function feedBundleUris(appinstallerXml) { - const xml = String(appinstallerXml || '').trim() - // Only our complete generated shape is eligible for destructive retention. - if (!/^(?:<\?xml[^?]*\?>\s*)?]*>[\s\S]*<\/AppInstaller>$/.test(xml)) return [] - const elements = [...xml.matchAll(/<(?:MainPackage|MainBundle)\b[^>]*\/>/g)] - if (elements.length !== 1) return [] - const uri = elements[0][0].match(/\bUri="([^"]+)"/) - return uri && /\.(?:msixbundle|msix)$/i.test(uri[1]) ? [uri[1]] : [] -} - -export function referencedFeedBundleFilenames(appinstallerXml) { - return feedBundleUris(appinstallerXml).map(uri => uri.split('/').pop()) -} - -/** - * Full bucket keys a manifest references: each referenced bundle inside its - * feed dir, plus any MainPackage/MainBundle Uri carrying an absolute path - * (e.g. /releases/tag//… — tag-archive targets), protected by exact key. - */ -export function feedReferencedKeys(dir, appinstallerXml) { - const keys = [] - for (const uri of feedBundleUris(appinstallerXml)) { - keys.push(`${dir}/${uri.split('/').pop()}`) - try { - const p = new URL(uri, 'https://placeholder.invalid').pathname - if (p.startsWith('/releases/')) keys.push(decodeURIComponent(p.slice(1))) - } catch { /* relative Uri — already covered by the basename key */ } - } - return keys -} - -/** - * Canary feed-dir retention: the `-canary.YYYYMMDD` matcher cannot see feed - * bundles (numeric 4-part MSIX version names). A bundle is doomed when its - * feed dir's manifests were ALL readable AND it is referenced by none of - * them AND its actual list LastModified predates cutoffMs. Fail-closed on - * every unknown: unreadable/unrecognized manifest (zero references) blocks - * the whole dir; stable dirs and unknown dirs are never pruned; a missing - * LastModified keeps the object. - * - * @param {string[]} keys all bucket keys - * @param {Record} feedXmlByDir dir → manifest texts (null = unreadable) - * @param {Record} lastModifiedMs key → epoch ms from listObjects - * @param {number} cutoffMs - * @returns {string[]} doomed feed-dir bundle keys - */ -export function staleFeedBundleKeys(keys, feedXmlByDir, lastModifiedMs = {}, cutoffMs = -Infinity) { - const doomed = [] - for (const [dir, manifests] of Object.entries(feedXmlByDir || {})) { - if (!/\/canary$/.test(dir.replace(/\/+$/, ''))) continue // canaries only - const referenced = new Set() - let blocked = false - for (const xml of manifests || []) { - const names = referencedFeedBundleFilenames(xml) - if (names.length === 0) { - // Unreadable or unrecognized manifest in this dir → prune nothing here. - blocked = true - console.warn(`::warning::feed manifest unreadable/unrecognized, skipping feed retention for ${dir}/`) - break - } - for (const name of names) referenced.add(name) - } - if (blocked || referenced.size === 0) continue - const prefix = `${dir.replace(/\/+$/, '')}/` - for (const key of keys) { - if (!key.startsWith(prefix)) continue - if (!/\.(?:msixbundle|msix)$/i.test(key)) continue // pointers + metadata stay - if (referenced.has(key.slice(prefix.length))) continue - const lm = lastModifiedMs[key] - if (!Number.isFinite(lm) || lm >= cutoffMs) continue // keep-days grace (fail-closed) - doomed.push(key) - } - } - return doomed -} - -export async function cmdPrune({ keepDays, dryRun }) { - const accountId = requiredEnv('CLOUDFLARE_R2_ACCOUNT_ID') - const accessKeyId = requiredEnv('CLOUDFLARE_R2_ACCESS_KEY_ID') - const secretKey = requiredEnv('CLOUDFLARE_R2_SECRET_ACCESS_KEY') - const bucket = requiredEnv('CLOUDFLARE_R2_BUCKET') - const creds = { accessKeyId, secretKey } - const base = s3Endpoint(accountId) - - // Cutoff dated by the canary suffix in the KEY (like release.py's - // --prune-canaries: a re-uploaded old tag never resets its clock). - const cutoff = new Date(Date.now() - keepDays * 86400_000).toISOString().slice(0, 10).replace(/-/g, '') - const cutoffMs = Date.now() - keepDays * 86400_000 - const { keys, lastModified } = await listObjects() - const now = new Date().toISOString().replace(/[-:]/g, '').replace(/\.\d{3}/, '') - - // Read EVERY .appinstaller per feed dir (union of references protects the - // dir; any unreadable/unrecognized one blocks retention for that dir — - // handled inside staleFeedBundleKeys, null = unreadable). - const feedXmlByDir = {} - const protectedKeys = new Set() - for (const key of keys.filter((k) => k.endsWith('.appinstaller'))) { - const dir = key.slice(0, key.lastIndexOf('/')) - const xml = await getObject(creds, base, bucket, key, now) - if (feedBundleUris(xml).length === 0) { - throw new Error(`Cannot establish live references from ${key}; refusing to prune`) - } - ;(feedXmlByDir[dir] ??= []).push(xml) - for (const k of feedReferencedKeys(dir, xml)) protectedKeys.add(k) - } - - for (const key of keys.filter(key => key.startsWith('releases/darwin/') && key.endsWith('-mac.yml'))) { - const { macFeedReferences } = await import('./darwin-feed.mjs') - const text = await getObject(creds, base, bucket, key, now) - for (const reference of macFeedReferences(text)) protectedKeys.add(reference) - } - - const doomed = [ - ...canaryDoomedKeys(keys, cutoff), - ...staleFeedBundleKeys(keys, feedXmlByDir, lastModified, cutoffMs), - // Live referenced objects (incl. tag-archive targets) are never deleted. - ].filter((key) => !protectedKeys.has(key)) - - if (doomed.length === 0) { - console.log(`✓ r2: no canary objects older than ${keepDays} days`) - return - } - for (const key of doomed.sort()) { - if (dryRun) { - console.log(`(dry-run) would delete r2:${key}`) - continue - } - const url = `${base}/${bucket}/${encodeKeyPath(key)}` - const { res } = await signedFetch('DELETE', url, { bodyHash: EMPTY_SHA, creds, now }) - if (!res.ok) process.exit(1) - console.log(`deleted r2:${key}`) - } -} - -// --------------------------------------------------------------------------- -// CLI -// --------------------------------------------------------------------------- - -function usage() { - console.error(`usage: - node scripts/r2-release.mjs put --tag vX.Y.Z --key --file [--key-is-full] - node scripts/r2-release.mjs finalize --tag vX.Y.Z --dir - node scripts/r2-release.mjs list [--prefix

] - node scripts/r2-release.mjs prune-canaries --keep-days [--dry-run] - - --key-is-full: the --key is a FULL object key (e.g. releases/win32/stable/…), - not a filename to archive under releases/tag//.`) - process.exit(2) -} - -export function isMain() { - return process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1] -} - -export async function main(argv = process.argv.slice(2)) { - const [cmd, ...rest] = argv - const args = {} - for (let i = 0; i < rest.length; i++) { - const flag = rest[i] - if (['--tag', '--key', '--file', '--prefix', '--keep-days', '--dir', '--variant'].includes(flag)) { - args[flag.slice(2)] = rest[++i] - } else if (flag === '--dry-run' || flag === '--key-is-full') { - args[flag.slice(2)] = true - } else { - usage() - } - } - if (cmd === 'put') { - const tag = args.tag || process.env.HERMES_PAYLOAD_TAG - if (!tag || !args.key || !args.file) usage() - await cmdPut({ tag, key: args.key, file: args.file, keyIsFull: Boolean(args['key-is-full']) }) - } else if (cmd === 'finalize') { - if (!args.tag || !args.dir) usage() - await cmdFinalize({ tag: args.tag, dir: args.dir, variant: args.variant }) - } else if (cmd === 'list') { - await cmdList({ prefix: args.prefix ?? '' }) - } else if (cmd === 'prune-canaries') { - const keepDays = Number(args['keep-days']) - if (!Number.isFinite(keepDays) || keepDays <= 0) usage() - // '--dry-run' parses to args['dry-run'] (flag.slice(2) keeps the dash). - await cmdPrune({ keepDays, dryRun: Boolean(args['dry-run']) }) - } else { - usage() - } -} - -if (isMain()) { - main().catch((err) => { - console.error(`::error::${err?.stack ?? err}`) - process.exit(1) - }) -} diff --git a/scripts/release-content-types.json b/scripts/release-content-types.json new file mode 100644 index 0000000000..22ac92ff1c --- /dev/null +++ b/scripts/release-content-types.json @@ -0,0 +1,12 @@ +{ + ".appinstaller": "application/appinstaller", + ".msixbundle": "application/msixbundle", + ".msix": "application/msix", + ".deb": "application/vnd.debian.binary-package", + ".gz": "application/gzip", + ".asc": "text/plain", + "release.gpg": "application/pgp-signature", + "inrelease": "text/plain", + "release": "text/plain", + "packages": "text/plain" +} diff --git a/scripts/release.py b/scripts/release.py index 5ab8dd6229..b962e48c07 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -2155,42 +2155,31 @@ def list_remotes() -> list[str]: def dispatch_desktop_build(tag: str, gh_repo: str | None) -> bool: - """Start the desktop bundled build for ``tag``. Returns True on success. + """Dispatch the release pipeline after its draft exists. - This is how EVERY release of either kind gets its installers. The - workflow takes workflow_dispatch only, because that is one of the two - events GITHUB_TOKEN is allowed to raise: the scheduled canary pushes - its tag as github-actions[bot], and a bot-pushed tag starts no - workflow run at all. A tag-push trigger would therefore work for a - hand-cut stable release and silently do nothing for the canary. - - Called after the draft release exists — its body is where the - builds-pending / builds-table jobs splice the download tables (the - binaries themselves go to the R2 bucket; the release carries notes - only). The workflow FILE is taken from the repo's DEFAULT BRANCH (not - the tag): a tag-dispatched run scopes actions/cache under - refs/heads/refs/tags/ — a mangled per-tag scope that can never be - restored by a later canary, so every canary rebuilt from cold. On a - branch ref the caches scope to refs/heads/ and persist across - canaries. The build CODE still comes from the tag via the `tag` input - (the workflow checks out ${{ inputs.tag }}), so an old-tag rebuild - builds the old snapshot with the current workflow. + Stable workflows run on the tag so all reusable checks see the same + commit. Their gate owns artifact publication and channel promotion. + Canary builds keep default-branch workflow/cache scope and tagged inputs. + Explicit dispatch also works for tags created by GITHUB_TOKEN. """ - cmd = [ - "gh", "workflow", "run", "desktop-bundled-release.yml", - "--ref", "main", # placeholder — replaced after the which-guard below - "-f", f"tag={tag}", - "-f", "upload_release=true", - ] + canary = _CANARY_TAG_RE.fullmatch(tag) is not None + from scripts.releases.semver import STABLE_TAG + if not canary and not STABLE_TAG.fullmatch(tag): + raise ValueError("Expected an exact stable or canary release tag") + workflow = "desktop-bundled-release.yml" if canary else "stable-release.yml" + cmd = ["gh", "workflow", "run", workflow, "--ref", "main" if canary else tag, + "-f", f"tag={tag}"] + if canary: + cmd += ["-f", "upload_release=true"] if gh_repo: cmd += ["--repo", gh_repo] if not shutil.which("gh"): - print(" ✗ Cannot start the desktop build: `gh` CLI not found.") + print(" ✗ Cannot start the release pipeline: `gh` CLI not found.") print(f" Start it manually: {' '.join(cmd)}") return False - dispatch_ref = _default_branch(gh_repo) or "main" + dispatch_ref = (_default_branch(gh_repo) or "main") if canary else tag cmd[cmd.index("--ref") + 1] = dispatch_ref result = subprocess.run( @@ -2198,11 +2187,11 @@ def dispatch_desktop_build(tag: str, gh_repo: str | None) -> bool: errors="replace", cwd=str(REPO_ROOT), ) if result.returncode != 0: - print(f" ✗ Could not start the desktop build: {result.stderr.strip()}") + print(f" ✗ Could not start the release pipeline: {result.stderr.strip()}") print(f" Start it manually: {' '.join(cmd)}") return False - print(f" ✓ Desktop build started for {tag} (workflow from {dispatch_ref})") + print(f" ✓ {workflow} started for {tag} (workflow from {dispatch_ref})") return True @@ -3032,13 +3021,8 @@ def main(): print(" Continue manually after fixing access:") print(f" git push {push_remote} HEAD --tags") - # Create the GitHub release as a DRAFT (it carries the notes only), - # then start the desktop build. The build stages the installers and - # feed files to the R2 bucket and the finalize job publishes the - # feeds; the builds-table job renders the download links into this - # draft's body. Publishing now would expose an artifact-less - # release; a stable release is published by hand once the matrix - # is green. + # Keep the release hidden until the stable pipeline completes all + # validation, artifact publication and channel promotion. changelog_file = REPO_ROOT / ".release_notes.md" changelog_file.write_text(changelog, encoding="utf-8") @@ -3072,10 +3056,8 @@ def main(): print(f" ✓ GitHub draft release created: {result.stdout.strip()}") dispatch_desktop_build(tag_name, gh_repo) print(f"\n 🎉 Release v{new_version} ({tag_name}) drafted!") - print(" The Desktop Bundled Release workflow stages installers to the R2 bucket.") - print(" Publish once it is green:") - repo_flag = f" --repo {gh_repo}" if gh_repo else "" - print(f" gh release edit {tag_name}{repo_flag} --draft=false") + print(" Stable Release runs full CI, Docker and package acceptance.") + print(" It publishes artifacts and advances stable only after the required gates pass.") else: if result is None: print(" ✗ GitHub release skipped: `gh` CLI not found.") diff --git a/scripts/releases/darwin.py b/scripts/releases/darwin.py new file mode 100644 index 0000000000..d27b543395 --- /dev/null +++ b/scripts/releases/darwin.py @@ -0,0 +1,208 @@ +"""Validate and conditionally publish macOS update feeds.""" +from __future__ import annotations + +import json +from pathlib import Path +import re +from typing import Any, Callable + +from . import r2 as r2_module +from .semver import compare, is_valid_version + +ARCHES = ("arm64", "x64") +_HASH_PATTERN = re.compile(r"^[A-Za-z0-9+/]{86}==$") +_TAG_PATTERN = re.compile(r"^v\d+\.\d+\.\d+(?:-canary\.\d{14})?$") + + +def _darwin_feed(channel: str, light: bool = False) -> dict[str, Any]: + """Read the same feed facts as the desktop runtime.""" + with (Path(__file__).resolve().parents[2] / "apps/desktop/update-feed.json").open(encoding="utf-8") as file: + feeds = json.load(file) + if channel not in feeds: + raise TypeError(f"Unknown update channel: {channel}") + return feeds[channel]["light" if light else "bundled"] + + +# --------------------------------------------------------------------------- +# Feed parsing / merging (pure) +# --------------------------------------------------------------------------- + +def parse_mac_feed(text: str) -> dict[str, Any]: + import yaml # lazy: PyYAML loads only on the feed path + + feed = yaml.safe_load(text) + if ( + not isinstance(feed, dict) + or not is_valid_version(str(feed.get("version", ""))) + or not isinstance(feed.get("files"), list) + or not feed["files"] + ): + raise ValueError("Invalid macOS update feed") + for file_entry in feed["files"]: + if ( + not isinstance(file_entry, dict) + or not isinstance(file_entry.get("url"), str) + or not isinstance(file_entry.get("sha512"), str) + or not _HASH_PATTERN.match(file_entry["sha512"]) + or not isinstance(file_entry.get("size"), int) + or isinstance(file_entry.get("size"), bool) + or file_entry["size"] <= 0 + ): + raise ValueError("Invalid macOS artifact metadata") + if feed.get("path") and not any( + isinstance(f, dict) + and f.get("url") == feed.get("path") + and f.get("sha512") == feed.get("sha512") + for f in feed["files"] + ): + raise ValueError("Legacy feed path/hash disagrees with files") + return feed + + +_MAC_URL_PATTERN = re.compile( + r"^/releases/tag/v[0-9A-Za-z.+-]+/[0-9A-Za-z._+-]+\.(zip|dmg)$" +) + + +def mac_feed_references(text: str) -> list[str]: + """Every bucket key a published feed references (artifact + .blockmap).""" + feed = parse_mac_feed(text) + references: list[str] = [] + for file_entry in feed["files"]: + # Only our immutable artifact namespace is eligible for publication + # or pruning; anything else fails loudly instead of half-publishing. + if not _MAC_URL_PATTERN.match(file_entry["url"]): + raise ValueError(f"Invalid macOS artifact path: {file_entry['url']}") + key = file_entry["url"][1:] + references.extend([key, f"{key}.blockmap"]) + return references + + +def merge_mac_feeds(legs: dict[str, str], tag: str, light: bool = False) -> dict[str, Any]: + """Validate both native legs, merge them, and rewrite artifact URLs into + the immutable per-release tag namespace.""" + import yaml # lazy + + version = tag[1:] if isinstance(tag, str) and tag.startswith("v") else "" + if not is_valid_version(version) or not _TAG_PATTERN.match(tag): + raise ValueError("Invalid macOS release tag") + selection = _darwin_feed("canary" if "-canary." in version else "stable", light) + expected = [f"{arch}-{selection['fileName']}" for arch in ARCHES] + if sorted(legs.keys()) != sorted(expected): + raise ValueError("Expected exactly one ARM64 and one x64 macOS feed") + files: dict[str, dict[str, Any]] = {} + first: dict[str, Any] | None = None + for index, name in enumerate(expected): + leg = parse_mac_feed(legs[name]) + if str(leg.get("version")) != version: + raise ValueError(f"Feed version does not match {tag}") + prefix = f"{'HermesLight' if light else 'HermesBundled'}-{version}-mac-{ARCHES[index]}" + if not any(f.get("url") == f"{prefix}.zip" for f in leg["files"]): + raise ValueError(f"Missing native ZIP for {ARCHES[index]}") + for file_entry in leg["files"]: + if file_entry.get("url") not in (f"{prefix}.zip", f"{prefix}.dmg"): + raise ValueError(f"Wrong variant or architecture: {file_entry.get('url')}") + rewritten = {**file_entry, "url": f"/releases/tag/{tag}/{file_entry['url']}"} + prior = files.get(file_entry["url"]) + if prior is not None and prior != rewritten: + raise ValueError(f"Conflicting artifact: {file_entry['url']}") + files[file_entry["url"]] = rewritten + if first is None: + first = leg + assert first is not None + merged = {**first, "files": list(files.values())} + if merged.get("path"): + merged["path"] = f"/releases/tag/{tag}/{merged['path']}" + text = yaml.safe_dump(merged, width=100000, default_flow_style=False, sort_keys=False) + mac_feed_references(text) # publish only a feed that parses back clean + return { + "key": f"{selection['directory']}/{selection['fileName']}", + "text": text, + "files": merged["files"], + "version": version, + } + + +# --------------------------------------------------------------------------- +# Publication (transport-verified) +# --------------------------------------------------------------------------- + +def publish_mac_feed( + plan: dict[str, Any], + transport: dict[str, Callable[..., Any]], +) -> None: + """The transport verifies immutable bytes and conditionally replaces one + pointer. Downgrade rejection: semver.compare against the live feed.""" + read = transport["read"] + verify = transport["verify"] + write = transport["write"] + live = read(plan["key"]) + if live: + old_feed = parse_mac_feed(live["text"]) + next_feed = parse_mac_feed(plan["text"]) + order = compare(plan["version"], str(old_feed["version"])) + if order < 0: + raise ValueError("Refusing to move the macOS feed backward") + if order == 0: + if old_feed != next_feed: + raise ValueError("Refusing to replace published version with different artifacts") + return + for file_entry in plan["files"]: + verify(file_entry["url"][1:], file_entry) + write(plan["key"], plan["text"], live["etag"] if live else None) + published = read(plan["key"]) + if not published or published["text"] != plan["text"]: + raise ValueError("macOS feed readback differs from publication") + + +# --------------------------------------------------------------------------- +# finalize (real signed transport) +# --------------------------------------------------------------------------- + +def finalize(tag: str, dir: str, variant: str | None = None) -> None: + """Validate both native legs, verify their streamed bytes, then replace + the feed pointer (conditional write, then readback).""" + if variant and variant != "light": + raise ValueError("Unknown macOS variant") + creds, base, bucket = r2_module.credentials() + now = r2_module.amz_timestamp() + import os + + legs = { + name: open(os.path.join(dir, name), encoding="utf-8").read() + for name in sorted(os.listdir(dir)) + if name.endswith("-mac.yml") + } + plan = merge_mac_feeds(legs, tag, variant == "light") + + def read(key: str) -> dict[str, str] | None: + try: + response = r2_module.signed_request( + "GET", f"{base}/{bucket}/{r2_module.encode_key_path(key)}", creds=creds, now=now + ) + except r2_module.R2RequestError as err: + if err.status == 404: + return None # first publish: no live feed yet + raise + if response.status >= 400: + raise r2_module.R2RequestError("GET", key, response.status) + etag = response.header("etag") + if not etag: + raise ValueError(f"No ETag for {key}") + return {"text": response.text(), "etag": etag} + + def verify(key: str, file_entry: dict[str, Any]) -> None: + r2_module.verify_remote_artifact( + f"{base}/{bucket}/{r2_module.encode_key_path(key)}", + creds, now, file_entry["size"], file_entry["sha512"], + ) + + def write(key: str, text: str, etag: str | None) -> None: + r2_module.put_object( + creds, base, bucket, key, text.encode("utf-8"), now, + "application/yaml", + {"If-Match": etag} if etag else {"If-None-Match": "*"}, + ) + + publish_mac_feed(plan, {"read": read, "verify": verify, "write": write}) + print(f"OK r2: finalized {tag} -> {plan['key']}") diff --git a/scripts/releases/docker.py b/scripts/releases/docker.py new file mode 100644 index 0000000000..0ca6c06afb --- /dev/null +++ b/scripts/releases/docker.py @@ -0,0 +1,133 @@ +"""Validate staged Docker artifact identities and publish receipts.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys + +MANIFEST_SCHEMA = 1 +SHA256 = re.compile(r"[a-f0-9]{64}") +GIT_SHA = re.compile(r"[a-f0-9]{40}") +from scripts.releases.semver import STABLE_TAG +ARCHES = ("amd64", "arm64") + +class DockerReleaseError(ValueError): + """Raised when a phase/manifest violates the staged-release contract.""" + + +def require_stable_tag(tag: str) -> str: + if not isinstance(tag, str) or not STABLE_TAG.fullmatch(tag or ""): + raise DockerReleaseError(f"Not a stable release tag: {tag!r}") + return tag + + +def build_manifest(tag: str, commit: str, digests: dict[str, str], archive_sha256: dict[str, str] | None = None) -> dict: + """Digest manifest emitted by the test phase (artifact ``docker-test-manifest``).""" + require_stable_tag(tag) + if not isinstance(commit, str) or not GIT_SHA.fullmatch(commit): + raise DockerReleaseError(f"Invalid release commit: {commit!r}") + if sorted(digests) != sorted(ARCHES): + raise DockerReleaseError(f"Manifest needs per-arch digests for {ARCHES}, got {sorted(digests)}") + for arch, digest in digests.items(): + if not SHA256.fullmatch(digest): + raise DockerReleaseError(f"Invalid digest for {arch}: {digest!r}") + manifest = { + "schema": MANIFEST_SCHEMA, + "tag": tag, + "commit": commit, + "digests": {arch: digests[arch] for arch in ARCHES}, + } + if archive_sha256 is not None: + if sorted(archive_sha256) != sorted(ARCHES): + raise DockerReleaseError(f"Manifest needs per-arch archive hashes for {ARCHES}") + manifest["archives"] = {arch: archive_sha256[arch] for arch in ARCHES} + return manifest + + +def parse_manifest(raw: bytes) -> dict: + try: + manifest = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DockerReleaseError(f"Manifest is not valid JSON: {exc}") from exc + if not isinstance(manifest, dict) or manifest.get("schema") != MANIFEST_SCHEMA: + raise DockerReleaseError("Manifest schema mismatch") + require_stable_tag(manifest.get("tag", "")) + if not isinstance(manifest.get("commit"), str) or not GIT_SHA.fullmatch(manifest["commit"]): + raise DockerReleaseError("Manifest commit is not a full git SHA") + digests = manifest.get("digests") + if not isinstance(digests, dict) or sorted(digests) != sorted(ARCHES): + raise DockerReleaseError(f"Manifest needs per-arch digests for {ARCHES}") + for arch, digest in digests.items(): + if not isinstance(digest, str) or not SHA256.fullmatch(digest): + raise DockerReleaseError(f"Invalid digest for {arch}") + archives = manifest.get("archives", {}) + if archives and (not isinstance(archives, dict) or sorted(archives) != sorted(ARCHES)): + raise DockerReleaseError(f"Manifest archive hashes must cover {ARCHES}") + if "list-digest" in manifest and not re.fullmatch(r"sha256:[a-f0-9]{64}", manifest["list-digest"]): + raise DockerReleaseError("Invalid published manifest-list digest") + return manifest + + +def verify_manifest(manifest: dict, tag: str, commit: str) -> None: + """Fail the publish/promote phase unless the manifest matches the release identity.""" + if manifest.get("tag") != tag or manifest.get("commit") != commit: + raise DockerReleaseError( + f"Tested manifest identity {manifest.get('tag')}@{manifest.get('commit')} " + f"does not match release {tag}@{commit}" + ) + + +def sha256_file(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p_manifest = sub.add_parser("manifest", help="Emit the tested-image digest manifest JSON") + p_manifest.add_argument("--tag", required=True) + p_manifest.add_argument("--commit", required=True) + p_manifest.add_argument("--digest-amd64", required=True) + p_manifest.add_argument("--digest-arm64", required=True) + p_manifest.add_argument("--archive-amd64", default="", help="Optional sha256 file of the amd64 image archive") + p_manifest.add_argument("--archive-arm64", default="") + + p_verify = sub.add_parser("verify", help="Verify a downloaded manifest against the release identity") + p_verify.add_argument("--tag", required=True) + p_verify.add_argument("--commit", required=True) + p_verify.add_argument("manifest", help="Path to the downloaded manifest JSON") + + args = parser.parse_args(argv) + try: + if args.command == "manifest": + archive_hashes = {} + for arch, path in (("amd64", args.archive_amd64), ("arm64", args.archive_arm64)): + if path: + archive_hashes[arch] = sha256_file(path) + manifest = build_manifest( + args.tag, + args.commit, + {"amd64": args.digest_amd64, "arm64": args.digest_arm64}, + archive_hashes or None, + ) + print(json.dumps(manifest, indent=2)) + else: + with open(args.manifest, "rb") as handle: + manifest = parse_manifest(handle.read()) + verify_manifest(manifest, args.tag, args.commit) + print(json.dumps(manifest)) + except DockerReleaseError as exc: + print(f"::error::{exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/releases/pick_tags.py b/scripts/releases/pick_tags.py new file mode 100644 index 0000000000..f8e982d13f --- /dev/null +++ b/scripts/releases/pick_tags.py @@ -0,0 +1,34 @@ +"""Sample installed release baselines by creation time across version schemes.""" +import argparse +import json +import re +import subprocess +from pathlib import Path + + +def pick_tags(repo: Path, count: int, exclude: str = "") -> list[str]: + if not 1 <= count <= 10: + raise ValueError("Tag count must be between one and ten") + raw = subprocess.check_output(["git", "-C", str(repo), "for-each-ref", "--sort=creatordate", + "--format=%(refname:short)", "refs/tags/v*"], text=True, encoding="utf-8") + tags = [tag for tag in raw.splitlines() if re.fullmatch(r"v\d+\.\d+\.\d+(?:\.\d+)?", tag) and tag != exclude] + if not tags: + raise ValueError("No released baseline tags remain") + if len(tags) <= count: + return tags + if count == 1: + return [tags[-1]] + return [tags[(slot * (len(tags) - 1) * 2 + count - 1) // ((count - 1) * 2)] for slot in range(count)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path.cwd()) + parser.add_argument("--count", type=int, default=3) + parser.add_argument("--exclude-ref", default="") + args = parser.parse_args() + print(json.dumps(pick_tags(args.repo, args.count, args.exclude_ref))) + + +if __name__ == "__main__": + main() diff --git a/scripts/releases/r2.py b/scripts/releases/r2.py new file mode 100644 index 0000000000..399183b673 --- /dev/null +++ b/scripts/releases/r2.py @@ -0,0 +1,888 @@ +"""Stream and verify release objects through the signed R2 transport.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import hmac +import http.client +import os +import re +import sys +import time +from datetime import datetime, timezone +from typing import Callable, Iterable +from urllib.parse import quote, urlparse + +REGION = "auto" +SERVICE = "s3" +EMPTY_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +_CONTENT_TYPES_PATH = Path(__file__).resolve().parents[1] / "release-content-types.json" +with _CONTENT_TYPES_PATH.open(encoding="utf-8") as _file: + _CONTENT_TYPES = tuple(json.load(_file).items()) + +def content_type_for(filename: str) -> str | None: + lower = filename.lower() + base = lower[lower.rfind("/") + 1:] + for key, mime in _CONTENT_TYPES: + if "." in key: + if lower.endswith(key): + return mime + elif base == key: + return mime + return None + + +# --------------------------------------------------------------------------- +# SigV4 (pure; tests pin these against botocore-generated vectors) +# --------------------------------------------------------------------------- + +_UNRESERVED = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~" +) + + +def rfc3986_encode(value: str) -> str: + """RFC3986 encode: everything except unreserved [A-Za-z0-9-_.~].""" + out: list[str] = [] + for char in value: + if char in _UNRESERVED: + out.append(char) + else: + out.extend(f"%{b:02X}" for b in char.encode("utf-8")) + return "".join(out) + + +def canonical_query(params: dict[str, str]) -> str: + """Canonical query string: params sorted by encoded key (then value).""" + pairs = sorted( + (rfc3986_encode(k), rfc3986_encode(str(v))) for k, v in params.items() + ) + return "&".join(f"{k}={v}" for k, v in pairs) + + +def _header_value(headers: dict[str, str], name: str) -> str: + for key, value in headers.items(): + if key.lower() == name: + return str(value) + raise KeyError(name) + + +def canonical_request( + method: str, + path: str, + query: str, + headers: dict[str, str], + payload_hash: str, +) -> str: + """Canonical request for one S3-style request. Header names are + canonicalized lowercase + sorted; values are read case-insensitively + (the mixed-case 'Content-Type' regression is pinned by a test).""" + names = sorted(k.lower() for k in headers) + whitespace = re.compile(r"\s+") + + def _collapsed(name: str) -> str: + return whitespace.sub(" ", _header_value(headers, name).strip()) + + canonical_headers = "\n".join(f"{n}:{_collapsed(n)}" for n in names) + return "\n".join( + [method, path, query, canonical_headers, "", ";".join(names), payload_hash] + ) + + +def string_to_sign(canonical: str, date: str, scope: str) -> str: + sts_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return "\n".join(["AWS4-HMAC-SHA256", date, scope, sts_hash]) + + +def _hmac(key: bytes, msg: str) -> bytes: + return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + + +def signature(sts_text: str, secret_key: str, date: str, region: str, service: str) -> str: + k_date = _hmac(f"AWS4{secret_key}".encode("utf-8"), date) + k_region = _hmac(k_date, region) + k_service = _hmac(k_region, service) + k_signing = _hmac(k_service, "aws4_request") + return hmac.new(k_signing, sts_text.encode("utf-8"), hashlib.sha256).hexdigest() + + +def auth_header( + *, + method: str, + host: str, + path: str, + query: str, + headers: dict[str, str], + payload_hash: str, + access_key_id: str, + secret_key: str, + now: str, + region: str = REGION, + service: str = SERVICE, +) -> str: + """Full AWS4-HMAC-SHA256 Authorization header value for one request.""" + date = now[:8] + canonical = canonical_request(method, path, query, headers, payload_hash) + sts = string_to_sign(canonical, now, f"{date}/{region}/{service}/aws4_request") + sig = signature(sts, secret_key, date, region, service) + signed_headers = ";".join(sorted(k.lower() for k in headers)) + return ( + f"AWS4-HMAC-SHA256 Credential={access_key_id}/{date}/{region}/{service}/aws4_request, " + f"SignedHeaders={signed_headers}, Signature={sig}" + ) + + +# --------------------------------------------------------------------------- +# R2 request plumbing +# --------------------------------------------------------------------------- + +def s3_endpoint(account_id: str) -> str: + return f"https://{account_id}.r2.cloudflarestorage.com" + + +def encode_key_path(key: str) -> str: + """Encode an object key into the URI path, segment by segment.""" + return "/".join(rfc3986_encode(seg) for seg in key.split("/")) + + +def amz_timestamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +class R2RequestError(Exception): + def __init__(self, method: str, path: str, status: int, body: str = ""): + self.method, self.path, self.status, self.body = method, path, status, body + super().__init__(f"R2 {method} {path} -> {status}" + (f": {body[:300]}" if body else "")) + + +class Response: + def __init__(self, status: int, headers: list[tuple[str, str]], body: bytes): + self.status = status + self.headers = {k.lower(): v for k, v in headers} + self._body = body + + def text(self) -> str: + return self._body.decode("utf-8", errors="replace") + + def header(self, name: str) -> str | None: + return self.headers.get(name.lower()) + + +def r2_headers( + method: str, + host: str, + path: str, + query: str, + body_hash: str, + now: str, + creds: dict[str, str], + content_type: str | None = None, + content_length: int | None = None, + extra_headers: dict[str, str] | None = None, +) -> dict[str, str]: + headers = { + "host": host, + "x-amz-date": now, + "x-amz-content-sha256": body_hash, + } + # Content-Type / Content-Length / extras (Range, If-None-Match, ...) + # join BEFORE signing so they land in SignedHeaders like everything else. + if content_type: + headers["Content-Type"] = content_type + if content_length is not None: + headers["Content-Length"] = str(content_length) + if extra_headers: + headers.update(extra_headers) + headers["authorization"] = auth_header( + method=method, host=host, path=path, query=query, headers=headers, + payload_hash=body_hash, access_key_id=creds["access_key_id"], + secret_key=creds["secret_key"], now=now, + ) + return headers + + +def _connection(url: str, timeout: float) -> http.client.HTTPConnection: + parsed = urlparse(url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if parsed.scheme == "https": + import ssl + return http.client.HTTPSConnection( + parsed.hostname, port, context=ssl.create_default_context(), timeout=timeout + ) + return http.client.HTTPConnection(parsed.hostname, port, timeout=timeout) + + +_RETRYABLE_STATUSES = {500, 502, 503, 504} + + +def signed_request( + method: str, + url: str, + *, + body: bytes | None = None, + body_iter_factory: Callable[[], Iterable[bytes]] | None = None, + body_hash: str = EMPTY_SHA, + content_length: int | None = None, + creds: dict[str, str], + now: str, + content_type: str | None = None, + extra_headers: dict[str, str] | None = None, + tries: int = 3, +) -> Response: + """Signed request with retries. Path payloads get a FRESH stream per + attempt (a consumed iterator cannot be replayed). Raises R2RequestError + on a final non-2xx response.""" + parsed = urlparse(url) + query = parsed.query + last: Response | None = None + for attempt in range(1, tries + 1): + body_iter = body_iter_factory() if body_iter_factory is not None else None + headers = r2_headers( + method, parsed.netloc, parsed.path, query, body_hash, now, creds, + content_type=content_type, content_length=content_length, + extra_headers=extra_headers, + ) + conn = _connection(url, timeout=600.0) + try: + conn.request( + method, + parsed.path + ("?" + query if query else ""), + body=body_iter if body_iter is not None else body, + headers=headers, + ) + res = conn.getresponse() + data = res.read() + except (OSError, http.client.HTTPException): + if attempt == tries: + raise + time.sleep(float(attempt)) + continue + finally: + conn.close() + if body_iter is not None and hasattr(body_iter, "close"): + body_iter.close() + response = Response(res.status, res.getheaders(), data) + if 200 <= response.status < 300: + return response + last = response + if response.status in _RETRYABLE_STATUSES and attempt < tries: + time.sleep(1.0 * attempt) + continue + raise R2RequestError(method, parsed.path, response.status, response.text()) + raise R2RequestError(method, parsed.path, last.status if last else 0) + + +def stream_file(path: str, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: + with open(path, "rb") as handle: + while chunk := handle.read(chunk_size): + yield chunk + + +def file_sha256(path: str) -> str: + digest = hashlib.sha256() + for chunk in stream_file(path): + digest.update(chunk) + return digest.hexdigest() + + +def verify_remote_artifact( + url: str, + creds: dict[str, str], + now: str, + expected_size: int, + digest: str, + algorithm: str = "sha512", + encoding: str = "base64", + fetcher: Callable[[str], Response] | None = None, +) -> None: + """STREAMED GET (release artifacts are ~2GiB — never buffered whole); + size + digest must both match or the artifact is corrupt.""" + if fetcher is not None: + response = fetcher(url) + if response.status >= 400: + raise R2RequestError("GET", urlparse(url).path, response.status) + data = response._body # noqa: SLF001 — loopback-test responses are small + hash_obj = hashlib.new(algorithm) + hash_obj.update(data) + size = len(data) + else: + hash_obj, size = _stream_and_hash(url, creds, now, algorithm) + actual = ( + __import__("base64").b64encode(hash_obj.digest()).decode("ascii") + if encoding == "base64" + else hash_obj.hexdigest() + ) + if size != expected_size or actual != digest: + raise ValueError(f"Artifact checksum mismatch: {urlparse(url).path}") + + +def _stream_and_hash(url: str, creds: dict[str, str], now: str, algorithm: str): + """True streaming GET: hash + count bytes as chunks arrive off the + socket, never materializing the body in memory.""" + parsed = urlparse(url) + headers = r2_headers( + "GET", parsed.netloc, parsed.path, parsed.query, EMPTY_SHA, now, creds + ) + conn = _connection(url, timeout=600.0) + hash_obj = hashlib.new(algorithm) + size = 0 + try: + conn.request("GET", parsed.path + ("?" + parsed.query if parsed.query else ""), headers=headers) + res = conn.getresponse() + if res.status >= 400: + raise R2RequestError("GET", parsed.path, res.status) + while chunk := res.read(1024 * 1024): + hash_obj.update(chunk) + size += len(chunk) + finally: + conn.close() + return hash_obj, size + + +# --------------------------------------------------------------------------- +# Layout helpers (pure) +# --------------------------------------------------------------------------- + +def channel_for_tag(tag: str) -> str: + """'stable' for a stable tag, 'canary' for a -canary. tag.""" + return "canary" if re.search(r"-canary\.20\d{6}(?:\d{6})?$", tag) else "stable" + + +def staging_key_for(tag: str, filename: str) -> str: + return f"releases/tag/{tag}/{filename}" + + +def feed_dir_for(platform: str, channel: str) -> str: + return f"releases/{platform}/{channel}" + + +def cache_control_for(key: str) -> str | None: + """APT indexes are mutable; by-hash indexes and versioned packages are not.""" + if key.endswith(".appinstaller") or key.startswith("releases/stable/") or (key.startswith("releases/darwin/") and key.endswith("-mac.yml")): + return "no-store" + if not key.startswith("releases/termux/"): + return None + return ( + "public, max-age=31536000, immutable" + if "/by-hash/" in key or "/pool/" in key + else "no-store" + ) + + +# --------------------------------------------------------------------------- +# Credentials / env +# --------------------------------------------------------------------------- + +def required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + print( + f"::error::missing env {name} — see the header comment in scripts/releases/r2.py", + file=sys.stderr, + ) + raise SystemExit(2) + return value + + +def credentials() -> tuple[dict[str, str], str, str]: + """(creds, base, bucket) from the R2 env vars. No secrets are printed.""" + creds = { + "access_key_id": required_env("CLOUDFLARE_R2_ACCESS_KEY_ID"), + "secret_key": required_env("CLOUDFLARE_R2_SECRET_ACCESS_KEY"), + } + base = s3_endpoint(required_env("CLOUDFLARE_R2_ACCOUNT_ID")) + bucket = required_env("CLOUDFLARE_R2_BUCKET") + return creds, base, bucket + + +# --------------------------------------------------------------------------- +# put +# --------------------------------------------------------------------------- + +def put_object( + creds: dict[str, str], + base: str, + bucket: str, + key: str, + payload: str | bytes, + now: str, + content_type: str | None, + conditions: dict[str, str] | None = None, + fetcher: Callable[..., Response] | None = None, +) -> None: + """`payload` is a FILE PATH (streamed — the msixbundle is ~2.7GB; str or + os.PathLike accepted) or a small in-memory bytes body (feed manifests + from finalize). A 412 on an immutable path PUT verifies the remote bytes + match before treating the conflict as success.""" + conditions = conditions or {} + is_path = isinstance(payload, (str, os.PathLike)) + if is_path: + payload = os.fspath(payload) + size = os.path.getsize(payload) + body_hash = file_sha256(payload) + + def body_iter_factory() -> Iterable[bytes]: + return stream_file(payload) + + body: bytes | None = None + else: + size = len(payload) + body_hash = hashlib.sha256(payload).hexdigest() + body_iter_factory = None + body = payload + + cache_control = cache_control_for(key) + extra = dict(conditions) + if cache_control: + extra["Cache-Control"] = cache_control + + url = f"{base}/{bucket}/{encode_key_path(key)}" + last_error: Exception | None = None + response: Response | None = None + for _ in range(3): + try: + response = ( + fetcher(method="PUT", url=url, body_hash=body_hash, body=body, + content_length=size, extra_headers=extra) + if fetcher is not None + else signed_request( + "PUT", url, body=body, body_iter_factory=body_iter_factory, + body_hash=body_hash, content_length=size, creds=creds, now=now, + content_type=content_type, extra_headers=extra, + ) + ) + break + except R2RequestError as err: + last_error = err + if err.status == 412 and is_path and conditions.get("If-None-Match") == "*": + # Immutable conflict: verify the remote bytes ARE ours. + verify_remote_artifact( + url, creds, now, size, body_hash, algorithm="sha256", encoding="hex", + fetcher=fetcher, + ) + response = None + break + if err.status in _RETRYABLE_STATUSES: + time.sleep(1.0) + continue + raise + else: + raise last_error if last_error else R2RequestError("PUT", key, 0) + if response is not None and response.status >= 400: + raise R2RequestError("PUT", key, response.status, response.text()) + + # HEAD verifies the upload landed at the right size; fall back to a + # 1-byte ranged GET (Content-Range carries the authoritative total). + head = ( + fetcher(method="HEAD", url=url, body_hash=EMPTY_SHA) + if fetcher is not None + else signed_request("HEAD", url, creds=creds, now=now) + ) + remote_size = head.header("content-length") + if remote_size is None: + ranged = ( + fetcher(method="GET", url=url, body_hash=EMPTY_SHA, + extra_headers={"Range": "bytes=0-0"}) + if fetcher is not None + else signed_request("GET", url, creds=creds, now=now, + extra_headers={"Range": "bytes=0-0"}) + ) + content_range = ranged.header("content-range") or "" + match = re.search(r"bytes 0-0/(\d+)", content_range) + if not match: + raise R2RequestError( + "GET", key, ranged.status, + f"could not determine remote size (content-range: {content_range!r})", + ) + remote_size = match.group(1) + if str(remote_size) != str(size): + raise R2RequestError("HEAD", key, head.status, f"size mismatch (remote {remote_size}, local {size})") + print(f"OK r2: {key} ({size} bytes)") + + +def finalize(tag: str, dir: str, variant: str | None = None) -> None: + """Lazy re-export of the Darwin finalize so callers can treat + scripts.releases.r2 as the single transport surface.""" + from . import darwin as darwin_module + + darwin_module.finalize(tag=tag, dir=dir, variant=variant) + + +def put( + tag: str, + key: str, + file: str, + key_is_full: bool = False, + immutable: bool = False, +) -> None: + """Upload one artifact. `key` is a filename archived under + releases/tag// unless `key_is_full`. `immutable=True` sends + If-None-Match: * to reject replacement of existing bytes.""" + creds, base, bucket = credentials() + now = amz_timestamp() + key_path = key if key_is_full else staging_key_for(tag, key) + conditions = {"If-None-Match": "*"} if immutable else {} + put_object(creds, base, bucket, key_path, file, now, content_type_for(key), conditions) + + +# --------------------------------------------------------------------------- +# list + prune helpers (pure) +# --------------------------------------------------------------------------- + +def parse_list_xml(xml: str) -> dict: + """Parse a ListObjectsV2 XML body into keys/lastModified/truncated/nextToken.""" + import html + + keys = re.findall(r"([^<]+)", xml) + keys = [html.unescape(k) for k in keys] + last_modified: dict[str, int] = {} + truncated = "true" in xml + token_match = re.search(r"([^<]+)", xml) + + def _parse_epoch(text: str) -> int: + # R2 emits both '...Z' and fractional '...mmmZ' forms; accept either. + text = text.strip() + if "." in text: + return int( + datetime.strptime(text, "%Y-%m-%dT%H:%M:%S.%fZ") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + return int( + datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ") + .replace(tzinfo=timezone.utc) + .timestamp() + ) + + for block in re.findall(r"([\s\S]*?)", xml): + key_match = re.search(r"([^<]+)", block) + lm_match = re.search(r"([^<]+)", block) + if key_match and lm_match: + last_modified[html.unescape(key_match.group(1))] = _parse_epoch(lm_match.group(1)) + return { + "keys": keys, + "lastModified": last_modified, + "truncated": truncated, + "nextToken": html.unescape(token_match.group(1)) if token_match else None, + } + + +def list_objects( + prefix: str = "", + creds: dict[str, str] | None = None, + base: str | None = None, + bucket: str | None = None, + fetcher: Callable[..., Response] | None = None, +) -> dict: + if creds is None: + creds, base, bucket = credentials() + assert creds is not None and base is not None and bucket is not None + keys: list[str] = [] + last_modified: dict[str, int] = {} + token: str | None = None + while True: + params: dict[str, str] = {"list-type": "2", "max-keys": "1000"} + if prefix: + params["prefix"] = prefix + if token: + params["continuation-token"] = token + query = canonical_query(params) + url = f"{base}/{bucket}?{query}" + if fetcher is not None: + response = fetcher(method="GET", url=url, body_hash=EMPTY_SHA) + if response.status >= 400: + raise R2RequestError("GET", f"/{bucket}", response.status) + else: + response = signed_request("GET", url, creds=creds, now=amz_timestamp()) + parsed = parse_list_xml(response.text()) + keys.extend(parsed["keys"]) + last_modified.update(parsed["lastModified"]) + if not parsed["truncated"] or not parsed["nextToken"]: + break + token = parsed["nextToken"] + return {"keys": keys, "lastModified": last_modified} + + +def get_object( + creds: dict[str, str], base: str, bucket: str, key: str, now: str, + fetcher: Callable[..., Response] | None = None, +) -> str | None: + """GET one object's body, or None when it does not exist / cannot be read.""" + url = f"{base}/{bucket}/{encode_key_path(key)}" + response = ( + fetcher(method="GET", url=url, body_hash=EMPTY_SHA) + if fetcher is not None + else signed_request("GET", url, creds=creds, now=now) + ) + return response.text() if response.status < 400 else None + + +def canary_doomed_keys(keys: list[str], cutoff: str) -> list[str]: + """Keys whose own canary date (YYYYMMDD in the name) is before `cutoff`.""" + doomed = [] + for key in keys: + match = re.search(r"-canary\.(\d{8})", key) + if match and match.group(1) < cutoff: + doomed.append(key) + return doomed + + +def publish_feed_uploads(plan: dict, upload: Callable[[str, str], None]) -> None: + """Feed publish order: the immutable .msixbundle FIRST, the pointer LAST.""" + upload(f"{plan['channelDir']}/{plan['bundleFilename']}", plan["bundleFile"]) + upload(f"{plan['channelDir']}/{plan['appinstallerName']}", plan["appinstallerFile"]) + + +def referenced_feed_bundle_filenames(appinstaller_xml: str | None) -> list[str]: + return [uri.rsplit("/", 1)[-1] for uri in _feed_bundle_uris(appinstaller_xml)] + + +def feed_referenced_keys(dir_key: str, appinstaller_xml: str | None) -> list[str]: + """Full bucket keys a manifest references: each referenced bundle inside + its feed dir, plus any absolute /releases/... path Uri (tag-archive + targets), protected by exact key.""" + keys: list[str] = [] + for uri in _feed_bundle_uris(appinstaller_xml): + keys.append(f"{dir_key}/{uri.rsplit('/', 1)[-1]}") + if "/" in uri and uri.startswith(("http://", "https://")): + from urllib.parse import urlsplit, unquote + path = urlsplit(uri).path + if path.startswith("/releases/"): + keys.append(unquote(path[1:])) + return keys + + +def _feed_bundle_uris(appinstaller_xml: str | None) -> list[str]: + """Bundle basenames a .appinstaller manifest still references, parsed + from the KNOWN generated shape (MainPackage/MainBundle Uri attributes + only — never any Uri=" in the document). Unrecognized/empty -> [] and + the pruner treats [] as "block this directory" (fail closed).""" + xml = str(appinstaller_xml or "").strip() + if not re.fullmatch( + r"(?:<\?xml[^?]*\?>\s*)?]*>[\s\S]*", xml + ): + return [] + elements = re.findall(r"<(?:MainPackage|MainBundle)\b[^>]*/>", xml) + if len(elements) != 1: + return [] + uri_match = re.search(r"\bUri=\"([^\"]+)\"", elements[0]) + if uri_match and re.search(r"\.(?:msixbundle|msix)$", uri_match.group(1), re.I): + return [uri_match.group(1)] + return [] + + +def stale_feed_bundle_keys( + keys: list[str], + feed_xml_by_dir: dict[str, list[str | None]], + last_modified_ms: dict[str, float] | None = None, + cutoff_ms: float = float("-inf"), +) -> list[str]: + """Canary feed-dir retention: fail-closed on every unknown. A bundle is + doomed when its feed dir's manifests were ALL readable AND it is + referenced by none of them AND its list LastModified predates cutoff. + Unreadable/unrecognized manifest (zero references) blocks the whole dir; + stable dirs and unknown dirs are never pruned; a missing LastModified + keeps the object.""" + doomed: list[str] = [] + for dir_key, manifests in (feed_xml_by_dir or {}).items(): + if not re.search(r"/canary$", dir_key.rstrip("/")): + continue # canaries only + referenced: set[str] = set() + blocked = False + for xml in manifests or []: + names = referenced_feed_bundle_filenames(xml) + if not names: + blocked = True + print(f"::warning::feed manifest unreadable/unrecognized, skipping feed retention for {dir_key}/") + break + referenced.update(names) + if blocked or not referenced: + continue + prefix = f"{dir_key.rstrip('/')}/" + for key in keys: + if not key.startswith(prefix): + continue + if not re.search(r"\.(?:msixbundle|msix)$", key, re.I): + continue # pointers + metadata stay + if key[len(prefix):] in referenced: + continue + lm = (last_modified_ms or {}).get(key) + if lm is None or lm != lm or lm in (float("inf"),) or lm >= cutoff_ms: + continue # keep-days grace (fail-closed) + doomed.append(key) + return doomed + + +# --------------------------------------------------------------------------- +# prune-canaries +# --------------------------------------------------------------------------- + +def prune( + keep_days: int, + dry_run: bool = False, + fetcher: Callable[..., Response] | None = None, + now_epoch: float | None = None, +) -> None: + """Cutoff dated by the canary suffix in the KEY (a re-uploaded old tag + never resets its clock). Live referenced objects are never deleted.""" + creds, base, bucket = credentials() + current = now_epoch if now_epoch is not None else time.time() + cutoff_date = time.strftime( + "%Y%m%d", time.gmtime(current - keep_days * 86400) + ) + cutoff_ms = current - keep_days * 86400 + listing = list_objects(creds=creds, base=base, bucket=bucket, fetcher=fetcher) + keys = listing["keys"] + last_modified = listing["lastModified"] + now = amz_timestamp() + + feed_xml_by_dir: dict[str, list[str]] = {} + protected: set[str] = set() + for key in [k for k in keys if k.endswith(".appinstaller")]: + dir_key = key[: key.rfind("/")] + xml = get_object(creds, base, bucket, key, now, fetcher=fetcher) + if not _feed_bundle_uris(xml): + raise RuntimeError(f"Cannot establish live references from {key}; refusing to prune") + feed_xml_by_dir.setdefault(dir_key, []).append(xml or "") + for protected_key in feed_referenced_keys(dir_key, xml): + protected.add(protected_key) + + # Darwin feeds protect their artifacts (key + .blockmap) too. + from . import darwin as darwin_module + + for key in [k for k in keys if k.startswith("releases/darwin/") and k.endswith("-mac.yml")]: + text = get_object(creds, base, bucket, key, now, fetcher=fetcher) + if text is None: + # Fail closed: an unreadable Darwin feed might reference the very + # objects the pruner is about to delete. + raise RuntimeError(f"Cannot read Darwin feed {key}; refusing to prune") + for reference in darwin_module.mac_feed_references(text): + protected.add(reference) + + doomed = [ + key + for key in [ + *canary_doomed_keys(keys, cutoff_date), + *stale_feed_bundle_keys(keys, feed_xml_by_dir, last_modified, cutoff_ms), + ] + if key not in protected + ] + + if not doomed: + print(f"OK r2: no canary objects older than {keep_days} days") + return + for key in sorted(doomed): + if dry_run: + print(f"(dry-run) would delete r2:{key}") + continue + url = f"{base}/{bucket}/{encode_key_path(key)}" + response = ( + fetcher(method="DELETE", url=url, body_hash=EMPTY_SHA) + if fetcher is not None + else signed_request("DELETE", url, creds=creds, now=now) + ) + if response.status >= 400: + raise R2RequestError("DELETE", key, response.status) + print(f"deleted r2:{key}") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +USAGE = """usage: + python -m scripts.releases.r2 put --tag vX.Y.Z --key --file [--key-is-full] [--immutable] + python -m scripts.releases.r2 finalize --tag vX.Y.Z --dir [--variant light] + python -m scripts.releases.r2 list [--prefix

] + python -m scripts.releases.r2 prune-canaries --keep-days [--dry-run] + + --key-is-full: the --key is a FULL object key (e.g. releases/win32/stable/...), + not a filename to archive under releases/tag//. + --immutable: send If-None-Match: * (refuse to overwrite an existing object). +""" + +_VALUE_FLAGS = {"--tag", "--key", "--file", "--prefix", "--keep-days", "--dir", "--variant"} +_BOOL_FLAGS = {"--dry-run", "--key-is-full", "--immutable"} + + +def main(argv: list[str] | None = None) -> None: + argv = list(sys.argv[1:] if argv is None else argv) + if not argv: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + cmd, rest = argv[0], argv[1:] + args: dict[str, str | bool] = {} + i = 0 + while i < len(rest): + flag = rest[i] + if flag in _VALUE_FLAGS: + i += 1 + if i >= len(rest): + print(USAGE, file=sys.stderr) + raise SystemExit(2) + args[flag[2:]] = rest[i] + elif flag in _BOOL_FLAGS: + args[flag[2:]] = True + else: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + i += 1 + + if cmd == "put": + tag = args.get("tag") or os.environ.get("HERMES_PAYLOAD_TAG") + key, file = args.get("key"), args.get("file") + if not tag or not key or not file: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + immutable = bool(args.get("immutable")) + if not immutable: + # Preserve the JS CLI's derivation: darwin artifact staging keys + # are immutable per-release (rerun-safe via If-None-Match: *). + key_path = key if args.get("key-is-full") else staging_key_for(str(tag), str(key)) + immutable = key_path.startswith("releases/tag/") and bool( + re.search(r"-mac-(?:arm64|x64)\.(?:zip|dmg)(?:\.blockmap)?$", key_path) + ) + put( + tag=tag, + key=key, + file=file, + key_is_full=bool(args.get("key-is-full")), + immutable=bool(args.get("immutable")) or immutable, + ) + elif cmd == "finalize": + tag, dir_path = args.get("tag"), args.get("dir") + if not tag or not dir_path: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + from . import darwin as darwin_module + + darwin_module.finalize(tag=tag, dir=dir_path, variant=args.get("variant")) + elif cmd == "list": + listing = list_objects(prefix=str(args.get("prefix") or "")) + for key in listing["keys"]: + print(key) + elif cmd == "prune-canaries": + try: + keep_days = int(args.get("keep-days", "")) + except ValueError: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + if keep_days <= 0: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + prune(keep_days, dry_run=bool(args.get("dry-run"))) + else: + print(USAGE, file=sys.stderr) + raise SystemExit(2) + + +if __name__ == "__main__": + try: + main() + except SystemExit: + raise + except Exception as err: # pragma: no cover — CLI error surface + print(f"::error::{err}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/releases/semver.py b/scripts/releases/semver.py new file mode 100644 index 0000000000..3abbed9f00 --- /dev/null +++ b/scripts/releases/semver.py @@ -0,0 +1,54 @@ +"""Compare the stable and canary versions accepted by release feeds.""" +from __future__ import annotations + +import re +from hermes_cli.update_channel import _CANARY_TAG_RE + +STABLE_TAG = re.compile(r"v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)") + + +def is_valid_version(version: str) -> bool: + """semver.valid restricted to our release grammar (no build metadata, + no alphanumeric prerelease identifiers — those never appear in + generated tags).""" + if not isinstance(version, str): + return False + core, sep, tail = version.partition("-") + if sep and not STABLE_TAG.fullmatch("v" + core): + return False + if not sep: + return bool(STABLE_TAG.fullmatch("v" + version)) + if not _CANARY_TAG_RE.fullmatch("v" + version): + return False + # Canary timestamp is a fixed-length 14-digit numeric stamp. + stamp = tail.split(".", 1)[1] + return len(stamp) == 14 + + +def _prerelease_key(tail: str) -> list[int]: + """Numeric sort key for a canary suffix — 'canary.<14 digits>'.""" + return [int(tail.split(".", 1)[1])] + + +def compare(a: str, b: str) -> int: + """semver.compare for our grammar. Both sides must be valid release + versions (ValueError otherwise — feed publication fails loudly). + Semver ordering: stable 0.28.0 > any 0.28.0-canary.; stamps + compare numerically.""" + if not is_valid_version(a) or not is_valid_version(b): + raise ValueError(f"invalid release version(s): {a!r}, {b!r}") + a_core, _, a_tail = a.partition("-") + b_core, _, b_tail = b.partition("-") + if a_core != b_core: + ka = [int(p) for p in a_core.split(".")] + kb = [int(p) for p in b_core.split(".")] + return -1 if ka < kb else 1 + # Same core: a prerelease (canary) sorts BEFORE the stable release. + if a_tail and b_tail: + ka, kb = _prerelease_key(a_tail), _prerelease_key(b_tail) + return -1 if ka < kb else (1 if ka > kb else 0) + if a_tail: + return -1 + if b_tail: + return 1 + return 0 diff --git a/scripts/releases/stable.py b/scripts/releases/stable.py new file mode 100644 index 0000000000..b7da648ea3 --- /dev/null +++ b/scripts/releases/stable.py @@ -0,0 +1,229 @@ +"""Stable release admission, signed-package transitions and final release receipt.""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +import tomllib +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from scripts.releases.semver import STABLE_TAG +SHA = re.compile(r"[a-f0-9]{40}") +DIGEST = re.compile(r"[a-f0-9]{64}") +DESKTOP_TARGETS = ("windows/x64", "windows/arm64", "macos/x64", "macos/arm64") + + +def require_stable_identity(tag: str, commit: str, ref: str) -> None: + if not isinstance(tag, str) or not STABLE_TAG.fullmatch(tag) or not SHA.fullmatch(commit or "") or ref != f"refs/tags/{tag}": + raise ValueError("Stable release must run on its exact stable tag and commit") + + +def require_success(needs: dict, required: list[str]) -> None: + if not required or len(set(required)) != len(required): + raise ValueError("Invalid required-job list") + failures = [f"{name}={needs.get(name, {}).get('result', 'missing')}" + for name in required if needs.get(name, {}).get("result") != "success"] + if failures: + raise ValueError("Release blocked: " + ", ".join(failures)) + + +def validate_candidates(manifest: dict, tag: str, commit: str, public_base: str) -> dict: + require_stable_identity(tag, commit, f"refs/tags/{tag}") + if manifest.get("schema") != 1 or manifest.get("tag") != tag or manifest.get("commit") != commit or not isinstance(manifest.get("packages"), list): + raise ValueError("Candidate manifest does not match release identity") + prefix = urlsplit(f"{public_base.rstrip('/')}/releases/tag/{tag}/") + if prefix.scheme != "https" or prefix.username or prefix.password or not prefix.netloc: + raise ValueError("Public release origin must use HTTPS") + rows = {} + for item in manifest["packages"]: + target = f"{item.get('platform')}/{item.get('arch')}" + if target not in (*DESKTOP_TARGETS, "termux/aarch64") or target in rows or item.get("tag") != tag or item.get("commit") != commit: + raise ValueError(f"Invalid or duplicate candidate target: {target}") + artifact = item.get("artifact", {}) + url = urlsplit(artifact.get("url", "")) + decoded = unquote(url.path) + if any(part in (".", "..") for part in decoded.split("/")) or "\\" in decoded or "%" in decoded: + raise ValueError("Invalid artifact path encoding") + if (url.scheme, url.netloc) != (prefix.scheme, prefix.netloc) or not url.path.startswith(prefix.path) or url.query or url.fragment or url.username or url.password: + raise ValueError(f"Candidate package is outside its immutable tag archive: {target}") + if not DIGEST.fullmatch(artifact.get("sha256", "")) or not item.get("identity"): + raise ValueError(f"Invalid candidate digest or identity: {target}") + if item["platform"] == "windows": + windows_version(item.get("version", "")) + if not item.get("publisher") or not item.get("applicationId") or not url.path.endswith(".msixbundle"): + raise ValueError("Windows candidate needs publisher, applicationId and MSIX bundle") + elif item["platform"] == "macos": + if item.get("version") != tag[1:] or not re.fullmatch(r"[A-Z0-9]{10}", item.get("teamId", "")) or not url.path.endswith(".zip"): + raise ValueError("macOS candidate needs matching version, signing team and app ZIP") + rows[target] = item + if any(target not in rows for target in DESKTOP_TARGETS): + raise ValueError("Candidate manifest must cover Windows and macOS on both architectures") + return rows + + +def windows_version(value: str) -> tuple[int, ...]: + if not isinstance(value, str) or not re.fullmatch(r"\d+\.\d+\.\d+\.\d+", value): + raise ValueError("Windows package version must have four numeric components") + result = tuple(map(int, value.split("."))) + if any(n > 65535 for n in result): + raise ValueError("Windows package version exceeds 16 bits") + return result + + +def plan_transitions(previous: dict, candidate: dict, public_base: str) -> list[dict]: + old = validate_candidates(previous, previous.get("tag"), previous.get("commit"), public_base) + new = validate_candidates(candidate, candidate.get("tag"), candidate.get("commit"), public_base) + result = [] + for target in DESKTOP_TARGETS: + left, right = old[target], new[target] + if left["identity"] != right["identity"] or left["commit"] == right["commit"] or left["artifact"]["sha256"] == right["artifact"]["sha256"]: + raise ValueError("Update must preserve package identity and change the build") + if right["platform"] == "windows": + if (left["publisher"], left["applicationId"]) != (right["publisher"], right["applicationId"]): + raise ValueError("Update must preserve publisher and applicationId") + newer = windows_version(right["version"]) > windows_version(left["version"]) + else: + if left["teamId"] != right["teamId"]: + raise ValueError("Update must preserve signing team") + newer = tuple(map(int, right["version"].split("."))) > tuple(map(int, left["version"].split("."))) + if not newer: + raise ValueError("New package version must increase") + result.append({"target": target.replace("/", "-"), "transition": { + "schema": 1, "platform": right["platform"], "arch": right["arch"], "old": left, "new": right, + }}) + return result + + +def read_manifest(url: str, expected_hash: str | None = None, *, opener=urllib.request.urlopen) -> dict: + location = urlsplit(url) + if location.scheme != "https" or location.username or location.password: + raise ValueError("Manifest URL must use HTTPS without credentials") + with opener(url, timeout=60) as response: + if urlsplit(response.geturl()).scheme != "https": + raise ValueError("Manifest redirected outside HTTPS") + data = response.read(1024 * 1024 + 1) + if len(data) > 1024 * 1024: + raise ValueError("Release manifest exceeds size limit") + if expected_hash and hashlib.sha256(data).hexdigest() != expected_hash: + raise ValueError("Candidate manifest digest mismatch") + return json.loads(data) + + +def output(argv: list[str]) -> str: + return subprocess.check_output(argv, text=True, encoding="utf-8").strip() + + +def check_tag(env: dict, run=output) -> tuple[str, str]: + tag, commit = env.get("RELEASE_TAG"), env.get("GITHUB_SHA") + require_stable_identity(tag, commit, env.get("GITHUB_REF")) + actual = run(["git", "rev-parse", f"refs/tags/{tag}^{{commit}}"]) + remote = dict(line.split()[::-1] for line in run(["git", "ls-remote", "origin", f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"] ).splitlines()) + remote_commit = remote.get(f"refs/tags/{tag}^{{}}", remote.get(f"refs/tags/{tag}")) + if actual != commit or remote_commit != commit or run(["git", "rev-parse", "HEAD"]) != commit: + raise ValueError("Release tag or checkout moved") + run(["git", "fetch", "origin", "main"]) + run(["git", "merge-base", "--is-ancestor", commit, "origin/main"]) + return tag, commit + + +def emit(values: dict, env: dict) -> None: + with Path(env["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as file: + for key, value in values.items(): + file.write(f"{key}={value if isinstance(value, str) else json.dumps(value, separators=(',', ':'))}\n") + + +def read_candidate(env: dict) -> dict: + digest = env.get("CANDIDATE_MANIFEST_SHA256", "") + if not DIGEST.fullmatch(digest): + raise ValueError("Pinned candidate manifest digest is required") + return read_manifest(env["CANDIDATE_MANIFEST_URL"], digest) + + +def summary(text: str, env: dict) -> None: + with Path(env["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as file: + file.write(text + "\n") + + +def admit(env: dict) -> None: + tag, commit = check_tag(env) + with Path("pyproject.toml").open("rb") as file: + version = tomllib.load(file)["project"]["version"] + if f"v{version}" != tag: + raise ValueError("Stable tag must match the project version") + release = json.loads(output(["gh", "release", "view", tag, "--repo", env["GITHUB_REPOSITORY"], "--json", "tagName,isDraft,isPrerelease"])) + if release["tagName"] != tag or not release["isDraft"] or release["isPrerelease"]: + raise ValueError("Stable candidate must have a non-prerelease draft") + emit({"tag": tag, "commit": commit}, env) + summary(f"## Stable candidate {tag}\nCommit: {commit}\n\nDesktop Playwright E2E: deferred by owner, not passed.\nOSV findings retain the existing advisory policy.", env) + + +def transitions(env: dict) -> None: + from scripts.releases.r2 import put + + tag, commit = check_tag(env) + base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") + candidate = read_candidate(env) + validate_candidates(candidate, tag, commit, base) + try: + previous = read_manifest(env.get("BASELINE_MANIFEST_URL") or f"{base}/releases/stable/release-candidates.json") + except urllib.error.HTTPError as error: + if error.code == 404: + raise ValueError("No published stable package baseline. Supply baseline-manifest for an actual previous stable release; acceptance cannot be skipped.") from error + raise + published = json.loads(output(["gh", "release", "view", previous["tag"], "--repo", env["GITHUB_REPOSITORY"], "--json", "tagName,isDraft,isPrerelease"])) + if published["tagName"] != previous["tag"] or published["isDraft"] or published["isPrerelease"]: + raise ValueError("Upgrade baseline must be a published stable release") + matrices = {"windows": {"include": []}, "macos": {"include": []}} + for row in plan_transitions(previous, candidate, base): + transition = row["transition"] + name = f"acceptance-{row['target']}.json" + file = Path(env["RUNNER_TEMP"]) / name + file.write_text(json.dumps(transition), encoding="utf-8") + put(tag=tag, key=name, file=file, immutable=True) + url = f"{base}/releases/tag/{tag}/{name}" + if read_manifest(url) != transition: + raise ValueError("Transition manifest read-back mismatch") + matrices[transition["platform"]]["include"].append({"arch": transition["arch"], "manifest": url, "old": transition["old"]["tag"], "id": row["target"], "manifest_sha256": hashlib.sha256(file.read_bytes()).hexdigest()}) + emit(matrices, env) + + +def complete(env: dict) -> None: + from scripts.releases.r2 import put + + tag, commit = check_tag(env) + base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") + candidate = read_candidate(env) + validate_candidates(candidate, tag, commit, base) + file = Path(env["RUNNER_TEMP"]) / "release-candidates.json" + file.write_text(json.dumps(candidate), encoding="utf-8") + put(tag=tag, key="releases/stable/release-candidates.json", key_is_full=True, file=file) + if read_manifest(f"{base}/releases/stable/release-candidates.json") != candidate: + raise ValueError("Stable manifest read-back mismatch") + output(["gh", "release", "edit", tag, "--repo", env["GITHUB_REPOSITORY"], "--draft=false"]) + release = json.loads(output(["gh", "release", "view", tag, "--repo", env["GITHUB_REPOSITORY"], "--json", "isDraft"])) + if release["isDraft"]: + raise ValueError("Stable release remained a draft") + + +def main(argv: list[str] | None = None, env: dict | None = None) -> None: + argv = sys.argv[1:] if argv is None else argv + env = os.environ if env is None else env + if argv and argv[0] == "gate": + needs = json.loads(env["RELEASE_NEEDS"]) + summary("\n".join(f"- {name}: {needs.get(name, {}).get('result', 'missing')}" for name in argv[1:]), env) + require_success(needs, argv[1:]) + return + commands = {"admit": admit, "transitions": transitions, "complete": complete} + if len(argv) != 1 or argv[0] not in commands: + raise ValueError("Expected admit, gate, transitions or complete") + commands[argv[0]](env) + + +if __name__ == "__main__": + main() diff --git a/scripts/render-builds-table.py b/scripts/render-builds-table.py index 56beef91fb..81997bf4f1 100644 --- a/scripts/render-builds-table.py +++ b/scripts/render-builds-table.py @@ -3,7 +3,7 @@ Runs as the LAST job of desktop-bundled-release.yml, after every matrix leg has uploaded, and edits the GitHub release body in place. The tables -are built from the bucket's ACTUAL object names (scripts/r2-release.mjs +are built from the bucket's ACTUAL object names (scripts/releases/r2.py list --prefix releases/tag//), filtered to the tag's exact version — a missing artifact shows up as a missing row, never a dead link. The GitHub release carries the notes only; the binaries live in the R2 bucket @@ -128,11 +128,11 @@ def r2_object_names(tag: str) -> list[str]: Exact version match, never prefix: 'v0.28.0' must not pick up '0.28.0-canary.20260818...' objects (they live under their own tag directory, and the basename filter would reject them anyway). The list - call shells out to scripts/r2-release.mjs, which reads the R2 env vars - and needs only node (no npm ci in this job). + call shells out to scripts/releases/r2.py, which reads the R2 env vars + and needs only Python (no application environment). """ run = subprocess.run( - ["node", "scripts/r2-release.mjs", "list", "--prefix", f"releases/tag/{tag}/"], + [sys.executable, "-m", "scripts.releases.r2", "list", "--prefix", f"releases/tag/{tag}/"], capture_output=True, text=True, encoding="utf-8", errors="replace", ) if run.returncode != 0: diff --git a/scripts/stage-msixbundle.mjs b/scripts/stage-msixbundle.mjs index 6405a1c67a..3b30cec031 100644 --- a/scripts/stage-msixbundle.mjs +++ b/scripts/stage-msixbundle.mjs @@ -23,14 +23,13 @@ // node scripts/stage-msixbundle.mjs --tag vX.Y.Z [--variant bundled|light] // Reads HERMES_DESKTOP_VARIANT (bundled|light) from the environment; the // workflow runs this job once per variant. -import { createHash } from 'node:crypto' import { execFileSync } from 'node:child_process' import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' import { appIdentity, buildAppInstaller, resolveWinSdkTools } from './msix-shared.mjs' -import { publishFeedUploads } from './r2-release.mjs' + const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') @@ -65,7 +64,9 @@ if (process.platform !== 'win32') { process.exit(1) } +const candidate = args.includes('--candidate') const canary = /-canary\./.test(tag) +if (!canary && !candidate) throw new Error('Stable bundles must use the staged stable-release workflow') const channel = canary ? 'canary' : 'stable' const channelDir = `releases/win32/${variant === 'light' ? 'light/' : ''}${channel}` @@ -229,6 +230,12 @@ function resolveDotnetRuntimeDir() { return null } +if (candidate) { + execFileSync(signtool, ['verify', '/pa', bundle], { stdio: 'inherit' }) + console.log(`[stage-msixbundle] candidate ready: ${bundle}`) + process.exit(0) +} + // ── 2. .appinstaller + uploads ───────────────────────────────────────────── const baseUrl = String(process.env.CLOUDFLARE_R2_PUBLIC_URL || '').replace(/\/+$/, '') if (!baseUrl) { @@ -248,32 +255,24 @@ fs.writeFileSync(path.join(releaseDir, appinstallerName), appinstaller) const upload = (key, file, keyIsFull = true) => { // NOTE: no fs.readFileSync here — the msixbundle can exceed Node's 2GiB - // buffer limit (ERR_FS_FILE_TOO_LARGE). r2-release.mjs put reads + hashes + // buffer limit (ERR_FS_FILE_TOO_LARGE). scripts.releases.r2 put reads + hashes // the file itself; log the size via stat instead. const { size } = fs.statSync(file) console.log(`[stage-msixbundle] upload ${key} (${size} bytes)`) // Feed-dir keys are FULL object keys (releases/win32//…) — pass // --key-is-full so r2 put does NOT wrap them under releases/tag//. - // r2-release.mjs put derives Content-Type from the key extension. - execFileSync(process.execPath, ['scripts/r2-release.mjs', 'put', '--tag', tag, '--key', key, '--file', file, ...(keyIsFull ? ['--key-is-full'] : [])], { + // scripts.releases.r2 put derives Content-Type from the key extension. + execFileSync(process.env.HERMES_PYTHON || 'python', ['-m', 'scripts.releases.r2', 'put', '--tag', tag, '--key', key, '--file', file, ...(keyIsFull ? ['--key-is-full'] : [])], { cwd: REPO_ROOT, stdio: 'inherit' }) } -// C22 ordering: bundle FIRST, pointer LAST. r2-release.mjs PUTs then +// C22 ordering: bundle FIRST, pointer LAST. scripts.releases.r2 PUTs then // HEAD-verifies the remote content-length — a failed/short upload throws // and aborts this job before the pointer is written. -publishFeedUploads( - { - channelDir, - appinstallerName, - bundleFilename: `${name}-${version}-win.msixbundle`, - bundleFile: bundle, - appinstallerFile: path.join(releaseDir, appinstallerName), - }, - upload, -) +upload(`${channelDir}/${name}-${version}-win.msixbundle`, bundle) +upload(`${channelDir}/${appinstallerName}`, path.join(releaseDir, appinstallerName)) // The Store-submission .msix files were already uploaded to the tag archive // by the win legs (Store- prefix); nothing for this job to re-upload. diff --git a/scripts/termux/deb_version.py b/scripts/termux/deb_version.py index be75684fc2..6e2c02f4eb 100644 --- a/scripts/termux/deb_version.py +++ b/scripts/termux/deb_version.py @@ -25,7 +25,7 @@ import sys # The canary timestamp shape MUST match the canonical _CANARY_TAG_RE in # hermes_cli/update_channel.py (exactly 8 or 14 digits, 20-prefixed) and -# channelForTag in scripts/r2-release.mjs. Cross-referenced by +# channel_for_tag in scripts/releases/r2.py. Cross-referenced by # tests/test_termux_deb_version.py::test_canary_tag_shape_matches_canonical. _TAG_RE = re.compile( r"^v(?P0|[1-9]\d{0,2})\.(?P\d+)\.(?P\d+)" diff --git a/tests-js/bundle-inputs.test.mjs b/tests-js/bundle-inputs.test.mjs index 6f6e929ca6..7c20ee3b1e 100644 --- a/tests-js/bundle-inputs.test.mjs +++ b/tests-js/bundle-inputs.test.mjs @@ -64,7 +64,9 @@ test('staging streams actual bytes, verifies hashes and removes rejected partial manifest[slot].artifact.sha256 = createHash('sha256').update(bytes[slot]).digest('hex') } const out = path.join(directory, 'good') - const filename = await stageBundleInputs({ manifestUrl: `${base}/manifest.json`, platform: 'windows', arch: 'x64', out }) + const manifestSha256 = createHash('sha256').update(JSON.stringify(manifest)).digest('hex') + await expect(stageBundleInputs({ manifestUrl: `${base}/manifest.json`, platform: 'windows', arch: 'x64', out, manifestSha256: '0'.repeat(64) })).rejects.toThrow('manifest SHA-256') + const filename = await stageBundleInputs({ manifestUrl: `${base}/manifest.json`, platform: 'windows', arch: 'x64', out, manifestSha256 }) const result = JSON.parse(await readFile(filename, 'utf8')) for (const slot of ['old', 'new']) expect(await readFile(result[slot].artifact.path)).toEqual(bytes[slot]) manifest.old.artifact.sha256 = 'c'.repeat(64) diff --git a/tests-js/darwin-feed.test.mjs b/tests-js/darwin-feed.test.mjs deleted file mode 100644 index 0582d5192e..0000000000 --- a/tests-js/darwin-feed.test.mjs +++ /dev/null @@ -1,148 +0,0 @@ -import { createHash } from 'node:crypto' -import fs from 'node:fs' -import os from 'node:os' -import path from 'node:path' -import { afterEach, expect, it, vi } from 'vitest' -import yaml from 'js-yaml' -import { macFeedReferences, mergeMacFeeds, parseMacFeed, publishMacFeed } from '../scripts/darwin-feed.mjs' -import { cacheControlFor, cmdFinalize, cmdPrune, cmdPut } from '../scripts/r2-release.mjs' -import feedContract from '../apps/desktop/update-feed.cjs' - -function inputs(version = '0.28.0', light = false) { - const channel = version.includes('-canary.') ? 'canary' : 'stable' - const bytes = new Map() - const legs = Object.fromEntries(['arm64', 'x64'].map((arch, i) => { - const name = `${light ? 'HermesLight' : 'HermesBundled'}-${version}-mac-${arch}.zip` - const data = Buffer.from(`test artifact ${arch}`) - bytes.set(`releases/tag/v${version}/${name}`, data) - const file = { url: name, size: data.length, sha512: createHash('sha512').update(data).digest('base64') } - return [`${arch}-${channel}-mac.yml`, yaml.dump({ version, files: [file], path: name, sha512: file.sha512, releaseDate: `2026-09-0${i + 1}T00:00:00Z`, releaseNotes: 'two lines\nof release notes' })] - })) - return { legs, bytes } -} - -afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs() }) - -function credentials() { - for (const name of ['CLOUDFLARE_R2_ACCOUNT_ID', 'CLOUDFLARE_R2_ACCESS_KEY_ID', 'CLOUDFLARE_R2_SECRET_ACCESS_KEY', 'CLOUDFLARE_R2_BUCKET']) vi.stubEnv(name, 'fixture') -} - -it('never overwrites a published macOS artifact on a same-tag rerun', async () => { - credentials() - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mac-artifact-')) - const name = 'HermesBundled-0.28.0-mac-arm64.zip' - const file = path.join(dir, name) - const bytes = Buffer.from('already published artifact') - fs.writeFileSync(file, bytes) - vi.stubGlobal('fetch', vi.fn(async (_url, options) => { - if (options.method === 'PUT') { - expect(options.headers['If-None-Match']).toBe('*') - for await (const chunk of options.body) { expect(chunk.length).toBeGreaterThan(0) } - return new Response('', { status: 412 }) - } - if (options.method === 'HEAD') return new Response(null, { headers: { 'content-length': String(bytes.length) } }) - return new Response(bytes) - })) - try { await cmdPut({ tag: 'v0.28.0', key: name, file }) } - finally { fs.rmSync(dir, { recursive: true, force: true }) } -}) - -it('the real prune command protects live macOS ZIPs and blockmaps, and fails closed', async () => { - credentials() - const plan = mergeMacFeeds(inputs('0.28.0-canary.20200101000000').legs, 'v0.28.0-canary.20200101000000') - const references = macFeedReferences(plan.text) - const stale = 'releases/tag/v0.27.0-canary.20200101000000/unreferenced.zip' - const keys = [plan.key, ...references, stale] - const listing = `${keys.map(key => `${key}`).join('')}false` - const deleted = [] - let readable = true - vi.stubGlobal('fetch', vi.fn(async (url, options) => { - if (new URL(url).search) return new Response(listing) - if (options.method === 'DELETE') { deleted.push(decodeURIComponent(new URL(url).pathname)); return new Response('') } - return readable ? new Response(plan.text) : new Response('', { status: 503 }) - })) - await cmdPrune({ keepDays: 14, dryRun: false }) - expect(deleted).toEqual([`/fixture/${stale}`]) - deleted.length = 0 - readable = false - await expect(cmdPrune({ keepDays: 14, dryRun: false })).rejects.toThrow() - expect(deleted).toEqual([]) -}) - -it('merges native legs with different signatures/dates and preserves release metadata', () => { - const { legs } = inputs() - const plan = mergeMacFeeds(legs, 'v0.28.0') - const feed = parseMacFeed(plan.text) - expect(feed.files).toHaveLength(2) - expect(feed.releaseNotes).toBe('two lines\nof release notes') - expect(plan.key).toBe('releases/darwin/stable/stable-mac.yml') - expect(macFeedReferences(plan.text)).toEqual(feed.files.flatMap(file => [file.url.slice(1), `${file.url.slice(1)}.blockmap`])) - expect(cacheControlFor(plan.key)).toBe('no-store') - const selection = feedContract.darwinFeed('canary', true) - expect(mergeMacFeeds(inputs('0.29.0-canary.20260906000000', true).legs, 'v0.29.0-canary.20260906000000', true).key) - .toBe(`${selection.directory}/${selection.fileName}`) -}) - -it.each(['missing', 'version', 'variant', 'hash', 'legacy', 'traversal'])('rejects %s instead of publishing a partial or wrong feed', kind => { - const { legs } = inputs() - const key = 'arm64-stable-mac.yml' - const feed = yaml.load(legs[key]) - if (kind === 'missing') delete legs[key] - else { - if (kind === 'version') feed.version = '0.27.0' - if (kind === 'variant') feed.files[0].url = feed.files[0].url.replace('HermesBundled', 'HermesLight') - if (kind === 'hash') feed.files[0].sha512 = 'invalid' - if (kind === 'legacy') feed.sha512 = 'wrong' - if (kind === 'traversal') feed.files[0].url = '../other.zip' - legs[key] = yaml.dump(feed) - } - expect(() => mergeMacFeeds(legs, 'v0.28.0')).toThrow() -}) - -it('verifies all artifacts before a conditional pointer write and readback', async () => { - const plan = mergeMacFeeds(inputs().legs, 'v0.28.0') - let live = { text: mergeMacFeeds(inputs('0.27.0').legs, 'v0.27.0').text, etag: 'old' } - const events = [] - await publishMacFeed(plan, { - read: async () => live, - verify: async key => { events.push(key) }, - write: async (key, text, etag) => { expect(etag).toBe('old'); events.push(key); live = { text, etag: 'new' } } - }) - expect(events.at(-1)).toBe(plan.key) - expect(events).toHaveLength(3) - const write = vi.fn() - await expect(publishMacFeed(mergeMacFeeds(inputs('0.27.0').legs, 'v0.27.0'), { read: async () => live, verify: vi.fn(), write })).rejects.toThrow('backward') - await expect(publishMacFeed(plan, { read: async () => null, verify: async () => { throw new Error('corrupt bytes') }, write })).rejects.toThrow('corrupt bytes') - expect(write).not.toHaveBeenCalled() -}) - -it('finalize uses the real signed transport, validates streamed hashes and publishes last', async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'mac-feed-')) - const { legs, bytes } = inputs() - for (const [name, text] of Object.entries(legs)) fs.writeFileSync(path.join(dir, name), text) - for (const [key, value] of Object.entries({ CLOUDFLARE_R2_ACCOUNT_ID: 'fixture', CLOUDFLARE_R2_ACCESS_KEY_ID: 'fixture', CLOUDFLARE_R2_SECRET_ACCESS_KEY: 'fixture', CLOUDFLARE_R2_BUCKET: 'bucket' })) vi.stubEnv(key, value) - const calls = [] - let published - vi.stubGlobal('fetch', vi.fn(async (url, options) => { - const key = decodeURIComponent(new URL(url).pathname).replace('/bucket/', '') - const method = options.method || 'GET' - calls.push([method, key]) - expect(options.headers.authorization).toContain('AWS4-HMAC-SHA256') - if (method === 'PUT') { - expect(options.headers['If-None-Match']).toBe('*') - expect(options.headers['Cache-Control']).toBe('no-store') - published = options.body.toString() - return new Response('', { status: 200 }) - } - if (method === 'HEAD') return new Response(null, { headers: { 'content-length': Buffer.byteLength(published).toString() } }) - if (key.endsWith('-mac.yml')) return published ? new Response(published, { headers: { etag: 'new' } }) : new Response('', { status: 404 }) - if (bytes.has(key)) return new Response(bytes.get(key)) - throw new Error(`unexpected request ${key}`) - })) - try { - await cmdFinalize({ tag: 'v0.28.0', dir }) - expect(parseMacFeed(published).files).toHaveLength(2) - expect(calls.filter(([method]) => method === 'PUT')).toHaveLength(1) - expect(calls.slice(0, 3).every(([method]) => method === 'GET')).toBe(true) - } finally { fs.rmSync(dir, { recursive: true, force: true }) } -}) diff --git a/tests-js/r2-release.test.mjs b/tests-js/r2-release.test.mjs deleted file mode 100644 index 3b1d49cc1a..0000000000 --- a/tests-js/r2-release.test.mjs +++ /dev/null @@ -1,459 +0,0 @@ -/** - * scripts/r2-release.mjs — SigV4 signer pinned against AWS botocore (the - * reference implementation, 1.43.81) at a FIXED timestamp/creds, so the - * expected values are reproducible fixtures rather than self-consistency. - * - * Vectors were generated with the venv script at - * sigv4-venv/gen_vectors.py (botocore.auth.SigV4Auth / S3SigV4Auth, - * timestamp 20150830T123600Z, creds AKIDEXAMPLE): - * - get-vanilla generic signer (no x-amz-content-sha256), example.com - * - r2-put-payload S3 signer, payload hash signed, region auto - * - r2-list S3 signer, ListObjectsV2 with query params - * - r2-delete S3 signer, region auto - * The get-vanilla case also reproduces the public aws-sig-v4-test-suite - * request shape (verified by independent spec computation). - */ - -import assert from 'node:assert/strict' - -import { test, vi } from 'vitest' - -import { - authHeader, - canonicalQuery, - canonicalRequest, - channelForTag, - contentTypeFor, - cacheControlFor, - encodeKeyPath, - feedDirFor, - feedReferencedKeys, - canaryDoomedKeys, - publishFeedUploads, - referencedFeedBundleFilenames, - staleFeedBundleKeys, - stagingKeyFor, - parseListXml, - rfc3986Encode, -} from '../scripts/r2-release.mjs' - -const AKID = 'AKIDEXAMPLE' -const SECRET = 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY' -const NOW = '20150830T123600Z' -const EMPTY_SHA = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' - -test('get-vanilla matches the AWS test-suite vector', () => { - const authz = authHeader({ - method: 'GET', - host: 'example.com', - path: '/', - query: '', - headers: { host: 'example.com', 'x-amz-date': NOW }, - payloadHash: EMPTY_SHA, - accessKeyId: AKID, - secretKey: SECRET, - now: NOW, - region: 'us-east-1', - service: 'service', - }) - assert.equal( - authz, - 'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, ' + - 'SignedHeaders=host;x-amz-date, ' + - 'Signature=33399fd3d4a9d6104710c7c04005f7c959f8b1f8bf41b823587ed36b079e453f', - ) -}) - -test('r2-put-payload matches botocore (S3 signer, payload hash, region auto)', () => { - const bodyHash = '44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072' // sha256("Welcome to Amazon S3.") - const host = 'abc123.r2.cloudflarestorage.com' - const authz = authHeader({ - method: 'PUT', - host, - path: '/hermes-releases/HermesBundled-0.28.0-win-x64.msix', - query: '', - headers: { host, 'x-amz-date': NOW, 'x-amz-content-sha256': bodyHash }, - payloadHash: bodyHash, - accessKeyId: AKID, - secretKey: SECRET, - now: NOW, - }) - assert.equal( - authz, - 'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, ' + - 'SignedHeaders=host;x-amz-content-sha256;x-amz-date, ' + - 'Signature=05ba50acfb54042fac330848af50877e5fb477c4f2063c2f77f9cc80855eb1e9', - ) -}) - -test('r2-list matches botocore (canonical query sorted, empty payload hash)', () => { - const query = canonicalQuery({ 'list-type': '2', prefix: 'HermesBundled-0.28.0-', 'max-keys': '1000' }) - assert.equal(query, 'list-type=2&max-keys=1000&prefix=HermesBundled-0.28.0-') - const host = 'abc123.r2.cloudflarestorage.com' - const authz = authHeader({ - method: 'GET', - host, - path: '/hermes-releases', - query, - headers: { host, 'x-amz-date': NOW, 'x-amz-content-sha256': EMPTY_SHA }, - payloadHash: EMPTY_SHA, - accessKeyId: AKID, - secretKey: SECRET, - now: NOW, - }) - assert.equal( - authz, - 'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, ' + - 'SignedHeaders=host;x-amz-content-sha256;x-amz-date, ' + - 'Signature=3ec423c452a318664c85fbcc25667ad07201aedce688e3bb6b345b4baaa39d90', - ) -}) - -test('r2-delete matches botocore', () => { - const host = 'abc123.r2.cloudflarestorage.com' - const authz = authHeader({ - method: 'DELETE', - host, - path: '/hermes-releases/HermesBundled-0.28.0-canary.20260818-win-arm64.msix', - query: '', - headers: { host, 'x-amz-date': NOW, 'x-amz-content-sha256': EMPTY_SHA }, - payloadHash: EMPTY_SHA, - accessKeyId: AKID, - secretKey: SECRET, - now: NOW, - }) - assert.equal( - authz, - 'AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, ' + - 'SignedHeaders=host;x-amz-content-sha256;x-amz-date, ' + - 'Signature=ec5ccb76f701193b28aaca052cabbf2f084e9c71ffc64b872fa51d4e70dc6e55', - ) -}) - -test('rfc3986Encode escapes the AWS reserved set, keeps unreserved', () => { - assert.equal(rfc3986Encode('HermesBundled-0.28.0-win-x64.msix'), 'HermesBundled-0.28.0-win-x64.msix') - assert.equal(rfc3986Encode("a b!'()*c"), 'a%20b%21%27%28%29%2Ac') -}) - -test('encodeKeyPath encodes segment-wise, preserves separators', () => { - assert.equal(encodeKeyPath('HermesBundled-0.28.0-win-x64.msix'), 'HermesBundled-0.28.0-win-x64.msix') - assert.equal(encodeKeyPath('a b/c d'), 'a%20b/c%20d') -}) - -test('parseListXml extracts keys, truncation, continuation token, entities', () => { - const xml = ` - - hermes-releases - - 3 - 1000 - true - HermesBundled-0.28.0-win-x64.msix2026-08-18T00:00:00Z123 - a&b.msix2026-08-18T00:00:00Z1 - latest.yml2026-08-18T00:00:00Z2 - abc+def/= -` - const parsed = parseListXml(xml) - assert.deepEqual(parsed.keys, ['HermesBundled-0.28.0-win-x64.msix', 'a&b.msix', 'latest.yml']) - assert.equal(parsed.truncated, true) - assert.equal(parsed.nextToken, 'abc+def/=') -}) - -test('canaryDoomedKeys dates by the key suffix, ignores stable artifacts', () => { - const keys = [ - 'releases/tag/v0.28.0/HermesBundled-0.28.0-win-x64.msix', // stable — never doomed - 'releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix', - 'releases/tag/v0.28.0-canary.20260818/HermesBundled-0.28.0-canary.20260818-win-x64.msix', // today — kept - 'releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix.blockmap', - 'latest.yml', - 'canary.yml', - ] - assert.deepEqual(canaryDoomedKeys(keys, '20260814'), [ - 'releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix', - 'releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix.blockmap', - ]) -}) - -test('channelForTag maps stable vs canary', () => { - assert.equal(channelForTag('v0.28.0'), 'stable') - assert.equal(channelForTag('v0.28.0-canary.20260818101010'), 'canary') - assert.equal(channelForTag('v0.28.0-canary.20260818'), 'canary') -}) - -test('stagingKeyFor + feedDirFor produce the layout keys', () => { - assert.equal(stagingKeyFor('v0.28.0', 'HermesBundled-0.28.0-win-x64.msix'), - 'releases/tag/v0.28.0/HermesBundled-0.28.0-win-x64.msix') - assert.equal(feedDirFor('win32', 'stable'), 'releases/win32/stable') - assert.equal(feedDirFor('darwin', 'canary'), 'releases/darwin/canary') -}) - -test('contentTypeFor maps MSIX / App Installer artifacts to their MIME types', () => { - assert.equal(contentTypeFor('HermesBundled-0.28.0-win-x64.msix'), 'application/msix') - assert.equal(contentTypeFor('HermesBundled-0.28.0-win.msixbundle'), 'application/msixbundle') - assert.equal(contentTypeFor('stable.appinstaller'), 'application/appinstaller') - assert.equal(contentTypeFor('HermesBundled-0.28.0-mac-x64.dmg'), undefined) - assert.equal(contentTypeFor('latest-mac.yml'), undefined) - // Case-insensitive on the suffix. - assert.equal(contentTypeFor('X.APPINSTALLER'), 'application/appinstaller') -}) - -test('APT mutable metadata revalidates while immutable index bytes can cache', () => { - const feed = 'releases/termux/canary' - for (const name of ['key.asc', 'dists/hermes-canary/InRelease', 'dists/hermes-canary/Release', 'dists/hermes-canary/main/binary-aarch64/Packages.gz']) { - assert.equal(cacheControlFor(`${feed}/${name}`), 'no-store') - } - assert.equal(cacheControlFor(`${feed}/dists/hermes-canary/main/binary-aarch64/by-hash/SHA256/abcd`), 'public, max-age=31536000, immutable') - assert.equal(cacheControlFor(`${feed}/pool/h/hermes-agent_1.2.3_aarch64.deb`), 'public, max-age=31536000, immutable') - assert.equal(cacheControlFor('releases/win32/stable/stable.appinstaller'), undefined) -}) - -test('canonicalRequest reads mixed-case header values (Content-Type)', () => { - // Regression: canonicalRequest lowercased the header NAME for the canonical - // line but read the value with `headers[lowerName]` — a 'Content-Type' - // value came out as 'undefined' while R2 canonicalized 'application/msix', - // so every signed msix PUT (the only artifact with Content-Type) failed - // with SignatureDoesNotMatch / 403. Value lookup must be case-insensitive. - const host = 'abc123.r2.cloudflarestorage.com' - const now = '20150830T123600Z' - const bodyHash = '44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072' - const headers = { - host, - 'x-amz-date': now, - 'x-amz-content-sha256': bodyHash, - 'Content-Type': 'application/msix' - } - const canon = canonicalRequest('PUT', '/hermes-releases/HermesBundled-0.28.0-win-x64.msix', '', headers, bodyHash) - assert.ok(canon.includes('content-type:application/msix'), 'content-type value must survive canonicalization') - assert.ok(!canon.includes('undefined'), 'no undefined values leaked into the canonical request') - // The canonical line ordering + header list match what SigV4/R2 recompute. - assert.ok(canon.includes('content-type;host;x-amz-content-sha256;x-amz-date')) -}) - -// ── C22: artifact first, feed pointer last ────────────────────────────────── - -test('publishFeedUploads uploads the msixbundle BEFORE the .appinstaller pointer', () => { - const calls = [] - publishFeedUploads( - { - channelDir: 'releases/win32/canary', - appinstallerName: 'canary.appinstaller', - bundleFilename: 'HermesBundled-0.27.2.9-win.msixbundle', - bundleFile: 'C:/rel/HermesBundled-0.27.2.9-win.msixbundle', - appinstallerFile: 'C:/rel/canary.appinstaller', - }, - (key, file) => calls.push([key, file]), - ) - assert.deepEqual(calls, [ - ['releases/win32/canary/HermesBundled-0.27.2.9-win.msixbundle', 'C:/rel/HermesBundled-0.27.2.9-win.msixbundle'], - ['releases/win32/canary/canary.appinstaller', 'C:/rel/canary.appinstaller'], - ]) -}) - -test('publishFeedUploads never writes the pointer when the bundle upload fails', () => { - const calls = [] - assert.throws(() => - publishFeedUploads( - { - channelDir: 'releases/win32/stable', - appinstallerName: 'stable.appinstaller', - bundleFilename: 'HermesBundled-0.28.0.0-win.msixbundle', - bundleFile: 'bundle', - appinstallerFile: 'feed', - }, - (key) => { - calls.push(key) - throw new Error('R2 PUT -> 503') - }, - ), - ) - assert.deepEqual(calls, ['releases/win32/stable/HermesBundled-0.28.0.0-win.msixbundle']) -}) - -// ── C22: canary feed-dir retention (fail-closed, keep-days grace) ─────────── - -const CANARY_FEED_XML = ` - - - -` - -test('referencedFeedBundleFilenames reads MainPackage only; unrecognized -> []', () => { - assert.deepEqual(referencedFeedBundleFilenames(CANARY_FEED_XML), ['HermesBundled-0.27.2.9-win.msixbundle']) - // The AppInstaller ROOT Uri (the feed pointer itself) must NOT count. - assert.ok(!referencedFeedBundleFilenames(CANARY_FEED_XML).includes('canary.appinstaller')) - assert.deepEqual(referencedFeedBundleFilenames(''), []) - assert.deepEqual(referencedFeedBundleFilenames('ServiceUnavailable'), []) - // A bundle Uri OUTSIDE MainPackage/MainBundle is not a reference. - assert.deepEqual(referencedFeedBundleFilenames(''), []) -}) - -test('feedReferencedKeys protects the referenced bundle and absolute tag Uris by exact key', () => { - const tagFeed = CANARY_FEED_XML.replace( - /Uri="https:\/\/r2\.example\/releases\/win32\/canary\/HermesBundled-0\.27\.2\.9-win\.msixbundle"/, - 'Uri="https://r2.example/releases/tag/v0.27.2-canary.20260829/HermesBundled-0.27.2-win-x64.msix"', - ) - const keys = feedReferencedKeys('releases/win32/canary', tagFeed) - assert.ok(keys.includes('releases/win32/canary/HermesBundled-0.27.2-win-x64.msix')) - assert.ok(keys.includes('releases/tag/v0.27.2-canary.20260829/HermesBundled-0.27.2-win-x64.msix')) -}) - -const CANARY_DIR = 'releases/win32/canary' -const OLD_MS = Date.parse('2026-08-01T00:00:00Z') -const FRESH_MS = Date.parse('2026-09-03T00:00:00Z') -const CUTOFF_MS = Date.parse('2026-08-21T00:00:00Z') - -function canaryKeys(extra = []) { - return [ - `${CANARY_DIR}/canary.appinstaller`, - `${CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle`, // referenced - `${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`, // stale - `${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`, // uploaded, not yet pointed - 'releases/win32/stable/stable.appinstaller', - 'releases/win32/stable/HermesBundled-0.28.0.0-win.msixbundle', // referenced - 'releases/win32/stable/HermesBundled-0.27.0.0-win.msixbundle', // stale stable - ...extra, - ] -} - -function lastModifiedFor(keys, overrides = {}) { - const lm = {} - for (const k of keys) lm[k] = OLD_MS - return Object.assign(lm, overrides) -} - -test('staleFeedBundleKeys: old unreferenced canary bundle doomed, referenced and fresh ones kept', () => { - const keys = canaryKeys() - const lm = lastModifiedFor(keys, { - // Uploaded-but-not-yet-pointed canary bundle is FRESH -> survives grace. - [`${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`]: FRESH_MS, - }) - const doomed = staleFeedBundleKeys(keys, { [CANARY_DIR]: [CANARY_FEED_XML] }, lm, CUTOFF_MS) - assert.deepEqual(doomed, [`${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`]) -}) - -test('staleFeedBundleKeys: stable dirs are never pruned', () => { - const keys = canaryKeys() - const lm = lastModifiedFor(keys, { - // Uploaded-but-not-yet-pointed bundle is fresh here; its fate is covered - // by the dedicated grace test. - [`${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`]: FRESH_MS, - }) - const feeds = { - [CANARY_DIR]: [CANARY_FEED_XML], - 'releases/win32/stable': [CANARY_FEED_XML.replace(/0\.27\.2\.9/g, '0.28.0.0').replace('HermesBundled-0.27.2.9-win', 'HermesBundled-0.28.0.0-win')], - } - const doomed = staleFeedBundleKeys(keys, feeds, lm, CUTOFF_MS) - assert.deepEqual(doomed, [`${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`]) -}) - -test('staleFeedBundleKeys: empty/malformed manifest dooms NOTHING in its dir (fail closed)', () => { - const keys = canaryKeys() - for (const bad of ['', 'boom', null, '', - '', - '', - ]) { - assert.deepEqual( - staleFeedBundleKeys(keys, { [CANARY_DIR]: [bad] }, lastModifiedFor(keys), CUTOFF_MS), - [], - `bad manifest must block the dir: ${JSON.stringify(bad)}`, - ) - } -}) - -test('staleFeedBundleKeys: two manifests in one dir — union protected, one bad blocks all', () => { - const second = CANARY_FEED_XML.replace(/0\.27\.2\.9/g, '0.27.3.0').replace( - 'HermesBundled-0.27.2.9-win', - 'HermesBundled-0.27.3.0-win', - ) - const keys = canaryKeys([`${CANARY_DIR}/second.appinstaller`, `${CANARY_DIR}/HermesBundled-0.27.3.0-win.msixbundle`]) - const lm = lastModifiedFor(keys, { [`${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`]: FRESH_MS }) - // Union of both feeds: both referenced bundles kept, the rest pruned. - assert.deepEqual(staleFeedBundleKeys(keys, { [CANARY_DIR]: [CANARY_FEED_XML, second] }, lm, CUTOFF_MS), [ - `${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`, - ]) - // ONE unreadable/unrecognized manifest in the dir blocks feed retention. - assert.deepEqual(staleFeedBundleKeys(keys, { [CANARY_DIR]: [CANARY_FEED_XML, null] }, lm, CUTOFF_MS), []) - assert.deepEqual(staleFeedBundleKeys(keys, { [CANARY_DIR]: [CANARY_FEED_XML, ''] }, lm, CUTOFF_MS), []) -}) - -test('staleFeedBundleKeys: missing LastModified metadata keeps the object (fail closed)', () => { - const keys = canaryKeys() - for (const unknown of [undefined, NaN, Infinity]) { - const metadata = Object.fromEntries(keys.map(key => [key, unknown])) - const doomed = staleFeedBundleKeys(keys, { [CANARY_DIR]: [CANARY_FEED_XML] }, metadata, CUTOFF_MS) - assert.deepEqual(doomed, []) - } -}) - -test('prune-canaries --dry-run via main(): controlled clock, env restored, no DELETEs', async () => { - const { main } = await import('../scripts/r2-release.mjs') - // Clock: 2026-09-04, keep-days 14 -> cutoff 2026-08-21. - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-09-04T00:00:00Z')) - const STABLE_FEED = CANARY_FEED_XML.replace(/0\.27\.2\.9/g, '0.28.0.0').replace( - 'releases/win32/canary/canary.appinstaller', - 'releases/win32/stable/stable.appinstaller', - ) - const bucketKeys = [ - ['releases/tag/v0.27.2-canary.20260801034013/HermesBundled-0.27.2-canary.20260801034013-win-x64.msix', OLD_MS], - [`${CANARY_DIR}/canary.appinstaller`, OLD_MS], - [`${CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle`, OLD_MS], - [`${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`, OLD_MS], - [`${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`, FRESH_MS], - ['releases/win32/stable/stable.appinstaller', OLD_MS], - ['releases/win32/stable/HermesBundled-0.27.0.0-win.msixbundle', OLD_MS], - ] - const BUCKET_XML = ` -false -${bucketKeys.map(([k, t]) => ` ${k}${new Date(t).toISOString()}`).join('\n')} -` - const deletes = [] - const realFetch = globalThis.fetch - globalThis.fetch = async (url, init = {}) => { - const u = String(url) - if (init.method === 'DELETE') { - deletes.push(u) - return { ok: true, status: 204, headers: new Map(), text: async () => '' } - } - if (u.includes('list-type=2')) return { ok: true, status: 200, headers: new Map(), text: async () => BUCKET_XML } - if (u.endsWith('/canary.appinstaller')) return { ok: true, status: 200, headers: new Map(), text: async () => CANARY_FEED_XML } - if (u.endsWith('/stable.appinstaller')) return { ok: true, status: 200, headers: new Map(), text: async () => STABLE_FEED } - throw new Error(`unexpected fetch ${init.method ?? 'GET'} ${u}`) - } - const logs = [] - const origLog = console.log - const origWarn = console.warn - console.log = (...a) => logs.push(a.join(' ')) - console.warn = (...a) => logs.push('WARN ' + a.join(' ')) - for (const [k, v] of Object.entries({ - CLOUDFLARE_R2_ACCOUNT_ID: 'abc123', - CLOUDFLARE_R2_ACCESS_KEY_ID: 'AKIDEXAMPLE', - CLOUDFLARE_R2_SECRET_ACCESS_KEY: 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY', - CLOUDFLARE_R2_BUCKET: 'hermes-releases', - })) vi.stubEnv(k, v) - try { - await main(['prune-canaries', '--keep-days', '14', '--dry-run']) - } finally { - console.log = origLog - console.warn = origWarn - globalThis.fetch = realFetch - vi.unstubAllEnvs() - vi.useRealTimers() - } - // Doomed: the old tag-archive canary + the old UNREFERENCED canary bundle. - assert.ok(logs.some((l) => l.includes('would delete r2:releases/tag/v0.27.2-canary.20260801034013/'))) - assert.ok(logs.some((l) => l.includes(`would delete r2:${CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle`))) - // Survivors: referenced bundle (old but referenced), fresh not-yet-pointed - // bundle (keep-days grace), stable dir (never pruned), pointers. - for (const keep of [ - `r2:${CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle`, - `r2:${CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle`, - 'r2:releases/win32/stable/HermesBundled-0.27.0.0-win.msixbundle', - ]) { - assert.ok(!logs.some((l) => l.includes(`would delete ${keep}`)), `must survive: ${keep}`) - } - assert.ok(!logs.some((l) => l.includes('would delete') && l.includes('.appinstaller'))) - assert.deepEqual(deletes, [], 'dry-run must not issue any DELETE') - // Env restoration: stubEnv values were unstubbed (none pre-existed here). - assert.equal(process.env.CLOUDFLARE_R2_ACCOUNT_ID, undefined) -}) diff --git a/tests/ci/test_desktop_release_tag_admission.py b/tests/ci/test_desktop_release_tag_admission.py index 84a398b007..9aec958937 100644 --- a/tests/ci/test_desktop_release_tag_admission.py +++ b/tests/ci/test_desktop_release_tag_admission.py @@ -171,6 +171,9 @@ def _run_admission(clone: Path, tag: str) -> subprocess.CompletedProcess: env = _child_env( TAG=tag, GITHUB_OUTPUT=str(gh_output), + RELEASE_PHASE="" if "-canary." in tag else "candidate", + GITHUB_REF=f"refs/tags/{tag}", + GITHUB_SHA=_git("rev-parse", "HEAD", cwd=clone), # checkout@v6 runs run-steps with `bash -e -o pipefail`; -e/-o are on # the command line below, so nothing else is needed from the env. ) diff --git a/tests/ci/test_required_results.py b/tests/ci/test_required_results.py new file mode 100644 index 0000000000..4b96feaed3 --- /dev/null +++ b/tests/ci/test_required_results.py @@ -0,0 +1,200 @@ +"""Behavior tests for the strict CI aggregate gate (scripts/ci/required_results.py).""" + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "required_results.py" + +_spec = importlib.util.spec_from_file_location("required_results", SCRIPT) +required_results = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(required_results) + +evaluate_gate = required_results.evaluate_gate +PR_ONLY_JOBS = required_results.PR_ONLY_JOBS +DEFERRED_JOBS = required_results.DEFERRED_JOBS +EXCLUDED_JOBS = required_results.EXCLUDED_JOBS + + +def needs_of(entries): + """Build the needs-context shape ci.yaml's all-checks-pass receives.""" + return {name: {"result": result} for name, result in entries} + + +def test_non_release_failure_fails_gate_skip_counts_as_success(): + base = [("detect", "success"), ("tests", "success"), ("lint", "success")] + + assert evaluate_gate(needs_of(base))["ok"] is True + + with_failure = evaluate_gate(needs_of([*base, ("js-tests", "failure")])) + assert with_failure["ok"] is False + assert with_failure["failed"] == ["js-tests"] + + # Skipped lanes are fine when not releasing (e.g. docs-only PR). + with_skips = evaluate_gate( + needs_of([("detect", "success"), ("tests", "skipped"), ("rust-tests", "skipped"), ("lint", "success")]) + ) + assert with_skips["ok"] is True + assert with_skips["allowed_skips"] == ["rust-tests", "tests"] + + +def test_exclusion_sets_name_exactly_pr_only_and_deferred_jobs(): + assert PR_ONLY_JOBS == ("history-check", "lockfile-diff", "supply-chain", "review-labels") + assert DEFERRED_JOBS == ("e2e-desktop",) + assert EXCLUDED_JOBS == set(PR_ONLY_JOBS) | set(DEFERRED_JOBS) + + +def test_release_only_excluded_jobs_may_skip(): + needs = needs_of( + [ + ("detect", "success"), + ("tests", "success"), + ("js-tests", "success"), + ("osv-scanner", "success"), + # PR-only: never ran — release runs are push/tag events. + ("history-check", "skipped"), + ("lockfile-diff", "skipped"), + ("supply-chain", "skipped"), + ("review-labels", "skipped"), + # Deferred: Desktop E2E stays disabled. + ("e2e-desktop", "skipped"), + ] + ) + verdict = evaluate_gate(needs, release=True) + assert verdict["ok"] is True + assert verdict["allowed_skips"] == ["e2e-desktop", "history-check", "lockfile-diff", "review-labels", "supply-chain"] + + +def test_release_osv_skipped_is_a_failure_not_advisory(): + # Advisory FINDINGS do not make the EXECUTION advisory: the OSV scan + # must run to success on a release candidate. + verdict = evaluate_gate(needs_of([("detect", "success"), ("osv-scanner", "skipped")]), release=True) + assert verdict["ok"] is False + assert verdict["failed"] == ["osv-scanner"] + + +def test_release_skipped_required_job_fails_gate(): + needs = needs_of( + [ + ("detect", "success"), + ("tests", "success"), + # Everything tagged python (and python_prod/frontend) skipped on + # the release run: the strict gate must not wave this through. + ("tests-os", "skipped"), + ("lint", "skipped"), + ("js-tests", "skipped"), + ("infographic-check", "success"), + ] + ) + verdict = evaluate_gate(needs, release=True) + assert verdict["ok"] is False + assert verdict["failed"] == ["js-tests", "lint", "tests-os"] + + +def test_every_non_success_result_fails_in_release_mode(): + # cancelled / with-status / action_required / unknown / missing all + # fail; only `success` passes and only `skipped` on an excluded job + # is tolerated. + needs = needs_of( + [ + ("a", "cancelled"), + ("b", "with-status"), + ("c", "action_required"), + ("d", "unknown"), + ] + ) + verdict = evaluate_gate(needs, release=True) + assert verdict["ok"] is False + assert verdict["failed"] == ["a", "b", "c", "d"] + + # A needs entry with no result key at all (should never happen, but + # must not silently pass). + verdict = evaluate_gate({"x": {}}, release=True) + assert verdict["ok"] is False + assert verdict["failed"] == ["x"] + + +def test_release_failure_fails_gate_even_beside_allowed_skips(): + needs = needs_of( + [ + ("detect", "success"), + ("tests", "failure"), + ("history-check", "skipped"), + ("e2e-desktop", "skipped"), + ] + ) + verdict = evaluate_gate(needs, release=True) + assert verdict["ok"] is False + assert verdict["failed"] == ["tests"] + + +def test_empty_needs_fails_closed(): + assert evaluate_gate({}, release=True)["ok"] is False + assert evaluate_gate({}, release=False)["ok"] is False + assert evaluate_gate(None, release=True)["ok"] is False + + +def test_release_full_pipeline_all_green_passes(): + # What ci.yaml's all-checks-pass actually needs. A release run forces + # the classifier lanes true, so a healthy candidate looks like this. + all_green = [ + "detect", "tests", "tests-os", "lint", "js-tests", "installer-tests", + "rust-tests", "bootstrap-installer", "e2e-desktop", "docs-site", + "history-check", "contributor-check", "uv-lockfile", "infographic-check", + "case-collision-check", "lazy-deps-guard", "lockfile-diff", + "docker-lint", "profile-artifact-check", "icons-freshness-check", + "supply-chain", "review-labels", "osv-scanner", + ] + needs = needs_of((name, "success") for name in all_green) + assert evaluate_gate(needs, release=True)["ok"] is True + assert evaluate_gate(needs, release=False)["ok"] is True + + +def test_compact_results_maps_every_job_to_its_result(): + needs = needs_of([("detect", "success"), ("tests", "skipped")]) + assert required_results.compact_results(needs) == {"detect": "success", "tests": "skipped"} + assert required_results.compact_results(None) == {} + + +def test_render_report_reports_honest_skip_counts(): + needs = needs_of([("tests", "skipped"), ("lint", "failure"), ("detect", "success")]) + verdict = evaluate_gate(needs, release=True) + lines = "\n".join(required_results.render_report(needs, verdict)) + assert "⏭️ tests: skipped" in lines + assert "❌ lint: failure" in lines + assert "::error::2 job(s) failed: lint, tests" in lines + + +def test_cli_stdin_exit_codes_and_output(tmp_path): + gh_output = tmp_path / "github-output.txt" + green = needs_of([("detect", "success"), ("tests", "success"), ("e2e-desktop", "skipped")]) + + def run(needs, release, env_file): + # Inherit the full environment: on Windows a stripped env breaks + # subprocess startup (SystemRoot etc.). + env = dict(os.environ) + env.pop("GITHUB_OUTPUT", None) + if env_file is not None: + env["GITHUB_OUTPUT"] = str(env_file) + return subprocess.run( + [sys.executable, str(SCRIPT)] + (["--release"] if release else []), + input=json.dumps(needs), + capture_output=True, + text=True, + env=env, + ) + + ok = run(green, release=True, env_file=gh_output) + assert ok.returncode == 0 + assert "All checks passed" in ok.stdout + assert "needs-json=" in ok.stdout + assert "needs-json=" in gh_output.read_text(encoding="utf-8") + + bad = run(needs_of([("tests", "skipped")]), release=True, env_file=None) + assert bad.returncode == 1 + + failed = run(needs_of([("tests", "failure")]), release=False, env_file=None) + assert failed.returncode == 1 diff --git a/tests/ci/test_stable_release_graph.py b/tests/ci/test_stable_release_graph.py new file mode 100644 index 0000000000..6741370ebc --- /dev/null +++ b/tests/ci/test_stable_release_graph.py @@ -0,0 +1,49 @@ +"""The release workflow's dependency graph enforces publication ordering.""" +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] + + +def workflow(name): + return yaml.load((ROOT / ".github/workflows" / name).read_text(encoding="utf-8"), Loader=yaml.BaseLoader) + + +def ancestors(jobs, name): + seen = set() + pending = [name] + while pending: + needs = jobs[pending.pop()].get("needs", []) + for item in [needs] if isinstance(needs, str) else needs: + if item not in seen: + seen.add(item) + pending.append(item) + return seen + + +def test_release_reuses_whole_ci_and_docker_before_publication(): + jobs = workflow("stable-release.yml")["jobs"] + assert jobs["ci"]["uses"] == "./.github/workflows/ci.yaml" + assert jobs["ci"]["with"]["release"] == "true" + assert "secrets" not in jobs["ci"] + assert jobs["docker"]["uses"] == jobs["publish-docker"]["uses"] == jobs["promote-docker"]["uses"] + assert jobs["docker"]["with"]["release-phase"] == "test" + assert "ci" in ancestors(jobs, "docker") + required = {"ci", "docker", "nix", "pm-bundle", "install-e2e", "windows-packaged", "macos-packaged", "termux-checks", "windows-live", "candidates"} + assert required <= ancestors(jobs, "acceptance") + for name in ("publish-docker", "publish-bundles"): + assert required <= ancestors(jobs, name) + for name in ("promote-docker", "promote-bundles"): + assert {"publish-docker", "publish-bundles", "publication"} <= ancestors(jobs, name) + assert {"promote-docker", "promote-bundles"} <= ancestors(jobs, "complete") + for name in ("acceptance", "publication", "complete"): + assert jobs[name]["if"] == "always()" + + +def test_all_applicable_ci_jobs_are_aggregated_and_desktop_e2e_stays_deferred(): + jobs = workflow("ci.yaml")["jobs"] + checks = {name for name, job in jobs.items() if "uses" in job} + assert checks <= set(jobs["all-checks-pass"]["needs"]) + assert jobs["e2e-desktop"]["if"] == "false" + assert "workflow_call" in workflow("ci.yaml")["on"] diff --git a/tests/install/e2e-assets/bundle-inputs.mjs b/tests/install/e2e-assets/bundle-inputs.mjs index f0c55baa8d..8c867813d1 100644 --- a/tests/install/e2e-assets/bundle-inputs.mjs +++ b/tests/install/e2e-assets/bundle-inputs.mjs @@ -94,11 +94,14 @@ export async function downloadArtifact(artifact, destination) { } } -export async function stageBundleInputs({ manifestUrl, platform, arch, out, expectedCommit }) { +export async function stageBundleInputs({ manifestUrl, platform, arch, out, expectedCommit, manifestSha256 }) { const response = await fetch(artifactUrl(manifestUrl), { signal: AbortSignal.timeout(60_000) }) if (!response.ok) throw new Error(`Manifest download returned HTTP ${response.status}`) const text = await response.text() if (text.length > 1024 * 1024) throw new Error('Bundle input manifest is too large') + if (manifestSha256 && (!SHA256.test(manifestSha256) || createHash('sha256').update(text).digest('hex') !== manifestSha256)) { + throw new Error('Transition manifest SHA-256 mismatch') + } const manifest = validateBundleInputs(JSON.parse(text), platform, arch) if (expectedCommit && manifest.new.commit !== expectedCommit) { throw new Error('Candidate commit must equal the tested workflow SHA') @@ -124,5 +127,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.ar throw new Error('--manifest-url, --platform, --arch and --out are required') } console.log(await stageBundleInputs({ manifestUrl: values['manifest-url'], platform: values.platform, arch: values.arch, out: values.out, - expectedCommit: process.env.GITHUB_ACTIONS === 'true' ? process.env.GITHUB_SHA : undefined })) + expectedCommit: process.env.GITHUB_ACTIONS === 'true' ? process.env.GITHUB_SHA : undefined, + manifestSha256: process.env.BUNDLE_MANIFEST_SHA256 || undefined })) } diff --git a/tests/install/e2e-assets/mac-bundled-feed.mjs b/tests/install/e2e-assets/mac-bundled-feed.mjs index 26c9ec94b4..5b0d7c59f6 100644 --- a/tests/install/e2e-assets/mac-bundled-feed.mjs +++ b/tests/install/e2e-assets/mac-bundled-feed.mjs @@ -6,11 +6,11 @@ // CI runner with no npm install), pure where possible so tests-js can // assert the generated bytes. // -// Production contract (scripts/darwin-feed.mjs + apps/desktop/update-feed.cjs): +// Production contract (scripts/releases/darwin.py + apps/desktop/update-feed.cjs): // feed key releases/darwin//-mac.yml // merged url /releases/tag//.zip // A release publishes the MERGED feed (both arches merged by -// r2-release.mjs finalize). electron-updater on arm64 requests +// scripts.releases.r2 finalize). electron-updater on arm64 requests // --mac.yml; on x64 it requests -mac.yml. The // channel comes from the release TAG (canary tags serve canary feeds) — // never hard-coded — and must be identical on both manifest sides @@ -37,7 +37,7 @@ export async function sha512Base64(filePath) { /** * The electron-updater yml for one arch leg, in the shape electron-builder - * writes per arch and r2-release.mjs merges: per-arch top-level path/sha512 + * writes per arch and scripts.releases.r2 merges: per-arch top-level path/sha512 * plus files[] (url/sha512/size), releaseDate. Arch metadata names differ * by design; nothing here assumes they are equal across arches. */ diff --git a/tests/scripts/test_release_artifacts.py b/tests/scripts/test_release_artifacts.py new file mode 100644 index 0000000000..bcdef41f00 --- /dev/null +++ b/tests/scripts/test_release_artifacts.py @@ -0,0 +1,77 @@ +"""Native metadata and artifact publication use the same verified bytes.""" +import hashlib +import io +import json +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from scripts.bundles.release_artifacts import materialize, record, stamp_matches + + +def test_windows_metadata_is_read_from_package_and_stale_stamp_is_rejected(tmp_path): + tag, commit = 'v1.2.3', 'a' * 40 + root = tmp_path / 'release' + root.mkdir() + package = root / 'Product-win-x64.msix' + manifest = '' + + def write_package(sha): + with zipfile.ZipFile(package, 'w') as archive: + archive.writestr('AppxManifest.xml', manifest) + archive.writestr('app/resources/install-stamp.json', json.dumps({'tag': tag, 'commit': sha})) + + write_package(commit) + out = tmp_path / 'record.tar' + record('windows', 'x64', root, tag, commit, out) + with tarfile.open(out) as archive: + metadata = json.load(archive.extractfile('metadata-windows-x64.json')) + assert metadata['identity'] == 'Product' + assert metadata['version'] == '1.2.3.0' + assert metadata['publisher'] == 'CN=Test' + assert metadata['applicationId'] == 'App' + assert archive.extractfile(package.name).read() == package.read_bytes() + write_package('b' * 40) + with pytest.raises(ValueError, match='provenance'): + record('windows', 'x64', root, tag, commit, tmp_path / 'bad.tar') + with pytest.raises(ValueError, match='provenance'): + stamp_matches({}, tag, commit) + + +def test_materialize_validates_the_published_file_receipt_before_using_bytes(tmp_path, monkeypatch): + base, tag, commit = 'https://releases.example', 'v1.2.3', 'a' * 40 + data = b'package transport fixture, not native signing proof' + digest = hashlib.sha256(data).hexdigest() + files, packages = [], [] + for platform in ('windows', 'macos'): + for arch in ('x64', 'arm64'): + filename = f'{platform}-{arch}.' + ('msixbundle' if platform == 'windows' else 'zip') + url = f'{base}/releases/tag/{tag}/{filename}' + files.append({'path': filename, 'url': url, 'sha256': digest}) + packages.append({'platform': platform, 'arch': arch, 'identity': 'Product', 'tag': tag, 'commit': commit, + 'version': '1.2.3.0' if platform == 'windows' else '1.2.3', + **({'publisher': 'CN=Test', 'applicationId': 'App'} if platform == 'windows' else {'teamId': 'ABCDEFGHIJ'}), + 'artifact': {'url': url, 'sha256': digest}}) + manifest = {'schema': 1, 'tag': tag, 'commit': commit, 'packages': packages, 'files': files} + class Response(io.BytesIO): + def geturl(self): + return base + "/package" + + monkeypatch.setattr('urllib.request.urlopen', lambda *a, **kw: Response(data)) + materialize(manifest, tmp_path / 'good', public_base=base) + assert all((tmp_path / 'good' / f['path']).read_bytes() == data for f in files) + with pytest.raises(ValueError, match='one Store candidate'): + materialize(manifest, tmp_path / 'store-missing', public_base=base, store_only=True) + store = {'path': 'Store-App.msixbundle', 'url': f'{base}/releases/tag/{tag}/Store-App.msixbundle', 'sha256': digest} + files.append(store) + materialize(manifest, tmp_path / 'store', public_base=base, store_only=True) + assert [p.name for p in (tmp_path / 'store').iterdir()] == [store['path']] + files[0]['sha256'] = 'b' * 64 + with pytest.raises(ValueError, match='receipts differ'): + materialize(manifest, tmp_path / 'bad', public_base=base) + files[0]['sha256'] = digest + monkeypatch.setattr('urllib.request.urlopen', lambda *a, **kw: Response(b'changed bytes')) + with pytest.raises(ValueError, match='digest mismatch'): + materialize(manifest, tmp_path / 'corrupt', public_base=base) diff --git a/tests/scripts/test_release_canary.py b/tests/scripts/test_release_canary.py index 4457b23e73..9c3bd45f19 100644 --- a/tests/scripts/test_release_canary.py +++ b/tests/scripts/test_release_canary.py @@ -293,6 +293,26 @@ class TestCanaryStartsItsOwnBuild: dispatch = next(i for i, c in enumerate(calls) if c[:3] == ["gh", "workflow", "run"]) assert create < dispatch + def test_stable_dispatch_uses_the_tagged_full_release_gate(self, monkeypatch, tmp_path): + import subprocess + + calls = [] + monkeypatch.setattr(release, "REPO_ROOT", tmp_path) + monkeypatch.setattr(release.shutil, "which", lambda _: "gh") + monkeypatch.setattr(release, "_default_branch", lambda _: "main") + monkeypatch.setattr(release.subprocess, "run", lambda cmd, **kw: ( + calls.append(cmd) or subprocess.CompletedProcess(cmd, 0, "", "") + )) + assert release.dispatch_desktop_build("v1.2.3", "owner/repo") + command = calls[0] + assert command[:4] == ["gh", "workflow", "run", "stable-release.yml"] + assert command[command.index("--ref") + 1] == "v1.2.3" + assert "tag=v1.2.3" in command + assert "upload_release=true" not in command + with pytest.raises(ValueError): + release.dispatch_desktop_build("v1.2.3/other", "owner/repo") + assert len(calls) == 1 + def test_a_failed_dispatch_does_not_sink_the_release(self, monkeypatch, tmp_path): """The tag and draft are already pushed by then. Report the manual command and leave them; raising would strand a half-made release.""" diff --git a/tests/scripts/test_release_darwin.py b/tests/scripts/test_release_darwin.py new file mode 100644 index 0000000000..4f818e6394 --- /dev/null +++ b/tests/scripts/test_release_darwin.py @@ -0,0 +1,261 @@ +# tests/scripts/test_release_darwin.py — contract tests for the macOS +# electron-updater feed publication (port of tests-js/darwin-feed.test.mjs). +# Feed merge/validation is pure; publication runs against a REAL loopback +# HTTP server (http.server on 127.0.0.1), not a fabricated backend. + +from __future__ import annotations + +import hashlib +import os +import tempfile + +import pytest +import yaml + +from scripts.releases import darwin, r2 +from tests.scripts.test_release_r2 import r2_server # noqa: F401 — loopback fixture +from scripts.releases.darwin import ( + _darwin_feed, + mac_feed_references, + merge_mac_feeds, + parse_mac_feed, + publish_mac_feed, +) + + +def _inputs(version="0.28.0", light=False): + channel = "canary" if "-canary." in version else "stable" + bytes_by_key = {} + legs = {} + for i, arch in enumerate(("arm64", "x64")): + name = f"{'HermesLight' if light else 'HermesBundled'}-{version}-mac-{arch}.zip" + data = f"test artifact {arch}".encode() + bytes_by_key[f"releases/tag/v{version}/{name}"] = data + file_entry = { + "url": name, + "size": len(data), + "sha512": base64_sha512(data), + } + legs[f"{arch}-{channel}-mac.yml"] = yaml.safe_dump( + { + "version": version, + "files": [file_entry], + "path": name, + "sha512": file_entry["sha512"], + "releaseDate": f"2026-09-0{i + 1}T00:00:00Z", + "releaseNotes": "two lines\nof release notes", + }, + sort_keys=False, + ) + return legs, bytes_by_key + + +def base64_sha512(data: bytes) -> str: + return __import__("base64").b64encode(hashlib.sha512(data).digest()).decode("ascii") + + +def test_never_overwrites_a_published_macos_artifact_on_same_tag_rerun(r2_server): + name = "HermesBundled-0.28.0-mac-arm64.zip" + existing = b"already published artifact" + key = f"releases/tag/v0.28.0/{name}" + r2_server.store[key] = (existing, '"etag-1"') + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(existing) + path = handle.name + try: + r2.put("v0.28.0", name, path, immutable=True) + finally: + os.unlink(path) + # The immutable bytes are untouched and the conflict was verified, not ignored. + assert r2_server.store[key][0] == existing + + +def test_real_prune_protects_live_macos_artifacts_and_fails_closed(r2_server, monkeypatch): + legs, _bytes = _inputs("0.28.0-canary.20200101000000") + plan = merge_mac_feeds(legs, "v0.28.0-canary.20200101000000") + references = mac_feed_references(plan["text"]) + stale = "releases/tag/v0.27.0-canary.20200101000000/unreferenced.zip" + for key in [plan["key"], *references, stale]: + r2_server.store[key] = (b"x", '"e"') + r2_server.store[plan["key"]] = (plan["text"].encode(), '"e"') + monkeypatch.setattr(r2.time, "time", lambda: 1788547200.0) # 2026-09-04 + r2.prune(keep_days=14, dry_run=False) + assert stale not in r2_server.store + for reference in references: + assert reference in r2_server.store, reference + # Fail closed: an unreadable manifest aborts the prune, nothing deleted. + r2_server.store[stale] = (b"still protected until all feeds are readable", '"e"') + before = set(r2_server.store) + r2_server.store["releases/darwin/canary/canary-mac.yml"] = (b"503", '"e"') + with pytest.raises(ValueError): + r2.prune(keep_days=14, dry_run=False) + assert set(r2_server.store) == before + + +def test_merges_native_legs_and_preserves_release_metadata(): + legs, _bytes = _inputs() + plan = merge_mac_feeds(legs, "v0.28.0") + feed = parse_mac_feed(plan["text"]) + assert len(feed["files"]) == 2 + assert feed["releaseNotes"] == "two lines\nof release notes" + assert plan["key"] == "releases/darwin/stable/stable-mac.yml" + assert mac_feed_references(plan["text"]) == [ + ref + for f in feed["files"] + for ref in (f["url"][1:], f"{f['url'][1:]}.blockmap") + ] + assert r2.cache_control_for(plan["key"]) == "no-store" + selection = _darwin_feed("canary", True) + light_legs, _ = _inputs("0.29.0-canary.20260906000000", light=True) + light_plan = merge_mac_feeds(light_legs, "v0.29.0-canary.20260906000000", light=True) + assert light_plan["key"] == f"{selection['directory']}/{selection['fileName']}" + + +@pytest.mark.parametrize( + "kind", ["missing", "version", "variant", "hash", "legacy", "traversal"] +) +def test_rejects_broken_legs_instead_of_publishing(kind): + legs, _bytes = _inputs() + key = "arm64-stable-mac.yml" + feed = yaml.safe_load(legs[key]) + if kind == "missing": + del legs[key] + else: + if kind == "version": + feed["version"] = "0.27.0" + if kind == "variant": + feed["files"][0]["url"] = feed["files"][0]["url"].replace("HermesBundled", "HermesLight") + if kind == "hash": + feed["files"][0]["sha512"] = "invalid" + if kind == "legacy": + feed["sha512"] = "wrong" + if kind == "traversal": + feed["files"][0]["url"] = "../other.zip" + legs[key] = yaml.safe_dump(feed, sort_keys=False) + with pytest.raises(Exception): + merge_mac_feeds(legs, "v0.28.0") + + +def test_semver_grammar_rejects_non_release_versions(): + # The Python port never imports npm semver; it implements exactly the + # release grammar (vMAJOR.MINOR.PATCH[-canary.<14 digits>]) and fails + # loudly on anything else instead of guessing an order. + from scripts.releases.semver import compare, is_valid_version + + assert is_valid_version("0.28.0") and is_valid_version("0.28.0-canary.20260904101010") + assert not is_valid_version("0.28") and is_valid_version("2026.7.20") + assert not is_valid_version("0.28.0-beta.1") + assert compare("0.28.0", "0.27.9") == 1 + assert compare("0.28.0-canary.20260904101010", "0.28.0") == -1 # prerelease < release + assert compare("0.28.0-canary.20260904101010", "0.28.0-canary.20260904101011") == -1 + with pytest.raises(ValueError): + compare("nonsense", "0.28.0") + + +def test_publish_verifies_artifacts_before_conditional_write_and_readback(): + plan = merge_mac_feeds(_inputs()[0], "v0.28.0") + live = {"text": merge_mac_feeds(_inputs("0.27.0")[0], "v0.27.0")["text"], "etag": "old"} + events = [] + + def read(_key): + return live + + def verify(key, _file=None): + events.append(key) + + def write(key, text, etag): + assert etag == "old" + events.append(key) + live.clear() + live.update({"text": text, "etag": "new"}) + + publish_mac_feed(plan, {"read": read, "verify": verify, "write": write}) + assert events[-1] == plan["key"] + assert len(events) == 3 + + write_calls = [] + + def fail_write(key, text, etag): + write_calls.append(key) + + # Downgrade rejection: nothing written. + with pytest.raises(ValueError, match="backward"): + publish_mac_feed( + merge_mac_feeds(_inputs("0.27.0")[0], "v0.27.0"), + {"read": read, "verify": verify, "write": fail_write}, + ) + assert write_calls == [] + # Corrupt bytes abort before the pointer write. + def corrupt_verify(_key, _file=None): + raise ValueError("corrupt bytes") + + with pytest.raises(ValueError, match="corrupt bytes"): + publish_mac_feed(plan, {"read": lambda _k: None, "verify": corrupt_verify, "write": fail_write}) + assert write_calls == [] + + +def test_same_version_identical_feed_is_a_noop(): + plan = merge_mac_feeds(_inputs()[0], "v0.28.0") + live = {"text": plan["text"], "etag": "same"} + calls = [] + publish_mac_feed( + plan, + { + "read": lambda _k: live, + "verify": lambda k: calls.append(k), + "write": lambda k, t, e: calls.append(("write", k)), + }, + ) + assert calls == [] # identical published version → no writes at all + + +def test_finalize_uses_the_real_signed_transport_and_publishes_last(r2_server): + with tempfile.TemporaryDirectory() as dir_path: + legs, bytes_by_key = _inputs() + for name, text in legs.items(): + with open(os.path.join(dir_path, name), "w", encoding="utf-8") as handle: + handle.write(text) + for key, value in bytes_by_key.items(): + r2_server.store[key] = (value, '"e"') + darwin.finalize(tag="v0.28.0", dir=dir_path) + feed_key = "releases/darwin/stable/stable-mac.yml" + assert feed_key in r2_server.store + published = r2_server.store[feed_key][0].decode("utf-8") + assert len(parse_mac_feed(published)["files"]) == 2 + # Order: reads (legs verified) FIRST, then exactly one conditional PUT, + # then the readback — verify bytes before the pointer write. + methods = [m for m, _p, _h in r2_server.requests] + assert methods.count("PUT") == 1 + assert "PUT" not in methods[:3] + put_headers = [h for m, _p, h in r2_server.requests if m == "PUT"][0] + assert put_headers["If-None-Match"] == "*" + assert put_headers["Cache-Control"] == "no-store" + assert put_headers["Content-Type"] == "application/yaml" + # Every request was signed. + for _m, _p, headers in r2_server.requests: + assert headers["authorization"].startswith("AWS4-HMAC-SHA256 ") + + +def test_finalize_rejects_a_downgrade_over_the_live_feed(r2_server): + with tempfile.TemporaryDirectory() as dir_path: + legs, bytes_by_key = _inputs("0.27.0") + for name, text in legs.items(): + with open(os.path.join(dir_path, name), "w", encoding="utf-8") as handle: + handle.write(text) + for key, value in bytes_by_key.items(): + r2_server.store[key] = (value, '"e"') + # The live feed is already at 0.28.0. + newer = merge_mac_feeds(_inputs()[0], "v0.28.0") + r2_server.store["releases/darwin/stable/stable-mac.yml"] = ( + newer["text"].encode(), '"live"') + with pytest.raises(ValueError, match="backward"): + darwin.finalize(tag="v0.27.0", dir=dir_path) + # The live pointer was not replaced. + assert parse_mac_feed( + r2_server.store["releases/darwin/stable/stable-mac.yml"][0].decode() + )["version"] == "0.28.0" + + +def test_finalize_rejects_unknown_variant(): + with pytest.raises(ValueError, match="variant"): + darwin.finalize(tag="v0.28.0", dir=".", variant="dark") diff --git a/tests/scripts/test_release_docker.py b/tests/scripts/test_release_docker.py new file mode 100644 index 0000000000..14ef9a7b29 --- /dev/null +++ b/tests/scripts/test_release_docker.py @@ -0,0 +1,91 @@ +"""Tests for scripts/releases/docker.py — the staged Docker release contract.""" +from __future__ import annotations + +import importlib +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +docker = importlib.import_module("scripts.releases.docker") + +TAG = "v1.2.3" +COMMIT = "a" * 40 +DIGESTS = {"amd64": "b" * 64, "arm64": "c" * 64} + + +@pytest.fixture +def module_root() -> Path: + return Path(docker.__file__).resolve().parent.parent.parent + + +def test_build_manifest_roundtrip_and_identity_rejection() -> None: + manifest = docker.build_manifest(TAG, COMMIT, DIGESTS) + parsed = docker.parse_manifest(json.dumps(manifest).encode()) + docker.verify_manifest(parsed, TAG, COMMIT) + with pytest.raises(docker.DockerReleaseError): + docker.verify_manifest(parsed, "v1.2.4", COMMIT) + with pytest.raises(docker.DockerReleaseError): + docker.verify_manifest(parsed, TAG, "b" * 40) + + +def test_manifest_requires_both_arches() -> None: + with pytest.raises(docker.DockerReleaseError): + docker.build_manifest(TAG, COMMIT, {"amd64": DIGESTS["amd64"]}) + bad = dict(DIGESTS) + bad["arm64"] = "z" * 64 + with pytest.raises(docker.DockerReleaseError): + docker.build_manifest(TAG, COMMIT, bad) + + +def test_manifest_archive_hashes_optional_but_covering() -> None: + manifest = docker.build_manifest(TAG, COMMIT, DIGESTS, {"amd64": "d" * 64, "arm64": "e" * 64}) + assert manifest["archives"] == {"amd64": "d" * 64, "arm64": "e" * 64} + with pytest.raises(docker.DockerReleaseError): + docker.build_manifest(TAG, COMMIT, DIGESTS, {"amd64": "d" * 64}) + with pytest.raises(docker.DockerReleaseError): + docker.build_manifest(TAG, COMMIT, DIGESTS, {"amd64": "d" * 64, "riscv64": "e" * 64}) + + +def test_parse_manifest_rejects_garbage() -> None: + with pytest.raises(docker.DockerReleaseError): + docker.parse_manifest(b"not json") + with pytest.raises(docker.DockerReleaseError): + docker.parse_manifest(json.dumps({"schema": 2}).encode()) + + +def test_sha256_file(tmp_path: Path) -> None: + blob = tmp_path / "image.tar" + blob.write_bytes(b"payload") + assert docker.sha256_file(str(blob)) == __import__("hashlib").sha256(b"payload").hexdigest() + + +def test_cli_manifest_and_verify(tmp_path: Path, module_root: Path) -> None: + out = tmp_path / "manifest.json" + proc = subprocess.run( + [sys.executable, "-m", "scripts.releases.docker", "manifest", + "--tag", TAG, "--commit", COMMIT, + "--digest-amd64", DIGESTS["amd64"], "--digest-arm64", DIGESTS["arm64"]], + cwd=module_root, capture_output=True, text=True, + ) + assert proc.returncode == 0, proc.stderr + out.write_text(proc.stdout) + parsed = json.loads(proc.stdout) + assert parsed["digests"] == DIGESTS + + verify = subprocess.run( + [sys.executable, "-m", "scripts.releases.docker", "verify", + "--tag", TAG, "--commit", COMMIT, str(out)], + cwd=module_root, capture_output=True, text=True, + ) + assert verify.returncode == 0, verify.stderr + + bad = subprocess.run( + [sys.executable, "-m", "scripts.releases.docker", "verify", + "--tag", "v9.9.9", "--commit", COMMIT, str(out)], + cwd=module_root, capture_output=True, text=True, + ) + assert bad.returncode == 1 + assert "::error::" in bad.stderr diff --git a/tests/scripts/test_release_promotion.py b/tests/scripts/test_release_promotion.py new file mode 100644 index 0000000000..8d51688c05 --- /dev/null +++ b/tests/scripts/test_release_promotion.py @@ -0,0 +1,66 @@ +"""Promotion writes pointers only after verified candidate content exists.""" +import hashlib +import io +import json +from pathlib import Path + +import pytest + +from scripts.bundles import release_artifacts as artifacts +from scripts.releases import r2 +from tests.scripts.test_release_r2 import r2_server # noqa: F401 + + +def test_publish_and_promote_use_the_same_content_before_any_channel_write(tmp_path, monkeypatch, r2_server): + tag, commit = 'v1.2.3', 'a' * 40 + base = 'https://releases.example' + content = { + 'app.msixbundle': b'windows transport bytes', + 'arm64.zip': b'mac arm transport bytes', 'x64.zip': b'mac x64 transport bytes', + 'apt/pool/package.deb': b'deb transport bytes', + 'apt/dists/hermes-stable/InRelease': b'signed-index transport bytes', + } + files = [{'path': name, 'url': f'{base}/releases/tag/{tag}/{name}', 'sha256': hashlib.sha256(data).hexdigest()} for name, data in content.items()] + by_name = {item['path']: item for item in files} + packages = [] + for platform in ('windows', 'macos'): + for arch in ('x64', 'arm64'): + file = by_name['app.msixbundle' if platform == 'windows' else arch + '.zip'] + packages.append({'platform': platform, 'arch': arch, 'tag': tag, 'commit': commit, 'identity': 'App', + 'version': '1.2.3.0' if platform == 'windows' else '1.2.3', + **({'publisher': 'CN=Test', 'applicationId': 'App'} if platform == 'windows' else {'teamId': 'ABCDEFGHIJ'}), + 'artifact': {'url': file['url'], 'sha256': file['sha256']}}) + manifest = {'schema': 1, 'tag': tag, 'commit': commit, 'files': files, 'packages': packages} + + class Response(io.BytesIO): + def geturl(self): + return base + + def fetch(url, **kwargs): + if '/releases/tag/' in url: + return Response(content[url.split(f'/releases/tag/{tag}/')[1]]) + return Response(r2_server.store[url.split(base + '/')[1]][0]) + + monkeypatch.setattr('urllib.request.urlopen', fetch) + artifacts.publish(manifest, tmp_path / 'publish', base) + assert 'releases/termux/stable/pool/package.deb' in r2_server.store + assert not any(key.endswith(('.appinstaller', 'InRelease')) for key in r2_server.store) + + events = [] + real_put = r2.put + def put(**kwargs): + events.append(kwargs['key']) + real_put(**kwargs) + monkeypatch.setattr(r2, 'put', put) + monkeypatch.setattr(r2, 'finalize', lambda **kwargs: events.append('mac-feed')) + artifacts.promote(manifest, tmp_path / 'promote', base) + assert events[0] == 'mac-feed' + assert events[-1] == 'releases/termux/stable/dists/hermes-stable/InRelease' + assert 'releases/win32/stable/stable.appinstaller' in r2_server.store + assert b'/releases/tag/v1.2.3/app.msixbundle' in r2_server.store['releases/win32/stable/stable.appinstaller'][0] + + events.clear() + content['app.msixbundle'] = b'changed artifact' + with pytest.raises(ValueError, match='digest mismatch'): + artifacts.promote(manifest, tmp_path / 'broken', base) + assert events == [] diff --git a/tests/scripts/test_release_r2.py b/tests/scripts/test_release_r2.py new file mode 100644 index 0000000000..cdda83bbe8 --- /dev/null +++ b/tests/scripts/test_release_r2.py @@ -0,0 +1,712 @@ +# tests/scripts/test_release_r2.py — contract tests for the Python R2 +# release transport (port of tests-js/r2-release.test.mjs). The SigV4 +# vectors pin the signer against botocore (the reference implementation, +# 1.43.81) at a FIXED timestamp/creds, so the expected values are +# reproducible fixtures rather than self-consistency: +# - get-vanilla generic signer (no x-amz-content-sha256), example.com +# - r2-put-payload S3 signer, payload hash signed, region auto +# - r2-list S3 signer, ListObjectsV2 with query params +# - r2-delete S3 signer, region auto +# The get-vanilla case also reproduces the public aws-sig-v4-test-suite +# request shape (verified by independent spec computation). +# +# Protocol behavior (put/finalize/prune) is tested against a REAL loopback +# HTTP server (http.server on 127.0.0.1) — no fabricated backend output. + +from __future__ import annotations + +import base64 +import io +import json +import os +import re +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from scripts.releases import r2 +from scripts.releases.r2 import ( + auth_header, + cache_control_for, + canary_doomed_keys, + canonical_query, + canonical_request, + channel_for_tag, + content_type_for, + encode_key_path, + feed_dir_for, + feed_referenced_keys, + parse_list_xml, + publish_feed_uploads, + referenced_feed_bundle_filenames, + rfc3986_encode, + staging_key_for, + stale_feed_bundle_keys, +) + +AKID = "AKIDEXAMPLE" +SECRET = "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY" +NOW = "20150830T123600Z" +EMPTY_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + +def _auth(**kwargs): + kwargs.setdefault("access_key_id", AKID) + kwargs.setdefault("secret_key", SECRET) + kwargs.setdefault("now", NOW) + return auth_header(**kwargs) + + +# ── SigV4 vectors ─────────────────────────────────────────────────────────── + +def test_get_vanilla_matches_the_aws_test_suite_vector(): + authz = _auth( + method="GET", + host="example.com", + path="/", + query="", + headers={"host": "example.com", "x-amz-date": NOW}, + payload_hash=EMPTY_SHA, + region="us-east-1", + service="service", + ) + assert authz == ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, " + "SignedHeaders=host;x-amz-date, " + "Signature=33399fd3d4a9d6104710c7c04005f7c959f8b1f8bf41b823587ed36b079e453f" + ) + + +def test_r2_put_payload_matches_botocore(): + body_hash = "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" # sha256("Welcome to Amazon S3.") + host = "abc123.r2.cloudflarestorage.com" + authz = _auth( + method="PUT", + host=host, + path="/hermes-releases/HermesBundled-0.28.0-win-x64.msix", + query="", + headers={"host": host, "x-amz-date": NOW, "x-amz-content-sha256": body_hash}, + payload_hash=body_hash, + ) + assert authz == ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, " + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " + "Signature=05ba50acfb54042fac330848af50877e5fb477c4f2063c2f77f9cc80855eb1e9" + ) + + +def test_r2_list_matches_botocore(): + query = canonical_query( + {"list-type": "2", "prefix": "HermesBundled-0.28.0-", "max-keys": "1000"} + ) + assert query == "list-type=2&max-keys=1000&prefix=HermesBundled-0.28.0-" + host = "abc123.r2.cloudflarestorage.com" + authz = _auth( + method="GET", + host=host, + path="/hermes-releases", + query=query, + headers={"host": host, "x-amz-date": NOW, "x-amz-content-sha256": EMPTY_SHA}, + payload_hash=EMPTY_SHA, + ) + assert authz == ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, " + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " + "Signature=3ec423c452a318664c85fbcc25667ad07201aedce688e3bb6b345b4baaa39d90" + ) + + +def test_r2_delete_matches_botocore(): + host = "abc123.r2.cloudflarestorage.com" + authz = _auth( + method="DELETE", + host=host, + path="/hermes-releases/HermesBundled-0.28.0-canary.20260818-win-arm64.msix", + query="", + headers={"host": host, "x-amz-date": NOW, "x-amz-content-sha256": EMPTY_SHA}, + payload_hash=EMPTY_SHA, + ) + assert authz == ( + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/auto/s3/aws4_request, " + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " + "Signature=ec5ccb76f701193b28aaca052cabbf2f084e9c71ffc64b872fa51d4e70dc6e55" + ) + + +# ── Encoding / layout helpers ─────────────────────────────────────────────── + +def test_rfc3986_encode_escapes_the_aws_reserved_set_keeps_unreserved(): + assert rfc3986_encode("HermesBundled-0.28.0-win-x64.msix") == "HermesBundled-0.28.0-win-x64.msix" + assert rfc3986_encode("a b!'()*c") == "a%20b%21%27%28%29%2Ac" + + +def test_encode_key_path_encodes_segment_wise_preserves_separators(): + assert encode_key_path("HermesBundled-0.28.0-win-x64.msix") == "HermesBundled-0.28.0-win-x64.msix" + assert encode_key_path("a b/c d") == "a%20b/c%20d" + + +def test_parse_list_xml_extracts_keys_truncation_token_entities(): + xml = ( + '\n' + '\n' + " hermes-releases\n \n" + " 3\n 1000\n" + " true\n" + " HermesBundled-0.28.0-win-x64.msix" + "2026-08-18T00:00:00Z123\n" + " a&b.msix" + "2026-08-18T00:00:00Z1\n" + " latest.yml" + "2026-08-18T00:00:00Z2\n" + " abc+def/=\n" + "" + ) + parsed = parse_list_xml(xml) + assert parsed["keys"] == ["HermesBundled-0.28.0-win-x64.msix", "a&b.msix", "latest.yml"] + assert parsed["truncated"] is True + assert parsed["nextToken"] == "abc+def/=" + + +def test_canary_doomed_keys_dates_by_the_key_suffix(): + keys = [ + "releases/tag/v0.28.0/HermesBundled-0.28.0-win-x64.msix", # stable — never doomed + "releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix", + "releases/tag/v0.28.0-canary.20260818/HermesBundled-0.28.0-canary.20260818-win-x64.msix", # today — kept + "releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix.blockmap", + "latest.yml", + "canary.yml", + ] + assert canary_doomed_keys(keys, "20260814") == [ + "releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix", + "releases/tag/v0.28.0-canary.20260801/HermesBundled-0.28.0-canary.20260801-win-x64.msix.blockmap", + ] + + +def test_channel_for_tag_maps_stable_vs_canary(): + assert channel_for_tag("v0.28.0") == "stable" + assert channel_for_tag("v0.28.0-canary.20260818101010") == "canary" + assert channel_for_tag("v0.28.0-canary.20260818") == "canary" + + +def test_staging_key_and_feed_dir_layout_keys(): + assert staging_key_for("v0.28.0", "HermesBundled-0.28.0-win-x64.msix") == ( + "releases/tag/v0.28.0/HermesBundled-0.28.0-win-x64.msix" + ) + assert feed_dir_for("win32", "stable") == "releases/win32/stable" + assert feed_dir_for("darwin", "canary") == "releases/darwin/canary" + + +def test_content_type_for_maps_msix_and_appinstaller(): + assert content_type_for("HermesBundled-0.28.0-win-x64.msix") == "application/msix" + assert content_type_for("HermesBundled-0.28.0-win.msixbundle") == "application/msixbundle" + assert content_type_for("stable.appinstaller") == "application/appinstaller" + assert content_type_for("HermesBundled-0.28.0-mac-x64.dmg") is None + assert content_type_for("latest-mac.yml") is None + # Case-insensitive on the suffix. + assert content_type_for("X.APPINSTALLER") == "application/appinstaller" + + +def test_apt_mutable_metadata_revalidates_immutable_bytes_cache(): + feed = "releases/termux/canary" + for name in [ + "key.asc", + "dists/hermes-canary/InRelease", + "dists/hermes-canary/Release", + "dists/hermes-canary/main/binary-aarch64/Packages.gz", + ]: + assert cache_control_for(f"{feed}/{name}") == "no-store" + assert cache_control_for(f"{feed}/dists/hermes-canary/main/binary-aarch64/by-hash/SHA256/abcd") == ( + "public, max-age=31536000, immutable" + ) + assert cache_control_for(f"{feed}/pool/h/hermes-agent_1.2.3_aarch64.deb") == ( + "public, max-age=31536000, immutable" + ) + assert cache_control_for("releases/win32/stable/stable.appinstaller") == "no-store" + + +def test_canonical_request_reads_mixed_case_header_values(): + # Regression: the canonical line must carry the VALUE of a mixed-case + # header ('Content-Type'), never a placeholder. + host = "abc123.r2.cloudflarestorage.com" + body_hash = "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" + headers = { + "host": host, + "x-amz-date": NOW, + "x-amz-content-sha256": body_hash, + "Content-Type": "application/msix", + } + canon = canonical_request( + "PUT", "/hermes-releases/HermesBundled-0.28.0-win-x64.msix", "", headers, body_hash + ) + assert "content-type:application/msix" in canon + assert "undefined" not in canon + assert "content-type;host;x-amz-content-sha256;x-amz-date" in canon + + +# ── C22: artifact first, feed pointer last ────────────────────────────────── + +def test_publish_feed_uploads_bundle_before_pointer(): + calls = [] + publish_feed_uploads( + { + "channelDir": "releases/win32/canary", + "appinstallerName": "canary.appinstaller", + "bundleFilename": "HermesBundled-0.27.2.9-win.msixbundle", + "bundleFile": "C:/rel/HermesBundled-0.27.2.9-win.msixbundle", + "appinstallerFile": "C:/rel/canary.appinstaller", + }, + lambda key, file: calls.append([key, file]), + ) + assert calls == [ + ["releases/win32/canary/HermesBundled-0.27.2.9-win.msixbundle", "C:/rel/HermesBundled-0.27.2.9-win.msixbundle"], + ["releases/win32/canary/canary.appinstaller", "C:/rel/canary.appinstaller"], + ] + + +def test_publish_feed_uploads_never_writes_pointer_when_bundle_fails(): + calls = [] + + def upload(key, _file): + calls.append(key) + raise RuntimeError("R2 PUT -> 503") + + with pytest.raises(RuntimeError): + publish_feed_uploads( + { + "channelDir": "releases/win32/stable", + "appinstallerName": "stable.appinstaller", + "bundleFilename": "HermesBundled-0.28.0.0-win.msixbundle", + "bundleFile": "bundle", + "appinstallerFile": "feed", + }, + upload, + ) + assert calls == ["releases/win32/stable/HermesBundled-0.28.0.0-win.msixbundle"] + + +# ── C22: canary feed-dir retention (fail-closed, keep-days grace) ─────────── + +CANARY_FEED_XML = ( + '\n' + '\n' + ' \n' + "\n" +) + + +def test_referenced_feed_bundle_filenames_reads_main_package_only(): + names = referenced_feed_bundle_filenames(CANARY_FEED_XML) + assert names == ["HermesBundled-0.27.2.9-win.msixbundle"] + # The AppInstaller ROOT Uri (the feed pointer itself) must NOT count. + assert "canary.appinstaller" not in names + assert referenced_feed_bundle_filenames("") == [] + assert referenced_feed_bundle_filenames("ServiceUnavailable") == [] + # A bundle Uri OUTSIDE MainPackage/MainBundle is not a reference. + assert referenced_feed_bundle_filenames('') == [] + + +def test_feed_referenced_keys_protects_bundle_and_absolute_tag_uris(): + tag_feed = CANARY_FEED_XML.replace( + 'Uri="https://r2.example/releases/win32/canary/HermesBundled-0.27.2.9-win.msixbundle"', + 'Uri="https://r2.example/releases/tag/v0.27.2-canary.20260829/HermesBundled-0.27.2-win-x64.msix"', + ) + keys = feed_referenced_keys("releases/win32/canary", tag_feed) + assert "releases/win32/canary/HermesBundled-0.27.2-win-x64.msix" in keys + assert "releases/tag/v0.27.2-canary.20260829/HermesBundled-0.27.2-win-x64.msix" in keys + + +CANARY_DIR = "releases/win32/canary" +OLD_MS = 1785542400 # 2026-08-01T00:00:00Z +FRESH_MS = 1788480000 # 2026-09-03T00:00:00Z +CUTOFF_MS = 1787356800 # 2026-08-21T00:00:00Z + + +def _canary_keys(extra=()): + return [ + f"{CANARY_DIR}/canary.appinstaller", + f"{CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle", # referenced + f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle", # stale + f"{CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle", # uploaded, not yet pointed + "releases/win32/stable/stable.appinstaller", + "releases/win32/stable/HermesBundled-0.28.0.0-win.msixbundle", # referenced + "releases/win32/stable/HermesBundled-0.27.0.0-win.msixbundle", # stale stable + *extra, + ] + + +def _last_modified_for(keys, overrides=None): + lm = {k: OLD_MS for k in keys} + lm.update(overrides or {}) + return lm + + +def test_stale_feed_bundle_keys_old_unreferenced_doomed_referenced_and_fresh_kept(): + keys = _canary_keys() + lm = _last_modified_for(keys, {f"{CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle": FRESH_MS}) + doomed = stale_feed_bundle_keys(keys, {CANARY_DIR: [CANARY_FEED_XML]}, lm, CUTOFF_MS) + assert doomed == [f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle"] + + +def test_stale_feed_bundle_keys_stable_dirs_never_pruned(): + keys = _canary_keys() + lm = _last_modified_for(keys, {f"{CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle": FRESH_MS}) + stable_feed = CANARY_FEED_XML.replace("0.27.2.9", "0.28.0.0").replace( + "HermesBundled-0.27.2.9-win", "HermesBundled-0.28.0.0-win" + ) + feeds = {CANARY_DIR: [CANARY_FEED_XML], "releases/win32/stable": [stable_feed]} + doomed = stale_feed_bundle_keys(keys, feeds, lm, CUTOFF_MS) + assert doomed == [f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle"] + + +@pytest.mark.parametrize( + "bad", + [ + "", + "boom", + None, + '', + '', + "", + ], +) +def test_stale_feed_bundle_keys_malformed_manifest_blocks_dir(bad): + keys = _canary_keys() + assert stale_feed_bundle_keys(keys, {CANARY_DIR: [bad]}, _last_modified_for(keys), CUTOFF_MS) == [] + + +def test_stale_feed_bundle_keys_union_protected_one_bad_blocks_all(): + second = CANARY_FEED_XML.replace("0.27.2.9", "0.27.3.0").replace( + "HermesBundled-0.27.2.9-win", "HermesBundled-0.27.3.0-win" + ) + extra = [f"{CANARY_DIR}/second.appinstaller", f"{CANARY_DIR}/HermesBundled-0.27.3.0-win.msixbundle"] + keys = _canary_keys(extra) + lm = _last_modified_for(keys, {f"{CANARY_DIR}/HermesBundled-0.27.2.99-win.msixbundle": FRESH_MS}) + # Union of both feeds: both referenced bundles kept, the rest pruned. + assert stale_feed_bundle_keys(keys, {CANARY_DIR: [CANARY_FEED_XML, second]}, lm, CUTOFF_MS) == [ + f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle" + ] + # ONE unreadable/unrecognized manifest in the dir blocks feed retention. + assert stale_feed_bundle_keys(keys, {CANARY_DIR: [CANARY_FEED_XML, None]}, lm, CUTOFF_MS) == [] + assert stale_feed_bundle_keys(keys, {CANARY_DIR: [CANARY_FEED_XML, ""]}, lm, CUTOFF_MS) == [] + + +@pytest.mark.parametrize("unknown", [None, float("nan"), float("inf")]) +def test_stale_feed_bundle_keys_missing_lastmodified_keeps_object(unknown): + keys = _canary_keys() + metadata = {key: unknown for key in keys} + doomed = stale_feed_bundle_keys(keys, {CANARY_DIR: [CANARY_FEED_XML]}, metadata, CUTOFF_MS) + assert doomed == [] + + +# ── Real loopback HTTP protocol tests ─────────────────────────────────────── + +class _R2StubHandler(BaseHTTPRequestHandler): + """Minimal R2-shaped backend: in-memory objects, records requests.""" + + server_version = "r2-stub/1" + + def log_message(self, *args): # keep test output clean + pass + + def _key(self): + from urllib.parse import unquote, urlsplit + + path = unquote(urlsplit(self.path).path) + return path.split("/", 2)[2] if path.count("/") >= 2 else path.lstrip("/") + + def do_GET(self): + store = self.server.store # type: ignore[attr-defined] + self.server.requests.append(("GET", self.path, dict(self.headers))) # type: ignore[attr-defined] + if "list-type=2" in self.path: + body = self.server.listing_xml() # type: ignore[attr-defined] + self._send(200, body, {"content-type": "application/xml"}) + return + key = self._key() + if key in store: + body, ctype = store[key] + headers = {"etag": '"abc"'} + if self.headers.get("Range"): + headers["content-range"] = f"bytes 0-0/{len(body)}" + self._send(206, body[:1], headers) + return + self._send(200, body, headers) + else: + self._send(404, b"NoSuchKey") + + def do_HEAD(self): + self.server.requests.append(("HEAD", self.path, dict(self.headers))) # type: ignore[attr-defined] + key = self._key() + if key in self.server.store: # type: ignore[attr-defined] + body, _ = self.server.store[key] # type: ignore[attr-defined] + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.send_header("ETag", '"abc"') + self.end_headers() + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_PUT(self): + server = self.server # type: ignore[attr-defined] + server.requests.append(("PUT", self.path, dict(self.headers))) + length = int(self.headers.get("Content-Length", "0")) + data = self.rfile.read(length) if length else b"" + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + data = self._read_chunked() + key = self._key() + if self.headers.get("If-None-Match") == "*" and key in server.store: + self._send(412, b"Precondition Failed") + return + if self.headers.get("If-Match") and self.headers["If-Match"] != server.store.get(key, (b"",))[1]: + self._send(412, b"Precondition Failed") + return + server.store[key] = (data, self.headers.get("If-Match", '"new"')) + self._send(200, b"") + + def do_DELETE(self): + server = self.server # type: ignore[attr-defined] + server.requests.append(("DELETE", self.path, dict(self.headers))) + key = self._key() + server.store.pop(key, None) + self._send(204, b"") + + def _read_chunked(self): + data = b"" + while True: + size_line = self.rfile.readline().strip() + size = int(size_line.split(b";")[0], 16) + if size == 0: + self.rfile.readline() + return data + data += self.rfile.read(size) + self.rfile.readline() + + def _send(self, status, body, headers=None): + if isinstance(body, str): + body = body.encode("utf-8") + self.send_response(status) + self.send_header("Content-Length", str(len(body))) + for k, v in (headers or {}).items(): + self.send_header(k.title(), v) + self.end_headers() + self.wfile.write(body) + + +@pytest.fixture +def r2_server(monkeypatch): + server = ThreadingHTTPServer(("127.0.0.1", 0), _R2StubHandler) + server.store = {} + server.requests = [] + + def listing_xml(): + parts = ["false"] + for key, (body, _etag) in server.store.items(): + lm = "2026-08-01T00:00:00Z" + parts.append(f"{key}{lm}") + parts.append("") + return "".join(parts) + + server.listing_xml = listing_xml # type: ignore[attr-defined] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + # Point the transport at the loopback endpoint. + monkeypatch.setenv("CLOUDFLARE_R2_ACCOUNT_ID", "loopback") + monkeypatch.setattr(r2, "s3_endpoint", lambda _account: f"http://127.0.0.1:{server.server_port}") + monkeypatch.setenv("CLOUDFLARE_R2_ACCESS_KEY_ID", AKID) + monkeypatch.setenv("CLOUDFLARE_R2_SECRET_ACCESS_KEY", SECRET) + monkeypatch.setenv("CLOUDFLARE_R2_BUCKET", "hermes-releases") + yield server + server.shutdown() + server.server_close() + + +def test_put_streams_a_file_and_verifies_size(r2_server): + import tempfile + + payload = os.urandom(5 * 1024 * 1024 + 123) # multi-chunk stream + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(payload) + path = handle.name + try: + r2.put("v0.28.0", "artifact.bin", path) + finally: + os.unlink(path) + key = "releases/tag/v0.28.0/artifact.bin" + stored, _etag = r2_server.store[key] + assert stored == payload + puts = [r for r in r2_server.requests if r[0] == "PUT"] + assert len(puts) == 1 + # Every request carries a signed Authorization header. + for _method, _path, headers in r2_server.requests: + assert headers["authorization"].startswith("AWS4-HMAC-SHA256 ") + # The streamed body's hash was signed as x-amz-content-sha256. + import hashlib + + expected_hash = hashlib.sha256(payload).hexdigest() + assert puts[0][2]["x-amz-content-sha256"] == expected_hash + + +def test_put_immutable_conflict_verifies_remote_bytes(r2_server): + import tempfile + + existing = b"already published artifact" + key = "releases/tag/v0.28.0/HermesBundled-0.28.0-mac-arm64.zip" + r2_server.store[key] = (existing, '"etag-1"') + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(existing) + path = handle.name + try: + r2.put("v0.28.0", "HermesBundled-0.28.0-mac-arm64.zip", path, immutable=True) + finally: + os.unlink(path) + # Nothing was overwritten: the remote bytes are unchanged. + assert r2_server.store[key][0] == existing + + +def test_put_immutable_conflict_with_corrupt_remote_fails(r2_server): + import tempfile + + key = "releases/tag/v0.28.0/HermesBundled-0.28.0-mac-x64.zip" + r2_server.store[key] = (b"corrupt different bytes", '"etag-1"') + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(b"local truth") + path = handle.name + try: + with pytest.raises(ValueError, match="checksum mismatch"): + r2.put("v0.28.0", "HermesBundled-0.28.0-mac-x64.zip", path, immutable=True) + finally: + os.unlink(path) + assert r2_server.store[key][0] == b"corrupt different bytes" + + +def test_list_prints_keys_and_paginates(r2_server, capsys): + r2_server.store["releases/tag/v0.28.0/a.msix"] = (b"x", '"e"') + r2_server.store["releases/tag/v0.28.0/b.yml"] = (b"y", '"e"') + r2.list_objects(prefix="releases/tag/v0.28.0/") + # (list_objects returns; the CLI prints) + from scripts.releases.r2 import list_objects as _lo + + result = _lo(prefix="releases/tag/v0.28.0/") + assert sorted(result["keys"]) == ["releases/tag/v0.28.0/a.msix", "releases/tag/v0.28.0/b.yml"] + assert "list-type=2" in [r for r in r2_server.requests if r[0] == "GET"][0][1] + + +def test_prune_canaries_dry_run_issues_no_delete(r2_server, capsys, monkeypatch): + old_bundle = f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle" + referenced = f"{CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle" + r2_server.store[f"{CANARY_DIR}/canary.appinstaller"] = (CANARY_FEED_XML.encode(), '"e"') + r2_server.store[referenced] = (b"bundle", '"e"') + r2_server.store[old_bundle] = (b"stale", '"e"') + r2_server.store["releases/tag/v0.27.2-canary.20260801000000/old.zip"] = (b"z", '"e"') + + monkeypatch.setattr(r2.time, "time", lambda: 1788547200.0) # 2026-09-04 + r2.prune(keep_days=14, dry_run=True) + out = capsys.readouterr().out + assert f"would delete r2:{old_bundle}" in out + assert "would delete r2:releases/tag/v0.27.2-canary.20260801000000/old.zip" in out + assert f"would delete r2:{referenced}" not in out + assert "would delete" not in out.split("appinstaller")[0] if ".appinstaller" in out else True + assert not any(r[0] == "DELETE" for r in r2_server.requests) + + +def test_prune_canaries_real_delete_only_the_doomed(r2_server, capsys, monkeypatch): + old_bundle = f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle" + referenced = f"{CANARY_DIR}/HermesBundled-0.27.2.9-win.msixbundle" + r2_server.store[f"{CANARY_DIR}/canary.appinstaller"] = (CANARY_FEED_XML.encode(), '"e"') + r2_server.store[referenced] = (b"bundle", '"e"') + r2_server.store[old_bundle] = (b"stale", '"e"') + monkeypatch.setattr(r2.time, "time", lambda: 1788547200.0) # 2026-09-04 + r2.prune(keep_days=14, dry_run=False) + assert old_bundle not in r2_server.store + assert referenced in r2_server.store + assert f"{CANARY_DIR}/canary.appinstaller" in r2_server.store + + +def test_prune_fails_closed_when_manifest_unreadable(r2_server, monkeypatch): + r2_server.store[f"{CANARY_DIR}/canary.appinstaller"] = (b"ServiceUnavailable", '"e"') + r2_server.store[f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle"] = (b"stale", '"e"') + monkeypatch.setattr(r2.time, "time", lambda: 1788547200.0) # 2026-09-04 + with pytest.raises(RuntimeError, match="refusing to prune"): + r2.prune(keep_days=14, dry_run=False) + # Nothing was deleted. + assert f"{CANARY_DIR}/HermesBundled-0.27.1.12000-win.msixbundle" in r2_server.store + + +def test_cli_usage_rejects_unknown_flags(capsys): + with pytest.raises(SystemExit): + r2.main(["bogus-command"]) + with pytest.raises(SystemExit): + r2.main(["put", "--wat", "x"]) + + +def test_verify_remote_artifact_streams_without_buffering(r2_server): + """REAL streaming proof: the hash is computed over socket-sized chunks + against the live loopback server — the artifact is never materialized + whole (a 2GiB artifact would OOM the buffered path).""" + import base64 + import hashlib + + payload = os.urandom(3 * 1024 * 1024 + 7) + key = "releases/tag/v0.28.0/stream-check.zip" + r2_server.store[key] = (payload, '"e"') + url = f"http://127.0.0.1:{r2_server.server_port}/hermes-releases/{key}" + r2.verify_remote_artifact( + url, + {"access_key_id": AKID, "secret_key": SECRET}, + NOW, + expected_size=len(payload), + digest=base64.b64encode(hashlib.sha512(payload).digest()).decode("ascii"), + ) + # Mismatched digest is rejected. + with pytest.raises(ValueError, match="checksum mismatch"): + r2.verify_remote_artifact( + url, + {"access_key_id": AKID, "secret_key": SECRET}, + NOW, + expected_size=len(payload), + digest=base64.b64encode(hashlib.sha512(b"other").digest()).decode("ascii"), + ) + + +def test_put_accepts_pathlib_paths(r2_server): + import pathlib + import tempfile + + payload = b"pathlib input" + with tempfile.NamedTemporaryFile(delete=False) as handle: + handle.write(payload) + path = pathlib.Path(handle.name) + try: + r2.put("v0.28.0", "pathlib.bin", path) + finally: + os.unlink(path) + assert r2_server.store["releases/tag/v0.28.0/pathlib.bin"][0] == payload + + +def test_parse_list_xml_handles_fractional_and_plain_timestamps(): + parsed = parse_list_xml( + "" + "a2026-08-18T00:00:00.123Z" + "b2026-08-18T00:00:00Z" + "" + ) + from datetime import datetime, timezone + + assert parsed["lastModified"]["a"] == int( + datetime(2026, 8, 18, 0, 0, 0, 123000, tzinfo=timezone.utc).timestamp() + ) + assert parsed["lastModified"]["b"] == int( + datetime(2026, 8, 18, tzinfo=timezone.utc).timestamp() + ) + + +def test_canonical_header_whitespace_is_collapsed(): + canon = canonical_request( + "PUT", "/p", "", {"host": "h", "Content-Type": "application/msix extra\tvalue"}, "x" + ) + assert "content-type:application/msix extra value" in canon diff --git a/tests/scripts/test_release_tag_sampling.py b/tests/scripts/test_release_tag_sampling.py new file mode 100644 index 0000000000..4f6e34651c --- /dev/null +++ b/tests/scripts/test_release_tag_sampling.py @@ -0,0 +1,20 @@ +"""Release sampling follows tag chronology across CalVer and SemVer.""" +import os +import subprocess + +from scripts.releases.pick_tags import pick_tags + + +def test_sampling_includes_newest_stable_and_excludes_candidate(tmp_path): + def git(*args, env=None): + return subprocess.run(["git", *args], cwd=tmp_path, env=env, check=True, capture_output=True, text=True) + + git("init", "-b", "main") + git("config", "user.name", "fixture") + git("config", "user.email", "fixture@example.invalid") + for index, tag in enumerate(["v2026.7.20", "v0.27.0", "v0.28.0"]): + env = {**os.environ, "GIT_AUTHOR_DATE": f"2026-08-{index + 1:02d}T00:00:00Z", "GIT_COMMITTER_DATE": f"2026-08-{index + 1:02d}T00:00:00Z"} + git("commit", "--allow-empty", "-m", tag, env=env) + git("tag", tag) + assert pick_tags(tmp_path, 1, "v0.28.0") == ["v0.27.0"] + assert pick_tags(tmp_path, 3, "v0.28.0") == ["v2026.7.20", "v0.27.0"] diff --git a/tests/scripts/test_stable_release.py b/tests/scripts/test_stable_release.py new file mode 100644 index 0000000000..88fe752db4 --- /dev/null +++ b/tests/scripts/test_stable_release.py @@ -0,0 +1,138 @@ +"""Release gates and package transitions bind the intended immutable artifacts.""" +import copy +import hashlib +import io +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.releases.stable import ( + check_tag, plan_transitions, read_manifest, require_stable_identity, + require_success, validate_candidates, +) + +BASE = "https://releases.example" +ROOT = Path(__file__).resolve().parents[2] + + +def candidates(tag, commit, digest): + packages = [] + for platform in ("windows", "macos"): + for arch in ("x64", "arm64"): + packages.append({ + "platform": platform, "arch": arch, "tag": tag, "commit": commit, + "identity": "test.application", + "version": f"{tag[1:]}.0" if platform == "windows" else tag[1:], + **({"publisher": "CN=Test", "applicationId": "App"} if platform == "windows" else {"teamId": "ABCDEFGHIJ"}), + "artifact": {"sha256": digest, + "url": f"{BASE}/releases/tag/{tag}/{arch}" + (".msixbundle" if platform == "windows" else ".zip")}, + }) + return {"schema": 1, "tag": tag, "commit": commit, "packages": packages} + + +def test_gate_requires_every_success_including_real_cli(tmp_path): + required = ["ci", "docker", "acceptance", "publication"] + success = {name: {"result": "success"} for name in required} + require_success(success, required) + for name in required: + for result in ("failure", "cancelled", "skipped", None): + needs = copy.deepcopy(success) + if result: + needs[name]["result"] = result + else: + del needs[name] + with pytest.raises(ValueError, match=name): + require_success(needs, required) + summary = tmp_path / "summary.md" + env = {**os.environ, "RELEASE_NEEDS": json.dumps(success), "GITHUB_STEP_SUMMARY": str(summary), "PYTHONPATH": str(ROOT)} + argv = [sys.executable, "-m", "scripts.releases.stable", "gate", *required] + assert subprocess.run(argv, cwd=tmp_path, env=env, capture_output=True).returncode == 0 + env["RELEASE_NEEDS"] = json.dumps({**success, "publication": {"result": "cancelled"}}) + result = subprocess.run(argv, cwd=tmp_path, env=env, capture_output=True, text=True, encoding="utf-8") + assert result.returncode != 0 + assert "publication=cancelled" in result.stderr + + +def test_transitions_bind_all_arches_identity_version_and_archive(): + old = candidates("v1.2.3", "a" * 40, "1" * 64) + new = candidates("v1.2.4", "b" * 40, "2" * 64) + require_stable_identity(new["tag"], new["commit"], "refs/tags/v1.2.4") + for tag, ref in [("v1.2.4", "refs/heads/main"), ("v1.2.4-canary.20260907143420", "refs/tags/v1.2.4-canary.20260907143420")]: + with pytest.raises(ValueError): + require_stable_identity(tag, new["commit"], ref) + transitions = plan_transitions(old, new, BASE) + assert {row["target"] for row in transitions} == {"windows-x64", "windows-arm64", "macos-x64", "macos-arm64"} + assert all(row["transition"]["new"]["commit"] == new["commit"] for row in transitions) + missing = copy.deepcopy(new) + missing["packages"].pop() + with pytest.raises(ValueError, match="both architectures"): + plan_transitions(old, missing, BASE) + with pytest.raises(ValueError, match="identity"): + validate_candidates(new, new["tag"], old["commit"], BASE) + for key, value in [("commit", old["commit"]), ("identity", "different"), ("publisher", "CN=Other")]: + changed = copy.deepcopy(new) + changed["packages"][0][key] = value + with pytest.raises(ValueError): + plan_transitions(old, changed, BASE) + mutable = copy.deepcopy(new) + mutable["packages"][0]["artifact"]["url"] = f"{BASE}/releases/win32/stable/current.msixbundle" + with pytest.raises(ValueError, match="immutable"): + plan_transitions(old, mutable, BASE) + for suffix in ("../other.zip", "%2e%2e/other.zip", "%252e%252e/other.zip"): + traversal = copy.deepcopy(new) + traversal["packages"][0]["artifact"]["url"] = f"{BASE}/releases/tag/{new['tag']}/{suffix}" + with pytest.raises(ValueError, match="path encoding"): + plan_transitions(old, traversal, BASE) + with pytest.raises(ValueError, match="increase"): + plan_transitions(new, old, BASE) + + +def test_manifest_digest_and_tag_movement_fail_closed(tmp_path, monkeypatch): + data = b'{"schema":1}' + + class Response(io.BytesIO): + def geturl(self): + return BASE + "/manifest.json" + + def opener(url, timeout): + return Response(data) + + assert read_manifest(BASE, hashlib.sha256(data).hexdigest(), opener=opener) == {"schema": 1} + with pytest.raises(ValueError, match="digest"): + read_manifest(BASE, "f" * 64, opener=opener) + commit = "a" * 40 + env = {"RELEASE_TAG": "v1.2.3", "GITHUB_SHA": commit, "GITHUB_REF": "refs/tags/v1.2.3"} + + def git(argv): + if argv[1] == "ls-remote": + return f"{'b' * 40}\trefs/tags/v1.2.3\n{commit}\trefs/tags/v1.2.3^{{}}" + return commit if argv[1] == "rev-parse" else "" + + assert check_tag(env, git) == ("v1.2.3", commit) + with pytest.raises(ValueError, match="moved"): + check_tag(env, lambda argv: f"{'c' * 40}\trefs/tags/v1.2.3" if argv[1] == "ls-remote" else git(argv)) + + repo = tmp_path / "repo" + remote = tmp_path / "remote.git" + repo.mkdir() + monkeypatch.chdir(repo) + subprocess.run(["git", "init", "-b", "main"], check=True, capture_output=True) + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + subprocess.run(["git", "config", "user.name", "fixture"], check=True) + subprocess.run(["git", "config", "user.email", "fixture@example.invalid"], check=True) + (repo / "input").write_text("first", encoding="utf-8") + subprocess.run(["git", "add", "input"], check=True) + subprocess.run(["git", "commit", "-m", "first"], check=True, capture_output=True) + actual = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + subprocess.run(["git", "remote", "add", "origin", str(remote)], check=True) + subprocess.run(["git", "tag", "v1.2.3"], check=True) + subprocess.run(["git", "push", "origin", "main", "v1.2.3"], check=True, capture_output=True) + env["GITHUB_SHA"] = actual + assert check_tag(env) == ("v1.2.3", actual) + subprocess.run(["git", "--git-dir", str(remote), "update-ref", "-d", "refs/tags/v1.2.3"], check=True) + with pytest.raises(ValueError, match="moved"): + check_tag(env) diff --git a/tests/test_termux_deb_version.py b/tests/test_termux_deb_version.py index 1092fe1587..1d8e15cb3e 100644 --- a/tests/test_termux_deb_version.py +++ b/tests/test_termux_deb_version.py @@ -16,7 +16,7 @@ def test_canary_tag_shape_matches_canonical(): """Invariant: the deb versioner accepts EXACTLY the canary tags the canonical release tooling mints. The canonical shape lives in hermes_cli/update_channel.py:_CANARY_TAG_RE (8-or-14-digit, 20-prefixed - timestamps); scripts/r2-release.mjs:channelForTag parses the same shape. + timestamps); scripts/releases/r2.py:channel_for_tag parses the same shape. A tag this module accepts but the release flow would never mint (or vice versa) is version-drift between the .deb channel and the feed channel. """