diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 20cdba6ed3..b3510e61b8 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -98,12 +98,24 @@ on: type: string default: 'darwin-arm64,darwin-x64,win32-arm64,win32-x64,win32-bundle,linux-x64,linux-arm64,termux' outputs: - manifest-url: - value: ${{ jobs.candidate-manifest.outputs.manifest-url }} - description: Immutable candidate manifest - manifest-sha256: - value: ${{ jobs.candidate-manifest.outputs.manifest-sha256 }} - description: Digest of the candidate manifest + darwin-arm64-receipt-url: + value: ${{ jobs.stage-receipt-darwin-arm64.outputs.receipt-url }} + description: Immutable darwin-arm64 stable receipt + darwin-arm64-receipt-sha256: + value: ${{ jobs.stage-receipt-darwin-arm64.outputs.receipt-sha256 }} + description: Digest of the darwin-arm64 stable receipt + darwin-x64-receipt-url: + value: ${{ jobs.stage-receipt-darwin-x64.outputs.receipt-url }} + description: Immutable darwin-x64 stable receipt + darwin-x64-receipt-sha256: + value: ${{ jobs.stage-receipt-darwin-x64.outputs.receipt-sha256 }} + description: Digest of the darwin-x64 stable receipt + win32-bundle-receipt-url: + value: ${{ jobs.assemble-win32-bundle.outputs.receipt-url }} + description: Immutable win32-bundle stable receipt + win32-bundle-receipt-sha256: + value: ${{ jobs.assemble-win32-bundle.outputs.receipt-sha256 }} + description: Digest of the win32-bundle stable receipt workflow_dispatch: inputs: tag: @@ -1198,12 +1210,84 @@ jobs: shell: bash run: echo "::notice::linux bundled builds are disabled for now — re-enable in desktop-bundled-release.yml" + stage-receipt-darwin-arm64: + name: Stage the darwin-arm64 stable receipt + # The receipt is staged at the end of the build join and before the smoke: + # the install arm may start against bytes whose smoke has not run yet + # (decision 11); acceptance still blocks publication. + needs: [validate, build-darwin-arm64] + if: >- + !cancelled() && inputs.release-phase == 'candidate' + && needs.validate.result == 'success' && needs.validate.outputs.sha != '' + && needs.validate.outputs.darwin-arm64 == 'true' + && needs.build-darwin-arm64.result == 'success' + runs-on: ubuntu-24.04 + environment: release-signing + timeout-minutes: 15 + outputs: + receipt-url: ${{ steps.receipt.outputs.receipt-url }} + receipt-sha256: ${{ steps.receipt.outputs.receipt-sha256 }} + env: + RELEASE_TAG: ${{ inputs.tag }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ needs.validate.outputs.public-root }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + # Claim custody is re-checked in place: full history and the tags. + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/setup-pm + with: + cache-python: false + - id: receipt + run: python -m scripts.releases.stable stage-receipt --receipt darwin-arm64 + + stage-receipt-darwin-x64: + name: Stage the darwin-x64 stable receipt + needs: [validate, build-darwin-x64] + if: >- + !cancelled() && inputs.release-phase == 'candidate' + && needs.validate.result == 'success' && needs.validate.outputs.sha != '' + && needs.validate.outputs.darwin-x64 == 'true' + && needs.build-darwin-x64.result == 'success' + runs-on: ubuntu-24.04 + environment: release-signing + timeout-minutes: 15 + outputs: + receipt-url: ${{ steps.receipt.outputs.receipt-url }} + receipt-sha256: ${{ steps.receipt.outputs.receipt-sha256 }} + env: + RELEASE_TAG: ${{ inputs.tag }} + CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} + CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} + CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} + CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ needs.validate.outputs.public-root }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.validate.outputs.sha }} + # Claim custody is re-checked in place: full history and the tags. + fetch-depth: 0 + fetch-tags: true + - uses: ./.github/actions/setup-pm + with: + cache-python: false + - id: receipt + run: python -m scripts.releases.stable stage-receipt --receipt darwin-x64 + smoke-darwin-arm64: name: Smoke macOS arm64 dmg/zip - needs: [validate, build-darwin-arm64] + needs: [validate, build-darwin-arm64, stage-receipt-darwin-arm64] if: >- !cancelled() && needs.validate.result == 'success' && needs.validate.outputs.sha != '' && needs.build-darwin-arm64.result == 'success' && needs.validate.outputs.darwin-arm64 == 'true' + && needs.stage-receipt-darwin-arm64.result == 'success' && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && (inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' || inputs.channel != '') permissions: @@ -1231,10 +1315,11 @@ jobs: smoke-darwin-x64: name: Smoke macOS x64 dmg/zip - needs: [validate, build-darwin-x64] + needs: [validate, build-darwin-x64, stage-receipt-darwin-x64] if: >- !cancelled() && needs.validate.result == 'success' && needs.validate.outputs.sha != '' && needs.build-darwin-x64.result == 'success' && needs.validate.outputs.darwin-x64 == 'true' + && needs.stage-receipt-darwin-x64.result == 'success' && (inputs.release-phase == '' || inputs.release-phase == 'candidate') && (inputs.upload_release == true || inputs.release-phase == 'candidate' || inputs.build_commit != '' || inputs.channel != '') permissions: @@ -1329,6 +1414,9 @@ jobs: environment: release-signing cache-mode: read timeout-minutes: 45 + outputs: + receipt-url: ${{ steps.receipt.outputs.receipt-url }} + receipt-sha256: ${{ steps.receipt.outputs.receipt-sha256 }} env: CHANNEL_BUILD: ${{ needs.validate.outputs.channel-build }} CHANNEL_REQUEST_SHA256: ${{ needs.validate.outputs.channel-request-sha256 }} @@ -1489,6 +1577,17 @@ jobs: --name windows-universal --root apps/desktop/release --include '*.msixbundle' fi + - name: Stage the stable win32-bundle receipt + # Same receipt-before-smoke contract as the darwin arches (decision 11): + # the bundle and its per-arch metadata are staged and digest-verified; + # acceptance still blocks publication. + if: inputs.release-phase == 'candidate' + id: receipt + env: + RELEASE_TAG: ${{ inputs.tag }} + CLOUDFLARE_R2_PUBLIC_URL: ${{ needs.validate.outputs.public-root }} + run: python -m scripts.releases.stable stage-receipt --receipt win32-bundle + publish-channel: name: Publish the complete native-smoked channel build needs: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 4626e45103..e1a8b70e7e 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -119,8 +119,8 @@ jobs: tag-count: '3' exclude-ref: ${{ inputs.tag }} - candidates: - name: Build signed release candidates + candidates-darwin-arm64: + name: Build signed release candidates (darwin-arm64) needs: [admit, ci, docker] permissions: contents: write @@ -133,10 +133,91 @@ jobs: claim-tag: ${{ needs.admit.outputs.claim-tag }} claim-object: ${{ needs.admit.outputs.claim-object }} release-phase: candidate + jobs: darwin-arm64 + + candidates-darwin-x64: + name: Build signed release candidates (darwin-x64) + needs: [admit, ci, docker] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + claim-tag: ${{ needs.admit.outputs.claim-tag }} + claim-object: ${{ needs.admit.outputs.claim-object }} + release-phase: candidate + jobs: darwin-x64 + + candidates-win32-arm64: + name: Build signed release candidates (win32-arm64) + needs: [admit, ci, docker] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + claim-tag: ${{ needs.admit.outputs.claim-tag }} + claim-object: ${{ needs.admit.outputs.claim-object }} + release-phase: candidate + jobs: win32-arm64 + + candidates-win32-x64: + name: Build signed release candidates (win32-x64) + needs: [admit, ci, docker] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + claim-tag: ${{ needs.admit.outputs.claim-tag }} + claim-object: ${{ needs.admit.outputs.claim-object }} + release-phase: candidate + jobs: win32-x64 + + candidates-win32-bundle: + name: Build signed release candidates (win32-bundle) + needs: [admit, ci, docker, candidates-win32-arm64, candidates-win32-x64] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + claim-tag: ${{ needs.admit.outputs.claim-tag }} + claim-object: ${{ needs.admit.outputs.claim-object }} + release-phase: candidate + jobs: win32-bundle + + candidates-termux: + name: Build signed release candidates (termux) + needs: [admit, ci, docker] + permissions: + contents: write + actions: read + packages: write + id-token: write + uses: ./.github/workflows/desktop-bundled-release.yml + with: + tag: ${{ needs.admit.outputs.tag }} + claim-tag: ${{ needs.admit.outputs.claim-tag }} + claim-object: ${{ needs.admit.outputs.claim-object }} + release-phase: candidate + jobs: termux transitions: name: Pin actual OLD and NEW signed packages - needs: [admit, candidates] + needs: [admit, candidates-darwin-arm64, candidates-darwin-x64, candidates-win32-bundle] runs-on: ubuntu-24.04 environment: release-signing outputs: @@ -157,8 +238,12 @@ jobs: RELEASE_TAG: ${{ needs.admit.outputs.tag }} RELEASE_CLAIM_TAG: ${{ needs.admit.outputs.claim-tag }} RELEASE_CLAIM_OBJECT: ${{ needs.admit.outputs.claim-object }} - CANDIDATE_MANIFEST_URL: ${{ needs.candidates.outputs.manifest-url }} - CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} + RECEIPT_DARWIN_ARM64_URL: ${{ needs.candidates-darwin-arm64.outputs.darwin-arm64-receipt-url }} + RECEIPT_DARWIN_ARM64_SHA256: ${{ needs.candidates-darwin-arm64.outputs.darwin-arm64-receipt-sha256 }} + RECEIPT_DARWIN_X64_URL: ${{ needs.candidates-darwin-x64.outputs.darwin-x64-receipt-url }} + RECEIPT_DARWIN_X64_SHA256: ${{ needs.candidates-darwin-x64.outputs.darwin-x64-receipt-sha256 }} + RECEIPT_WIN32_BUNDLE_URL: ${{ needs.candidates-win32-bundle.outputs.win32-bundle-receipt-url }} + RECEIPT_WIN32_BUNDLE_SHA256: ${{ needs.candidates-win32-bundle.outputs.win32-bundle-receipt-sha256 }} BASELINE_MANIFEST_URL: ${{ inputs.baseline-manifest }} CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} @@ -262,7 +347,10 @@ jobs: acceptance: name: All release acceptance checks pass if: always() - needs: [admit, ci, docker, nix, pm-bundle, termux-checks, windows-live, install-e2e, candidates, transitions, windows-packaged, macos-packaged, bootstrap-version] + needs: [admit, ci, docker, nix, pm-bundle, termux-checks, windows-live, install-e2e, + candidates-darwin-arm64, candidates-darwin-x64, candidates-win32-arm64, + candidates-win32-x64, candidates-win32-bundle, candidates-termux, + transitions, windows-packaged, macos-packaged, bootstrap-version] runs-on: ubuntu-24.04 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -271,7 +359,7 @@ jobs: - uses: ./.github/actions/setup-pm with: cache-python: false - - run: python -m scripts.releases.stable gate admit ci docker nix pm-bundle termux-checks windows-live install-e2e candidates transitions windows-packaged macos-packaged bootstrap-version + - run: python -m scripts.releases.stable gate admit ci docker nix pm-bundle termux-checks windows-live install-e2e candidates-darwin-arm64 candidates-darwin-x64 candidates-win32-arm64 candidates-win32-x64 candidates-win32-bundle candidates-termux transitions windows-packaged macos-packaged bootstrap-version env: RELEASE_NEEDS: ${{ toJSON(needs) }} @@ -289,7 +377,8 @@ jobs: publish-bundles: name: Publish tested bundle artifacts - needs: [admit, acceptance, candidates] + # B4 wires the candidate-manifest digest back into manifest-sha256. + needs: [admit, acceptance] permissions: contents: write actions: read @@ -301,7 +390,6 @@ jobs: claim-tag: ${{ needs.admit.outputs.claim-tag }} claim-object: ${{ needs.admit.outputs.claim-object }} release-phase: publish - manifest-sha256: ${{ needs.candidates.outputs.manifest-sha256 }} publication: name: All artifact publication succeeded @@ -322,7 +410,7 @@ jobs: complete: name: Stable release is green if: always() - needs: [admit, ci, docker, acceptance, candidates, publication, publish-docker, windows-packaged, macos-packaged] + needs: [admit, ci, docker, acceptance, publication, publish-docker, windows-packaged, macos-packaged] runs-on: ubuntu-24.04 environment: release-signing permissions: @@ -341,17 +429,16 @@ jobs: with: toolchain: node cache-python: false - - run: python -m scripts.releases.stable gate admit ci docker acceptance candidates publication publish-docker + - run: python -m scripts.releases.stable gate admit ci docker acceptance publication publish-docker env: RELEASE_NEEDS: ${{ toJSON(needs) }} - name: Validate the accepted candidate archive + # B4 wires the candidate-manifest URL and digest outputs here. run: python -m scripts.releases.stable complete env: RELEASE_TAG: ${{ needs.admit.outputs.tag }} RELEASE_CLAIM_TAG: ${{ needs.admit.outputs.claim-tag }} RELEASE_CLAIM_OBJECT: ${{ needs.admit.outputs.claim-object }} - CANDIDATE_MANIFEST_URL: ${{ needs.candidates.outputs.manifest-url }} - CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} @@ -362,7 +449,6 @@ jobs: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ needs.admit.outputs.tag }} RELEASE_COMMIT: ${{ needs.admit.outputs.commit }} - CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} run: | python scripts/render-builds-table.py --tag "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \ --archive "$RELEASE_CLAIM_TAG" \ diff --git a/scripts/releases/stable.py b/scripts/releases/stable.py index 7b6a76eba6..6ac5f957a4 100644 --- a/scripts/releases/stable.py +++ b/scripts/releases/stable.py @@ -7,6 +7,7 @@ import os import re import subprocess import sys +import tempfile import tomllib import urllib.error import urllib.request @@ -89,6 +90,17 @@ RECEIPT_TARGETS = { "win32-bundle": ("windows/x64", "windows/arm64"), } +# The staged handoffs each receipt is assembled from. The Windows bundle +# receipt reads the per-arch metadata handoffs plus the universal bundle +# handoff; fetch() re-verifies every staged byte against its receipt digest, +# and validate_receipt enforces the signing facts (teamId, publisher). +RECEIPT_HANDOFFS = { + "darwin-arm64": ("darwin-arm64",), + "darwin-x64": ("darwin-x64",), + "win32-bundle": ("win32-x64", "win32-arm64", "windows-universal"), +} +RECEIPT_INCLUDES = ("metadata-*.json", "*.zip", "*.msixbundle") + def _validated_rows(manifest: dict, tag: str, commit: str, public_base: str, release_epoch: int | None, *, archive: str) -> dict: @@ -215,6 +227,81 @@ def plan_receipt_transitions(previous: dict, receipt_manifest: dict, receipt: st return [_transition_row(target, old[target], new[target]) for target in targets] +def _receipt_manifest(root: Path, receipt: str, tag: str, commit: str, archive: str, + public_base: str, release_epoch: int) -> dict: + """Rebuild one group's manifest rows from its staged, digest-verified handoffs.""" + from scripts.releases.handoff import fetch, receipt_name + from scripts.releases.r2 import staging_key_for + + names = RECEIPT_HANDOFFS.get(receipt) + if names is None: + raise ValueError(f"Unknown receipt: {receipt}") + # Re-downloading the staged bytes proves the group's handoff is complete + # and matches its receipt before this receipt is published. + fetch(tag=archive, commit=commit, names=list(names), root=root, + includes=list(RECEIPT_INCLUDES)) + digests = {} + for name in names: + for row in json.loads((root / receipt_name(name)).read_text(encoding="utf-8-sig"))["files"]: + digests[row["path"]] = row["sha256"] + rows = [json.loads(file.read_text(encoding="utf-8-sig")) + for file in sorted(root.glob("metadata-*.json"))] + universal = None + if receipt == "win32-bundle": + bundles = [file.name for file in root.glob("*.msixbundle") if not file.name.startswith("Store-")] + if len(bundles) != 1: + raise ValueError(f"Expected one universal bundle, found {len(bundles)}") + universal = bundles[0] + packages = [] + for row in rows: + filename = universal if row["platform"] == "windows" else row.get("filename") + if not filename or filename not in digests or not (root / filename).is_file(): + raise ValueError(f"Receipt {receipt} is missing staged bytes for " + f"{row['platform']}/{row['arch']}") + packages.append({ + key: value for key, value in { + **row, + "artifact": {"url": f"{public_base.rstrip('/')}/{staging_key_for(archive, filename)}", + "sha256": digests[filename]}, + }.items() if key != "filename" + }) + if row["platform"] != "windows" and not filename.endswith(".zip"): + raise ValueError(f"Receipt {receipt} needs a signed app ZIP for {row['platform']}/{row['arch']}") + if receipt == "win32-bundle": + from scripts.bundles.release_artifacts import validate_windows_bundle + + validate_windows_bundle(root / universal, + [row for row in rows if row["platform"] == "windows"]) + return {"schema": 2, "tag": tag, "commit": commit, "releaseEpoch": release_epoch, + "archive": archive, "packages": packages} + + +def stage_receipt(env: dict, receipt: str) -> None: + """Publish one group's signed receipt into its immutable attempt archive. + + The receipt names exactly that group's rows (decision 11): it is staged + before the group's smokes run, so it carries no smoke results, while + acceptance still blocks publication. + """ + from scripts.releases.r2 import put + + tag, commit, claim = stable_context(env) + base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") + archive = claim["claim_tag"] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = _receipt_manifest(root, receipt, tag, commit, archive, base, claim["claim_epoch"]) + validate_receipt(manifest, receipt, tag, commit, base, claim["claim_epoch"], archive=archive) + file = root / f"{receipt}-receipt.json" + file.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8") + put(tag=archive, key=file.name, file=str(file), immutable=True) + digest = hashlib.sha256(file.read_bytes()).hexdigest() + url = f"{base}/releases/tag/{archive}/{receipt}-receipt.json" + print(url) + print(digest) + emit({"receipt-url": url, "receipt-sha256": digest}, env) + + def read_manifest(url: str, expected_hash: str | None = None, *, expected_origin: str | None = None, opener=urllib.request.urlopen) -> dict: location = urlsplit(url) @@ -449,14 +536,30 @@ def verify(env: dict) -> None: "release-epoch": claim["claim_epoch"]}, env) -def transitions(env: dict) -> None: +RECEIPT_SOURCES = { + "darwin-arm64": "RECEIPT_DARWIN_ARM64", + "darwin-x64": "RECEIPT_DARWIN_X64", + "win32-bundle": "RECEIPT_WIN32_BUNDLE", +} + + +def _stage_transition(env: dict, archive: str, base: str, row: dict) -> dict: from scripts.releases.r2 import put - tag, commit, claim = stable_context(env) - base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") - archive = claim["claim_tag"] - candidate = read_candidate(env) - validate_candidates(candidate, tag, commit, base, claim["claim_epoch"], archive=archive) + transition = row["transition"] + name = f"acceptance-{row['target']}.json" + file = Path(env["RUNNER_TEMP"]) / name + file.write_text(json.dumps(transition), encoding="utf-8") + put(tag=archive, key=name, file=str(file), immutable=True) + url = f"{base}/releases/tag/{archive}/{name}" + if read_manifest(url) != transition: + raise ValueError("Transition manifest read-back mismatch") + return {"arch": transition["arch"], "manifest": url, "old": transition["old"]["tag"], + "id": row["target"], + "manifest_sha256": hashlib.sha256(file.read_bytes()).hexdigest()} + + +def _published_baseline(env: dict, base: str) -> dict: try: previous = read_manifest(env.get("BASELINE_MANIFEST_URL") or f"{base}/releases/stable/release-candidates.json", expected_origin=base) @@ -467,17 +570,27 @@ def transitions(env: dict) -> None: published = json.loads(output(["gh", "release", "view", previous["tag"], "--repo", env["GITHUB_REPOSITORY"], "--json", "tagName,isDraft,isPrerelease"])) if published["tagName"] != previous["tag"] or published["isDraft"] or published["isPrerelease"]: raise ValueError("Upgrade baseline must be a published stable release") + return previous + + +def _receipt_from_env(env: dict, receipt: str, prefix: str, base: str) -> dict: + digest = env.get(f"{prefix}_SHA256", "") + if not DIGEST.fullmatch(digest): + raise ValueError(f"Pinned {receipt} receipt digest is required") + return read_manifest(env[f"{prefix}_URL"], digest, expected_origin=base) + + +def transitions(env: dict) -> None: + tag, commit, claim = stable_context(env) + base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") + archive = claim["claim_tag"] + previous = _published_baseline(env, base) matrices = {"windows": {"include": []}, "macos": {"include": []}} - for row in plan_transitions(previous, candidate, base): - transition = row["transition"] - name = f"acceptance-{row['target']}.json" - file = Path(env["RUNNER_TEMP"]) / name - file.write_text(json.dumps(transition), encoding="utf-8") - put(tag=archive, key=name, file=file, immutable=True) - url = f"{base}/releases/tag/{archive}/{name}" - if read_manifest(url) != transition: - raise ValueError("Transition manifest read-back mismatch") - matrices[transition["platform"]]["include"].append({"arch": transition["arch"], "manifest": url, "old": transition["old"]["tag"], "id": row["target"], "manifest_sha256": hashlib.sha256(file.read_bytes()).hexdigest()}) + for receipt, prefix in RECEIPT_SOURCES.items(): + receipt_manifest = _receipt_from_env(env, receipt, prefix, base) + for row in plan_receipt_transitions(previous, receipt_manifest, receipt, base): + matrices[row["transition"]["platform"]]["include"].append( + _stage_transition(env, archive, base, row)) emit(matrices, env) @@ -635,9 +748,14 @@ def main(argv: list[str] | None = None, env: dict | None = None) -> None: summary("\n".join(f"- {name}: {needs.get(name, {}).get('result', 'missing')}" for name in argv[1:]), env) require_success(needs, argv[1:]) return + if argv and argv[0] == "stage-receipt": + if len(argv) != 3 or argv[1] != "--receipt" or argv[2] not in RECEIPT_TARGETS: + raise ValueError("Expected stage-receipt --receipt darwin-arm64|darwin-x64|win32-bundle") + stage_receipt(env, argv[2]) + return commands = {"admit": admit, "verify": verify, "transitions": transitions, "complete": complete} if len(argv) != 1 or argv[0] not in commands: - raise ValueError("Expected admit, verify, gate, transitions or complete") + raise ValueError("Expected admit, verify, gate, transitions, stage-receipt or complete") commands[argv[0]](env) diff --git a/tests/ci/test_stable_release_graph.py b/tests/ci/test_stable_release_graph.py index 56ca4d24de..71524738c8 100644 --- a/tests/ci/test_stable_release_graph.py +++ b/tests/ci/test_stable_release_graph.py @@ -37,8 +37,23 @@ def test_release_reuses_whole_ci_and_docker_before_publication(): assert jobs["docker"]["uses"] == jobs["publish-docker"]["uses"] assert jobs["docker"]["with"]["release-phase"] == "test" assert "ci" in ancestors(jobs, "docker") - required = {"ci", "docker", "nix", "pm-bundle", "install-e2e", "windows-packaged", "macos-packaged", "termux-checks", "windows-live", "candidates", "bootstrap-version"} + candidate_calls = ["candidates-darwin-arm64", "candidates-darwin-x64", "candidates-win32-arm64", + "candidates-win32-x64", "candidates-win32-bundle", "candidates-termux"] + required = {"ci", "docker", "nix", "pm-bundle", "install-e2e", "windows-packaged", "macos-packaged", + "termux-checks", "windows-live", "transitions", "bootstrap-version", *candidate_calls} assert required <= ancestors(jobs, "acceptance") + # B3: stable calls one build group at a time; the bundle group waits for + # both Windows arches, and the Linux groups are not called at all. + assert "candidates" not in jobs + for name, group in zip(candidate_calls, ("darwin-arm64", "darwin-x64", "win32-arm64", + "win32-x64", "win32-bundle", "termux")): + call = jobs[name] + assert call["uses"] == "./.github/workflows/desktop-bundled-release.yml" + assert call["with"]["release-phase"] == "candidate" + assert call["with"]["jobs"] == group + assert {"tag", "claim-tag", "claim-object"} <= set(call["with"]) + assert {"candidates-win32-arm64", "candidates-win32-x64"} <= set(jobs["candidates-win32-bundle"]["needs"]) + assert not any("linux" in name for name in jobs) # B5: publish-docker starts when the docker tests pass; it does not wait # for the acceptance join. publish-bundles still does. assert jobs["publish-docker"]["needs"] == ["admit", "docker"] @@ -65,7 +80,9 @@ def test_claim_custody_and_final_payload_identity_reach_every_privileged_phase() assert "autopublish" not in release["on"]["workflow_dispatch"]["inputs"] assert {"claim-tag", "claim-object", "tag", "commit", "version", "release-id", "release-epoch"} <= \ set(jobs["admit"]["outputs"]) - for name in ("candidates", "publish-bundles"): + for name in ("publish-bundles", *("candidates-darwin-arm64", "candidates-darwin-x64", + "candidates-win32-arm64", "candidates-win32-x64", + "candidates-win32-bundle", "candidates-termux")): call = jobs[name]["with"] assert call["tag"] == "${{ needs.admit.outputs.tag }}" assert call["claim-tag"] == "${{ needs.admit.outputs.claim-tag }}" @@ -74,6 +91,29 @@ def test_claim_custody_and_final_payload_identity_reach_every_privileged_phase() assert "release-epoch" in desktop["jobs"]["validate"]["outputs"] assert desktop["jobs"]["termux-deb"]["env"]["HERMES_RELEASE_EPOCH"] == \ "${{ needs.validate.outputs.release-epoch }}" + # B3: each build group stages its own receipt before its smoke, and the + # receipt URL and digest cross the call boundary as workflow outputs. + assert "manifest-url" not in desktop["on"]["workflow_call"]["outputs"] + assert "manifest-sha256" not in desktop["on"]["workflow_call"]["outputs"] + receipts = {"darwin-arm64": "stage-receipt-darwin-arm64", "darwin-x64": "stage-receipt-darwin-x64", + "win32-bundle": "assemble-win32-bundle"} + for group, job in receipts.items(): + producer = desktop["jobs"][job] + assert producer["outputs"]["receipt-url"] == "${{ steps.receipt.outputs.receipt-url }}" + assert producer["outputs"]["receipt-sha256"] == "${{ steps.receipt.outputs.receipt-sha256 }}" + for suffix, output in (("url", "receipt-url"), ("sha256", "receipt-sha256")): + expected = "${{ jobs." + job + ".outputs." + output + " }}" + assert desktop["on"]["workflow_call"]["outputs"][f"{group}-receipt-{suffix}"]["value"] == expected + for name in ("smoke-darwin-arm64", "smoke-darwin-x64"): + assert f"stage-receipt-{name.removeprefix('smoke-')}" in desktop["jobs"][name]["needs"] + for call, key, output in (("candidates-darwin-arm64", "RECEIPT_DARWIN_ARM64_URL", "darwin-arm64-receipt-url"), + ("candidates-darwin-arm64", "RECEIPT_DARWIN_ARM64_SHA256", "darwin-arm64-receipt-sha256"), + ("candidates-darwin-x64", "RECEIPT_DARWIN_X64_URL", "darwin-x64-receipt-url"), + ("candidates-darwin-x64", "RECEIPT_DARWIN_X64_SHA256", "darwin-x64-receipt-sha256"), + ("candidates-win32-bundle", "RECEIPT_WIN32_BUNDLE_URL", "win32-bundle-receipt-url"), + ("candidates-win32-bundle", "RECEIPT_WIN32_BUNDLE_SHA256", "win32-bundle-receipt-sha256")): + expected = "${{ needs." + call + ".outputs." + output + " }}" + assert jobs["transitions"]["steps"][-1]["env"][key] == expected for name in ("docker", "nix", "pm-bundle"): assert jobs[name]["with"]["version"] == "${{ needs.admit.outputs.version }}" for name in ("docker", "nix", "pm-bundle"): @@ -87,8 +127,8 @@ def test_claim_custody_and_final_payload_identity_reach_every_privileged_phase() if step.get("name", "").startswith("Validate the accepted candidate archive")) assert "DOCKER_MANIFEST_DIGEST" not in complete[validation]["env"] assert "RELEASE_ID" not in complete[validation]["env"] - assert complete[validation]["env"]["CANDIDATE_MANIFEST_SHA256"] == \ - "${{ needs.candidates.outputs.manifest-sha256 }}" + # B4 wires the candidate-manifest outputs back into these env entries. + assert "CANDIDATE_MANIFEST_SHA256" not in complete[validation]["env"] render = next(i for i, step in enumerate(complete) if step.get("name", "").startswith("Render the admitted")) reconcile = next(i for i, step in enumerate(complete) if step.get("name", "").startswith("Reconcile ordered")) assert validation < render < reconcile diff --git a/tests/scripts/test_stable_release.py b/tests/scripts/test_stable_release.py index d82577734f..8cb1c5741d 100644 --- a/tests/scripts/test_stable_release.py +++ b/tests/scripts/test_stable_release.py @@ -5,11 +5,13 @@ import json import os import subprocess import sys +import zipfile from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +from tests.scripts.test_release_r2 import r2_server # noqa: F401 from scripts.releases.draft_warning import ( WARNING_CLOSE, WARNING_OPEN, strip_draft_warning, ) @@ -655,3 +657,131 @@ def test_edit_draft_release_sends_the_notes_byte_for_byte(tmp_path, monkeypatch) assert release["body"].strip() == notes assert release["tag_name"] == "v1.2.3" and release["draft"] is True + + +# ── B3: stage-receipt ────────────────────────────────────────────────────── + +ATTEMPT = "rc.1-v1.2.3" +RECEIPT_COMMIT = "a" * 40 +WINDOWS_VERSION = "2026.5761.123.0" +RELEASE_EPOCH = 1_787_965_323 + + +def _fake_stable_context(): + def context(env): + return "v1.2.3", RECEIPT_COMMIT, {"claim_tag": ATTEMPT, "claim_object": "0" * 40, + "claim_epoch": RELEASE_EPOCH} + return context + + +def _stage_darwin_handoff(built, arch): + from scripts.releases import handoff + + package = f"HermesBundled-1.2.3-mac-{arch}.zip" + (built / package).write_bytes(f"signed mac zip: {arch}".encode()) + metadata = built / f"metadata-macos-{arch}.json" + metadata.write_text(json.dumps({ + "platform": "macos", "arch": arch, "tag": "v1.2.3", "commit": RECEIPT_COMMIT, + "baseVersion": "1.2.3", "identity": "Product", "version": "1.2.3", + "teamId": "ABCDEFGHIJ", "filename": package, + }), encoding="utf-8") + handoff.stage(ATTEMPT, RECEIPT_COMMIT, f"darwin-{arch}", built, [package, metadata.name]) + + +def _stage_windows_handoff(built, arch, *, with_metadata=True): + from scripts.releases import handoff + + package = f"HermesBundled-1.2.3-win-{arch}.msix" + (built / package).write_bytes(f"signed msix: {arch}".encode()) + includes = [package] + if with_metadata: + metadata = built / f"metadata-windows-{arch}.json" + metadata.write_text(json.dumps({ + "platform": "windows", "arch": arch, "tag": "v1.2.3", "commit": RECEIPT_COMMIT, + "baseVersion": "1.2.3", "identity": "Product", + "version": WINDOWS_VERSION, "executableVersion": WINDOWS_VERSION, + "publisher": "CN=Test", "applicationId": "App", + }), encoding="utf-8") + includes.append(metadata.name) + handoff.stage(ATTEMPT, RECEIPT_COMMIT, f"win32-{arch}", built, includes) + + +def _stage_universal_bundle(built): + from scripts.releases import handoff + + bundle = built / "Product-1.2.3-win.msixbundle" + with zipfile.ZipFile(bundle, "w") as archive: + archive.writestr("AppxMetadata/AppxBundleManifest.xml", + f'' + '' + '') + handoff.stage(ATTEMPT, RECEIPT_COMMIT, "windows-universal", built, ["*.msixbundle"]) + + +def _receipt_env(tmp_path, base): + return {"RELEASE_TAG": "v1.2.3", "CLOUDFLARE_R2_PUBLIC_URL": base, + "GITHUB_OUTPUT": str(tmp_path / "output")} + + +def test_stage_receipt_publishes_the_groups_signed_receipt(tmp_path, r2_server, https_origin, + monkeypatch, capsys): + from scripts.releases import stable + + https_origin.store = r2_server.store + built = tmp_path / "built" + built.mkdir() + _stage_darwin_handoff(built, "arm64") + monkeypatch.setattr(stable, "stable_context", _fake_stable_context()) + stable.main(["stage-receipt", "--receipt", "darwin-arm64"], _receipt_env(tmp_path, https_origin.base)) + + stored, _ = r2_server.store[f"releases/tag/{ATTEMPT}/darwin-arm64-receipt.json"] + receipt = json.loads(stored) + assert receipt["tag"] == "v1.2.3" and receipt["archive"] == ATTEMPT + assert receipt["releaseEpoch"] == RELEASE_EPOCH + assert [f"{row['platform']}/{row['arch']}" for row in receipt["packages"]] == ["macos/arm64"] + assert "smoke_results" not in receipt + url = f"{https_origin.base}/releases/tag/{ATTEMPT}/darwin-arm64-receipt.json" + digest = hashlib.sha256(stored).hexdigest() + printed = capsys.readouterr().out + assert url in printed and digest in printed + emitted = (tmp_path / "output").read_text(encoding="utf-8") + assert f"receipt-url={url}" in emitted and f"receipt-sha256={digest}" in emitted + + +def test_stage_receipt_publishes_both_windows_rows_from_the_bundle(tmp_path, r2_server, https_origin, + monkeypatch): + from scripts.releases import stable + + https_origin.store = r2_server.store + built = tmp_path / "built" + built.mkdir() + _stage_windows_handoff(built, "x64") + _stage_windows_handoff(built, "arm64") + _stage_universal_bundle(built) + monkeypatch.setattr(stable, "stable_context", _fake_stable_context()) + stable.main(["stage-receipt", "--receipt", "win32-bundle"], _receipt_env(tmp_path, https_origin.base)) + + stored, _ = r2_server.store[f"releases/tag/{ATTEMPT}/win32-bundle-receipt.json"] + receipt = json.loads(stored) + rows = {f"{row['platform']}/{row['arch']}": row for row in receipt["packages"]} + assert set(rows) == {"windows/x64", "windows/arm64"} + assert all(row["artifact"]["url"].endswith("Product-1.2.3-win.msixbundle") for row in rows.values()) + assert rows["windows/x64"]["artifact"]["url"].startswith(f"{https_origin.base}/releases/tag/{ATTEMPT}/") + assert rows["windows/x64"]["executableVersion"] == WINDOWS_VERSION + + +def test_stage_receipt_refuses_a_bundle_whose_arm64_row_is_absent(tmp_path, r2_server, https_origin, + monkeypatch): + from scripts.releases import stable + + https_origin.store = r2_server.store + built = tmp_path / "built" + built.mkdir() + _stage_windows_handoff(built, "x64") + # The arm64 leg staged its bytes but no metadata row: the receipt must refuse. + _stage_windows_handoff(built, "arm64", with_metadata=False) + _stage_universal_bundle(built) + monkeypatch.setattr(stable, "stable_context", _fake_stable_context()) + with pytest.raises(ValueError): + stable.main(["stage-receipt", "--receipt", "win32-bundle"], _receipt_env(tmp_path, https_origin.base)) + assert f"releases/tag/{ATTEMPT}/win32-bundle-receipt.json" not in r2_server.store