diff --git a/.github/workflows/desktop-bundled-release.yml b/.github/workflows/desktop-bundled-release.yml index 73925968a5..8bb0a20e27 100644 --- a/.github/workflows/desktop-bundled-release.yml +++ b/.github/workflows/desktop-bundled-release.yml @@ -18,7 +18,8 @@ name: Desktop Bundled Release # (needs build-win32): all downloadable builds stage without touching feeds. # smoke-darwin / smoke-win32 / smoke-win32-universal → native install + chat # publish-win32-updater → tested canary bytes + feed -# (needs BOTH Windows smoke matrices); stable promotion stays separately gated. +# (needs BOTH Windows smoke matrices); stable promotion is sequenced by +# the stable publication controller after GitHub publication. # stable-store → verified Store submission # (needs stable-publish): materialize the immutable accepted Store bundle # and submit via the MSStore CLI without rebuilding. @@ -2108,31 +2109,6 @@ jobs: test -f "${files[0]}" msstore publish "${files[0]}" -id "$MS_STORE_PRODUCT_ID" - stable-promote: - name: Promote verified stable bundle channels - needs: validate - if: inputs.release-phase == 'promote' - runs-on: ubuntu-latest-32-core - environment: release-signing - timeout-minutes: 90 - env: - RELEASE_TAG: ${{ inputs.tag }} - CANDIDATE_MANIFEST_SHA256: ${{ inputs.manifest-sha256 }} - GH_TOKEN: ${{ github.token }} - CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} - CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} - CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} - CLOUDFLARE_R2_BUCKET: ${{ vars.CLOUDFLARE_R2_BUCKET }} - CLOUDFLARE_R2_PUBLIC_URL: ${{ vars.CLOUDFLARE_R2_PUBLIC_URL }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ needs.validate.outputs.sha }} - - uses: ./.github/actions/setup-pm - with: - cache-python: false - - run: python -m scripts.ci.python_packages ruamel.yaml==0.18.17 -- -m scripts.bundles.release_artifacts promote --root verified - stable-phase-result: name: Stable bundle phase completed if: always() && inputs.release-phase != '' @@ -2148,8 +2124,7 @@ jobs: termux-deb, candidate-manifest, stable-publish, - stable-store, - stable-promote + stable-store ] runs-on: ubuntu-24.04 steps: @@ -2167,7 +2142,6 @@ jobs: phases = { 'candidate': ['validate', 'build-win32', 'build-darwin', 'assemble-win32-bundle', 'smoke-darwin', 'smoke-win32', 'smoke-win32-universal', 'termux-deb', 'candidate-manifest'], 'publish': ['validate', 'stable-publish', 'stable-store'], - 'promote': ['validate', 'stable-promote'], } require_success(json.loads(os.environ['RELEASE_NEEDS']), phases[os.environ['RELEASE_PHASE']]) PY diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0513272301..fc35d2ee82 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,17 +11,16 @@ on: # this same workflow, never across a workflow boundary. # # The ``release: published`` trigger was REMOVED on purpose: a GitHub release - # event must never rebuild or rewrite the stable Docker channel. Versioned - # tags, candidate tags and the stable/latest aliases are published only - # through the staged release path (workflow_call below), which the parent - # stable-release workflow gates behind full CI + package acceptance. + # event must never rebuild or rewrite the stable Docker channel. This staged + # path publishes only immutable version tags; the ordered stable publication + # controller moves stable/latest from the receipt-bound registry digest. pull_request: push: branches: [main] workflow_call: inputs: release-phase: - description: "Stable-release phase: 'test', 'publish' or 'promote'. Empty keeps the standalone triggers." + description: "Stable-release phase: 'test' or 'publish'. Empty keeps the standalone triggers." required: false type: string default: '' @@ -35,6 +34,10 @@ on: required: false type: string default: '' + outputs: + manifest-digest: + description: "Immutable digest of the published versioned multi-arch manifest." + value: ${{ jobs.release-publish-manifest.outputs.digest }} permissions: contents: read @@ -43,7 +46,7 @@ permissions: # its own image. PR runs reuse a PR-scoped group with # cancel-in-progress: true so rapid pushes to the same PR collapse to # the latest commit. Release runs include the run_id: several reusable calls -# (test/publish/promote) of this workflow live inside ONE parent run, and a +# (test/publish) of this workflow live inside ONE parent run, and a # shared group would cancel the parent mid-release. concurrency: group: docker-${{ github.event.pull_request.number || github.ref }}-${{ inputs.release-phase || 'standalone' }} @@ -71,7 +74,7 @@ jobs: case "$PHASE" in '') echo "phase=standalone" >> "$GITHUB_OUTPUT"; echo "release=false" >> "$GITHUB_OUTPUT" ;; test) echo "phase=test" >> "$GITHUB_OUTPUT"; echo "release=true" >> "$GITHUB_OUTPUT" ;; - publish|promote) + publish) echo "phase=$PHASE" >> "$GITHUB_OUTPUT"; echo "release=true" >> "$GITHUB_OUTPUT" ;; *) echo "::error::Invalid release-phase input: $PHASE"; exit 1 ;; esac @@ -379,8 +382,8 @@ jobs: # This is a registry-side operation — no building, no layer re-push — # so it runs in ~30 seconds. # - # Main pushes tag :main only. :latest is reserved for the release 'promote' - # phase (see release-promote below). + # Main pushes tag :main only. :latest is reserved for the ordered stable + # publication controller. # --------------------------------------------------------------------------- merge: if: >- @@ -550,14 +553,15 @@ jobs: if-no-files-found: error retention-days: 7 - # Assemble the versioned multi-arch manifest list from the pushed per-arch - # digests and emit the reference/digest manifest artifact consumed by the - # promote phase. Registry-side only; nothing is rebuilt. + # Assemble the immutable versioned multi-arch manifest and expose its digest + # to the parent release receipt. Registry-side only; nothing is rebuilt. release-publish-manifest: name: Assemble versioned manifest and digest receipt if: needs.mode.outputs.phase == 'publish' needs: [mode, release-publish] runs-on: ubuntu-latest + outputs: + digest: ${{ steps.list.outputs.digest }} timeout-minutes: 15 environment: container-publish env: @@ -670,7 +674,7 @@ jobs: release-phase-gate: name: Docker phase requirements met if: always() && inputs.release-phase != '' - needs: [mode, build, release-publish, release-publish-manifest, release-promote] + needs: [mode, build, release-publish, release-publish-manifest] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -680,7 +684,6 @@ jobs: BUILD: ${{ needs.build.result }} RELEASE_PUBLISH: ${{ needs.release-publish.result }} RELEASE_MANIFEST: ${{ needs.release-publish-manifest.result }} - RELEASE_PROMOTE: ${{ needs.release-promote.result }} MODE: ${{ needs.mode.result }} run: | set -euo pipefail @@ -697,107 +700,5 @@ jobs: [ "$RELEASE_MANIFEST" = success ] || failures+=("release-publish-manifest=$RELEASE_MANIFEST") [ "${#failures[@]}" -eq 0 ] || { printf '::error::%s\n' "${failures[@]}"; exit 1; } ;; - promote) - test "$RELEASE_PROMOTE" = success - ;; *) echo "::error::Unknown phase $PHASE"; exit 1 ;; esac - - # Release 'promote' phase: runs ONLY after the parent's global release gate - # (all bundle/acceptance/publication jobs succeeded). Repoints the - # user-facing stable aliases (stable AND latest) at the exact tested - # manifest-list digest from the publish phase, then reads the aliases back - # from the registry and fails if they do not resolve to that digest. - # This is the ONLY place in this workflow where stable/latest can move. - release-promote: - name: Promote stable Docker aliases - if: needs.mode.outputs.phase == 'promote' - needs: [mode] - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: container-publish - env: - RELEASE_TAG: ${{ inputs.tag }} - steps: - - name: Checkout release code (helper scripts only) - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ github.sha }} - - - name: Download release manifest from the publish phase (same run) - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: docker-publish-manifest-${{ inputs.tag }} - path: /tmp/manifest - - - name: Verify tested manifest identity - run: python3 -m scripts.releases.docker verify \ - --tag "$RELEASE_TAG" --commit "$GITHUB_SHA" /tmp/manifest/manifest.json - - - name: Set up Docker Buildx - id: buildx - continue-on-error: true - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - - name: Set up Docker Buildx (retry) - if: steps.buildx.outcome == 'failure' - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - - - name: Log in to Docker Hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Promote stable and latest to the tested digest - env: - IMAGE_NAME: ${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - LIST_DIGEST="$(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" - for i in 1 2 3; do - if docker buildx imagetools create \ - -t "${IMAGE_NAME}:stable" \ - -t "${IMAGE_NAME}:latest" \ - "${IMAGE_NAME}@${LIST_DIGEST}"; then - break - fi - if [ "$i" = 3 ]; then - echo "::error::imagetools create (promote) failed after 3 attempts" - exit 1 - fi - sleep 20 - done - - - name: Read back and verify the stable aliases - env: - IMAGE_NAME: ${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - EXPECTED="$(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" - sleep 10 # eventual consistency of just-created aliases - for alias in stable latest; do - for i in 1 2 3; do - got="$(docker buildx imagetools inspect "${IMAGE_NAME}:${alias}" \ - --format '{{json .Manifest.Digest}}' | tr -d '"')" && break - [ "$i" = 3 ] && { echo "::error::inspect ${alias} failed 3 times"; exit 1; } - sleep 20 - done - if [ "$got" != "$EXPECTED" ]; then - echo "::error::Alias ${alias} resolves to ${got}, expected ${EXPECTED}" - exit 1 - fi - echo "${IMAGE_NAME}:${alias} -> ${got} (verified)" - done - - - name: Promote receipt - env: - IMAGE_NAME: ${{ env.IMAGE_NAME }} - run: | - set -euo pipefail - { - echo "image: ${IMAGE_NAME}" - echo "tag: ${RELEASE_TAG}" - echo "digest: $(python3 -c 'import json;print(json.load(open("/tmp/manifest/manifest.json"))["list-digest"])')" - echo "aliases: stable,latest" - } | tee /tmp/manifest/promote-receipt.txt diff --git a/.github/workflows/stable-release-publication.yml b/.github/workflows/stable-release-publication.yml index d33159290b..6ed30fa384 100644 --- a/.github/workflows/stable-release-publication.yml +++ b/.github/workflows/stable-release-publication.yml @@ -20,7 +20,7 @@ concurrency: permissions: contents: write - actions: read + actions: write jobs: reconcile: @@ -42,6 +42,18 @@ jobs: with: toolchain: node cache-python: false + - name: Wait before retrying failed stable jobs + if: >- + github.event_name == 'workflow_run' && + github.event.workflow_run.run_attempt < 3 + run: sleep 900 + - name: Set up Docker Buildx for ordered alias promotion + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to Docker Hub for ordered alias promotion + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Reconcile stable drafts and protected heads run: python -m scripts.releases.sequencer env: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 92f051d1d0..1442719cd4 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -249,19 +249,10 @@ jobs: env: RELEASE_NEEDS: ${{ toJSON(needs) }} - promote-docker: - name: Advance stable Docker channel - needs: [admit, publication] - uses: ./.github/workflows/docker.yml - with: - release-phase: promote - tag: ${{ needs.admit.outputs.tag }} - version: ${{ needs.admit.outputs.version }} - complete: name: Stable release is green if: always() - needs: [admit, ci, docker, acceptance, candidates, publication, promote-docker, windows-packaged, macos-packaged] + needs: [admit, ci, docker, acceptance, candidates, publication, publish-docker, windows-packaged, macos-packaged] runs-on: ubuntu-24.04 environment: release-signing permissions: @@ -280,7 +271,7 @@ jobs: with: toolchain: node cache-python: false - - run: python -m scripts.releases.stable gate admit ci docker acceptance candidates publication promote-docker + - run: python -m scripts.releases.stable gate admit ci docker acceptance candidates publication publish-docker env: RELEASE_NEEDS: ${{ toJSON(needs) }} - name: Create the final tag and retarget the accepted release @@ -295,6 +286,7 @@ jobs: AUTOPUBLISH: ${{ inputs.autopublish }} CANDIDATE_MANIFEST_URL: ${{ needs.candidates.outputs.manifest-url }} CANDIDATE_MANIFEST_SHA256: ${{ needs.candidates.outputs.manifest-sha256 }} + DOCKER_MANIFEST_DIGEST: ${{ needs.publish-docker.outputs.manifest-digest }} CLOUDFLARE_R2_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_R2_ACCOUNT_ID }} CLOUDFLARE_R2_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_ACCESS_KEY_ID }} CLOUDFLARE_R2_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }} @@ -309,6 +301,13 @@ jobs: run: | python scripts/render-builds-table.py --tag "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \ --candidate-manifest-sha256 "$CANDIDATE_MANIFEST_SHA256" --candidate-commit "$RELEASE_COMMIT" + - name: Set up Docker Buildx for ordered alias promotion + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - name: Log in to Docker Hub for ordered alias promotion + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Reconcile ordered stable publication run: python -m scripts.releases.sequencer env: diff --git a/scripts/releases/channel_releases.py b/scripts/releases/channel_releases.py index abf0cb8bd6..7afd1f8941 100644 --- a/scripts/releases/channel_releases.py +++ b/scripts/releases/channel_releases.py @@ -184,6 +184,9 @@ def advance_stable(env: dict, release: dict, root: Path) -> dict: found = store.get(key) if found is None: raise ChannelError("Stable candidate manifest is unavailable") + digest = hashlib.sha256(found[0]).hexdigest() + if digest != release.get("candidate_manifest_sha256"): + raise ChannelError("Stable candidate manifest differs from the final release receipt") scoped_env = { **env, "RELEASE_TAG": release["tag"], @@ -191,7 +194,7 @@ def advance_stable(env: dict, release: dict, root: Path) -> dict: "RELEASE_CLAIM_TAG": release["claim_tag"], "RELEASE_CLAIM_OBJECT": release["claim_object"], "CANDIDATE_MANIFEST_URL": f"{public_base}/{key}", - "CANDIDATE_MANIFEST_SHA256": hashlib.sha256(found[0]).hexdigest(), + "CANDIDATE_MANIFEST_SHA256": digest, } return publish_release("stable-release", scoped_env, root) diff --git a/scripts/releases/docker.py b/scripts/releases/docker.py index 965a4e2def..7a9b7878ac 100644 --- a/scripts/releases/docker.py +++ b/scripts/releases/docker.py @@ -5,13 +5,16 @@ import argparse import hashlib import json import re +import subprocess import sys +import time MANIFEST_SCHEMA = 1 SHA256 = re.compile(r"[a-f0-9]{64}") GIT_SHA = re.compile(r"[a-f0-9]{40}") from hermes_cli.update_channel import STABLE_TAG_RE ARCHES = ("amd64", "arm64") +IMAGE = "nousresearch/hermes-agent" class DockerReleaseError(ValueError): """Raised when a phase/manifest violates the staged-release contract.""" @@ -87,6 +90,46 @@ def sha256_file(path: str) -> str: return digest.hexdigest() +def output(argv: list[str]) -> str: + return subprocess.check_output(argv, text=True, encoding="utf-8").strip().strip('"') + + +def _inspect(reference: str, run) -> str: + return run([ + "docker", "buildx", "imagetools", "inspect", reference, + "--format", "{{json .Manifest.Digest}}", + ]).strip('"') + + +def promote_stable(tag: str, digest: str, *, run=output, sleep=time.sleep) -> None: + """Move stable aliases from the immutable versioned registry receipt.""" + require_stable_tag(tag) + if not re.fullmatch(r"sha256:[a-f0-9]{64}", digest): + raise DockerReleaseError("Invalid published manifest-list digest") + if _inspect(f"{IMAGE}:{tag}", run) != digest: + raise DockerReleaseError("Docker versioned tag differs from the final release receipt") + command = [ + "docker", "buildx", "imagetools", "create", "-t", f"{IMAGE}:stable", + "-t", f"{IMAGE}:latest", f"{IMAGE}@{digest}", + ] + for attempt in range(3): + try: + run(command) + break + except subprocess.CalledProcessError: + if attempt == 2: + raise + sleep(20) + for alias in ("stable", "latest"): + for attempt in range(3): + if _inspect(f"{IMAGE}:{alias}", run) == digest: + break + if attempt < 2: + sleep(20) + else: + raise DockerReleaseError(f"Docker {alias} alias read-back mismatch") + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) diff --git a/scripts/releases/entrypoint.py b/scripts/releases/entrypoint.py index 406acbe886..b2415c96f2 100644 --- a/scripts/releases/entrypoint.py +++ b/scripts/releases/entrypoint.py @@ -32,6 +32,37 @@ def _claim_commit(repo: Path, tag: str) -> str: return _git(repo, "rev-parse", f"{tag}^{{commit}}") +def _refresh_claims(repo: Path, remote: str) -> None: + _git( + repo, "fetch", remote, + "+refs/heads/main:refs/remotes/hermes-release/main", + "+refs/tags/v*-rc:refs/tags/v*-rc", + ) + + +def _require_remote_main(repo: Path, commit: str) -> None: + result = subprocess.run( + ["git", "merge-base", "--is-ancestor", commit, "refs/remotes/hermes-release/main"], + cwd=repo, capture_output=True, + ) + if result.returncode != 0: + raise ReleaseRefused(f"{commit} is not on origin/main") + + +def _claim_collision(repo: Path, remote: str, tag: str, error: Exception) -> ReleaseRefused: + subprocess.run(["git", "tag", "--delete", tag], cwd=repo, capture_output=True) + try: + _git(repo, "fetch", remote, f"+refs/tags/{tag}:refs/tags/{tag}") + details = _git( + repo, "for-each-ref", f"refs/tags/{tag}", + "--format=%(taggername)|%(taggerdate:iso-strict)|%(*objectname)", + ) + except subprocess.CalledProcessError: + return ReleaseRefused(f"claim {tag} could not be pushed: {error}") + actor, when, commit = details.split("|", 2) + return ReleaseRefused(f"{tag} was claimed by {actor} at {when} for {commit}") + + def _highest_claim(repo: Path) -> tuple[str, str] | None: """The highest-version outstanding claim, as (version, commit).""" from scripts.releases.versioning import version_from_tag @@ -65,6 +96,8 @@ def _require_ancestry(repo: Path, commit: str) -> None: def release(commit: str, *, bump: str, repo: Path, remote: str, repository: str, execute, autopublish: bool = False) -> dict: """Claim the derived version, cut its draft, and start the gate.""" + _refresh_claims(repo, remote) + _require_remote_main(repo, commit) _require_ancestry(repo, commit) version = derive_next_version(published=None, claims=_claims(repo), bump=bump) tag = f"v{version}-rc" @@ -75,7 +108,17 @@ def release(commit: str, *, bump: str, repo: Path, remote: str, repository: str, "autopublish": autopublish, }, sort_keys=True, separators=(",", ":")) _git(repo, "tag", "-a", tag, commit, "-m", claim) - _git(repo, "push", remote, f"refs/tags/{tag}") + try: + _git(repo, "push", remote, f"refs/tags/{tag}") + except subprocess.CalledProcessError as error: + raise _claim_collision(repo, remote, tag, error) from error + ref = f"refs/tags/{tag}" + remote_ref = dict(line.split()[::-1] for line in _git( + repo, "ls-remote", remote, ref, f"{ref}^{{}}", + ).splitlines()) + if (remote_ref.get(ref) != _git(repo, "rev-parse", ref) + or remote_ref.get(f"{ref}^{{}}") != commit): + raise ReleaseRefused(f"claim {tag} did not persist with exact remote custody") url = f"https://github.com/{repository}/releases/tag/{tag}" try: execute([ @@ -93,13 +136,39 @@ def release(commit: str, *, bump: str, repo: Path, remote: str, repository: str, "autopublish": autopublish} -def publish(version: str, *, repository: str, dispatch) -> dict: +def _preflight_publish(version: str, repository: str, inspect) -> None: + from scripts.releases.versioning import version_from_tag + + rows = json.loads(inspect([ + "gh", "release", "list", "--repo", repository, "--limit", "100", + "--json", "tagName,isDraft,isPrerelease", + ])) + requested = tuple(map(int, version.split("."))) + published = [] + family = False + for row in rows: + tag = row.get("tagName") + family = family or tag in {f"v{version}", f"v{version}-rc"} + parsed = version_from_tag(tag) + if parsed and row.get("isDraft") is False and row.get("isPrerelease") is False: + published.append((tuple(map(int, parsed.split("."))), parsed)) + newer = [found for key, found in published if key > requested] + if newer: + latest = max(newer, key=lambda item: tuple(map(int, item.split(".")))) + raise ReleaseRefused(f"stable {version} is burned or superseded by {latest}") + if not family: + raise ReleaseRefused(f"stable {version} is burned or has no release draft") + + +def publish(version: str, *, repository: str, dispatch, inspect=None) -> dict: """Request ordered publication through the one production sequencer.""" tag = f"v{version}" from hermes_cli.update_channel import STABLE_TAG_RE if not STABLE_TAG_RE.fullmatch(tag): raise ReleaseRefused(f"{version} is not a stable version") + if inspect is not None: + _preflight_publish(version, repository, inspect) dispatch([ "gh", "workflow", "run", "stable-release-publication.yml", "--repo", repository, "--raw-field", f"version={version}", @@ -151,9 +220,18 @@ def _execute(repo: Path, command: list[str]) -> None: raise ReleaseRefused(completed.stderr.strip() or "release command failed") +def _inspect(repo: Path, command: list[str]) -> str: + completed = subprocess.run(command, cwd=repo, capture_output=True, text=True, encoding="utf-8") + if completed.returncode != 0: + raise ReleaseRefused(completed.stderr.strip() or "release inspection failed") + return completed.stdout + + def cmd_publish(args) -> None: repo, repository = _command_repository(args) - publish(args.version, repository=repository, dispatch=lambda command: _execute(repo, command)) + publish(args.version, repository=repository, + dispatch=lambda command: _execute(repo, command), + inspect=lambda command: _inspect(repo, command)) def cmd_abandon(args) -> None: diff --git a/scripts/releases/sequencer.py b/scripts/releases/sequencer.py index fe88a971c6..b4568420e7 100644 --- a/scripts/releases/sequencer.py +++ b/scripts/releases/sequencer.py @@ -12,11 +12,17 @@ import re import subprocess import sys import tempfile +import time +from datetime import datetime, timedelta, timezone from pathlib import Path from hermes_cli.update_channel import STABLE_TAG_RE CLAIM_TAG_RE = re.compile(r"^(v(?:0|[1-9]\d{0,2})\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))-rc$") +SHA256 = re.compile(r"[a-f0-9]{64}") +DOCKER_DIGEST = re.compile(r"sha256:[a-f0-9]{64}") +MAX_ATTEMPTS = 3 +RETRY_BACKOFF = timedelta(minutes=15) def _key(version: str) -> tuple[int, int, int]: @@ -31,6 +37,7 @@ def plan(claims: list[dict], *, head: str | None, head_key = _key(head) if head is not None else None flips: list[dict] = [] advances: list[dict] = [] + flush_green_chain = False for index, claim in enumerate(ordered): version = claim["version"] @@ -59,11 +66,13 @@ def plan(claims: list[dict], *, head: str | None, claim.get("autopublish", False) or version == requested_version or has_later_live_claim + or flush_green_chain ) if not eligible: break flips.append({"flip": version}) advances.append({"advance": version}) + flush_green_chain = flush_green_chain or has_later_live_claim return flips + advances @@ -98,6 +107,54 @@ def _workflow_runs(repository: str, run=output) -> list[dict]: return rows +def _utc(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("Workflow timestamp must include a timezone") + return parsed.astimezone(timezone.utc) + + +def classify_runs(runs: list[dict]) -> tuple[str, dict | None]: + """Keep failed claims live until two failed-job retries are exhausted.""" + if not runs: + return "burned", None + run_ids = {row.get("id") for row in runs} + if len(run_ids) != 1 or None in run_ids: + raise ValueError("Stable claim owns multiple workflow runs") + latest = max(runs, key=lambda row: row.get("run_attempt", 0)) + attempt = latest.get("run_attempt") + if not isinstance(attempt, int) or attempt < 1: + raise ValueError("Stable workflow run attempt is invalid") + if latest.get("status") != "completed": + return "running", None + if latest.get("conclusion") == "success": + raise ValueError("Stable workflow succeeded without a final tag") + if attempt >= MAX_ATTEMPTS: + return "burned", None + updated_at = latest.get("updated_at") + if not isinstance(updated_at, str): + raise ValueError("Failed stable workflow has no completion time") + return "running", { + "run_id": latest["id"], + "attempt": attempt, + "due_at": _utc(updated_at) + RETRY_BACKOFF, + } + + +def retry_due(records: list[dict], *, now: datetime | None = None) -> list[dict]: + """Return bounded failed-job retries whose backoff has elapsed.""" + now = now or datetime.now(timezone.utc) + retries = [] + for record in records: + retry = record.get("retry") + if retry is not None and retry["due_at"] <= now: + retries.append({ + "version": record["version"], "run_id": retry["run_id"], + "attempt": retry["attempt"] + 1, + }) + return retries + + def _remote_tags(run=output) -> dict[str, dict[str, str]]: refs: dict[str, dict[str, str]] = {} for line in run(["git", "ls-remote", "--tags", "origin", "refs/tags/v*"]).splitlines(): @@ -153,6 +210,8 @@ def discover(repository: str, run=output) -> list[dict]: release = family_releases[0] if family_releases else None final_ref = refs.get(tag) needs_retarget = False + final = None + retry = None if final_ref is not None: if set(final_ref) != {"object", "commit"}: @@ -164,8 +223,12 @@ def discover(repository: str, run=output) -> list[dict]: "schema": 1, "version": version, "commit": commit, "claimTag": claim_tag, "claimTagObject": claim_ref["object"], "autopublish": claim["autopublish"], + "candidateManifestSha256": final.get("candidateManifestSha256"), + "dockerManifestDigest": final.get("dockerManifestDigest"), } - if final != expected_final: + if (final != expected_final + or not SHA256.fullmatch(final["candidateManifestSha256"] or "") + or not DOCKER_DIGEST.fullmatch(final["dockerManifestDigest"] or "")): raise ValueError(f"{tag} metadata differs from {claim_tag}") if release is None or release.get("prerelease") is not False: raise ValueError(f"{tag} has no valid GitHub release") @@ -187,12 +250,7 @@ def discover(repository: str, run=output) -> list[dict]: raise ValueError(f"{claim_tag} draft state is invalid") matching_runs = [row for row in workflow_runs if row.get("head_branch") == claim_tag and row.get("head_sha") == commit] - if any(row.get("status") != "completed" for row in matching_runs): - state = "running" - elif any(row.get("conclusion") == "success" for row in matching_runs): - raise ValueError(f"{claim_tag} succeeded without a final tag") - else: - state = "burned" + state, retry = classify_runs(matching_runs) records.append({ "version": version, @@ -204,6 +262,9 @@ def discover(repository: str, run=output) -> list[dict]: "commit": commit, "release_id": release.get("id") if release else None, "needs_retarget": needs_retarget, + "candidate_manifest_sha256": final["candidateManifestSha256"] if final else None, + "docker_manifest_digest": final["dockerManifestDigest"] if final else None, + "retry": retry if final_ref is None else None, }) return sorted(records, key=lambda record: _key(record["version"])) @@ -211,10 +272,29 @@ def discover(repository: str, run=output) -> list[dict]: def reconcile(env: dict, *, run=output, read_head=None, advance_head=None) -> list[dict]: """Converge GitHub publication and protected heads oldest-first.""" - from scripts.releases import channel_releases, stable + from scripts.releases import channel_releases, docker, stable repository = env["GITHUB_REPOSITORY"] records = discover(repository, run) + retries = retry_due(records) + if retries: + for retry in retries: + endpoint = f"repos/{repository}/actions/runs/{retry['run_id']}" + run([ + "gh", "api", "--method", "POST", + f"{endpoint}/rerun-failed-jobs", + ]) + for attempt in range(6): + current = json.loads(run(["gh", "api", endpoint])) + if (current.get("id") == retry["run_id"] + and current.get("run_attempt") == retry["attempt"] + and current.get("status") != "completed"): + break + if attempt < 5: + time.sleep(5) + else: + raise ValueError(f"Stable retry {retry['run_id']} did not enter attempt {retry['attempt']}") + return [{"retry": retry["version"], "attempt": retry["attempt"]} for retry in retries] for record in records: if record["needs_retarget"]: stable.retarget_release(repository, record["release_id"], record["tag"], @@ -237,6 +317,7 @@ def reconcile(env: dict, *, run=output, read_head=None, advance_head=None) -> li if advance_head is None: def production_advance(record: dict) -> None: + docker.promote_stable(record["tag"], record["docker_manifest_digest"]) with tempfile.TemporaryDirectory() as directory: channel_releases.advance_stable(env, record, Path(directory)) advance_head = production_advance diff --git a/scripts/releases/stable.py b/scripts/releases/stable.py index 900041f6a5..23def5888b 100644 --- a/scripts/releases/stable.py +++ b/scripts/releases/stable.py @@ -270,8 +270,12 @@ def final_context(env: dict, run=output) -> tuple[str, str, dict]: "schema": 1, "version": admitted["version"], "commit": commit, "claimTag": claim_tag, "claimTagObject": claim_object, "autopublish": claim["autopublish"], + "candidateManifestSha256": final.get("candidateManifestSha256"), + "dockerManifestDigest": final.get("dockerManifestDigest"), } - if final != expected: + if (final != expected + or not DIGEST.fullmatch(final["candidateManifestSha256"] or "") + or not re.fullmatch(r"sha256:[a-f0-9]{64}", final["dockerManifestDigest"] or "")): raise ValueError("Final tag metadata differs from its claim") release = json.loads(run([ "gh", "api", f"repos/{repository}/releases/tags/{tag}", @@ -280,7 +284,9 @@ def final_context(env: dict, run=output) -> tuple[str, str, dict]: or release.get("prerelease") is not False or not release.get("published_at")): raise ValueError("Stable channel requires the published final release") return tag, commit, {**admitted, "claim_object": claim_object, - "autopublish": claim["autopublish"]} + "autopublish": claim["autopublish"], + "candidate_manifest_sha256": final["candidateManifestSha256"], + "docker_manifest_digest": final["dockerManifestDigest"]} def emit(values: dict, env: dict) -> None: @@ -372,7 +378,23 @@ def transitions(env: dict) -> None: emit(matrices, env) -def ensure_final_tag(tag: str, commit: str, claim: dict, run=output) -> str: +def _final_metadata(tag: str, commit: str, claim: dict, candidate_manifest_sha256: str, + docker_manifest_digest: str) -> dict: + if not DIGEST.fullmatch(candidate_manifest_sha256): + raise ValueError("Final tag candidate manifest digest is invalid") + if not re.fullmatch(r"sha256:[a-f0-9]{64}", docker_manifest_digest): + raise ValueError("Final tag Docker manifest digest is invalid") + return { + "schema": 1, "version": tag[1:], "commit": commit, + "claimTag": claim["claim_tag"], "claimTagObject": claim["claim_object"], + "autopublish": claim["autopublish"], + "candidateManifestSha256": candidate_manifest_sha256, + "dockerManifestDigest": docker_manifest_digest, + } + + +def ensure_final_tag(tag: str, commit: str, claim: dict, *, candidate_manifest_sha256: str, + docker_manifest_digest: str, run=output) -> str: """Create or verify the immutable annotated final tag.""" require_stable_identity(tag, commit) ref = f"refs/tags/{tag}" @@ -381,11 +403,9 @@ def ensure_final_tag(tag: str, commit: str, claim: dict, run=output) -> str: try: local_object = run(["git", "rev-parse", "--verify", ref]) except subprocess.CalledProcessError: - message = json.dumps({ - "schema": 1, "version": tag[1:], "commit": commit, - "claimTag": claim["claim_tag"], "claimTagObject": claim["claim_object"], - "autopublish": claim["autopublish"], - }, sort_keys=True, separators=(",", ":")) + message = json.dumps(_final_metadata( + tag, commit, claim, candidate_manifest_sha256, docker_manifest_digest, + ), sort_keys=True, separators=(",", ":")) run([ "git", "-c", "user.name=Hermes Release Automation", "-c", "user.email=release-bot@users.noreply.github.com", @@ -408,6 +428,10 @@ def ensure_final_tag(tag: str, commit: str, claim: dict, run=output) -> str: local_object = run(["git", "rev-parse", ref]) if local_object != tag_object or run(["git", "cat-file", "-t", local_object]) != "tag": raise ValueError("Final stable tag object differs from the verified remote") + metadata = json.loads(run(["git", "tag", "-l", tag, "--format=%(contents)"])) + if metadata != _final_metadata( + tag, commit, claim, candidate_manifest_sha256, docker_manifest_digest): + raise ValueError("Final stable tag metadata differs from the accepted artifacts") return tag_object @@ -441,7 +465,11 @@ def complete(env: dict) -> None: base = env["CLOUDFLARE_R2_PUBLIC_URL"].rstrip("/") candidate = read_candidate(env) validate_candidates(candidate, tag, commit, base) - ensure_final_tag(tag, commit, claim) + ensure_final_tag( + tag, commit, claim, + candidate_manifest_sha256=env["CANDIDATE_MANIFEST_SHA256"], + docker_manifest_digest=env.get("DOCKER_MANIFEST_DIGEST", ""), + ) release_id = env.get("RELEASE_ID", "") if not str(release_id).isdigit(): raise ValueError("Stable release database ID is required") diff --git a/tests/ci/test_desktop_release_tag_admission.py b/tests/ci/test_desktop_release_tag_admission.py index 82124e2b4d..0413be406d 100644 --- a/tests/ci/test_desktop_release_tag_admission.py +++ b/tests/ci/test_desktop_release_tag_admission.py @@ -60,7 +60,9 @@ def test_signing_jobs_pin_source_and_controller_revisions_not_mutable_tags(): continue ref = step.get("with", {}).get("ref") expected = "${{ needs.validate.outputs.sha }}" - if name in {"publish-channel"} or step.get("if") == "needs.validate.outputs.channel-build != ''": + if (name in {"publish-channel"} + or step.get("if") == "needs.validate.outputs.channel-build != ''" + or step.get("name") == "Return to the trusted receipt controller"): expected = "${{ github.sha }}" elif name == "validate": expected = "${{ (inputs.build_commit != '' || inputs.channel != '' || inputs.release-phase != '') && github.sha || inputs.tag }}" diff --git a/tests/ci/test_stable_release_graph.py b/tests/ci/test_stable_release_graph.py index 0dee7aef29..356ae0e8d0 100644 --- a/tests/ci/test_stable_release_graph.py +++ b/tests/ci/test_stable_release_graph.py @@ -27,16 +27,15 @@ def test_release_reuses_whole_ci_and_docker_before_publication(): assert jobs["ci"]["uses"] == "./.github/workflows/ci.yaml" assert jobs["ci"]["with"]["release"] == "true" assert "secrets" not in jobs["ci"] - assert jobs["docker"]["uses"] == jobs["publish-docker"]["uses"] == jobs["promote-docker"]["uses"] + assert jobs["docker"]["uses"] == jobs["publish-docker"]["uses"] assert jobs["docker"]["with"]["release-phase"] == "test" assert "ci" in ancestors(jobs, "docker") required = {"ci", "docker", "nix", "pm-bundle", "install-e2e", "windows-packaged", "macos-packaged", "termux-checks", "windows-live", "candidates"} assert required <= ancestors(jobs, "acceptance") for name in ("publish-docker", "publish-bundles"): assert required <= ancestors(jobs, name) - assert {"publish-docker", "publish-bundles", "publication"} <= ancestors(jobs, "promote-docker") - assert "promote-docker" in ancestors(jobs, "complete") - assert "promote-bundles" not in jobs + assert {"publish-docker", "publish-bundles", "publication"} <= ancestors(jobs, "complete") + assert "promote-docker" not in jobs and "promote-bundles" not in jobs for name in ("acceptance", "publication", "complete"): assert jobs[name]["if"] == "always()" @@ -64,6 +63,8 @@ def test_claim_custody_and_final_payload_identity_reach_every_privileged_phase() assert jobs[name]["with"]["version"] == "${{ needs.admit.outputs.version }}" complete = jobs["complete"]["steps"] final = next(i for i, step in enumerate(complete) if step.get("name", "").startswith("Create the final tag")) + assert complete[final]["env"]["DOCKER_MANIFEST_DIGEST"] == \ + "${{ needs.publish-docker.outputs.manifest-digest }}" render = next(i for i, step in enumerate(complete) if step.get("name", "").startswith("Render the admitted")) reconcile = next(i for i, step in enumerate(complete) if step.get("name", "").startswith("Reconcile ordered")) assert final < render < reconcile @@ -82,8 +83,10 @@ def test_publication_reconciler_has_every_recovery_trigger_and_shared_lock(): } reconcile = publication["jobs"]["reconcile"] assert reconcile["environment"] == "release-signing" - assert publication["permissions"] == {"contents": "write", "actions": "read"} + assert publication["permissions"] == {"contents": "write", "actions": "write"} assert "conclusion != 'success'" in reconcile["if"] checkout = reconcile["steps"][0] assert checkout["with"]["ref"] == "${{ github.event.repository.default_branch }}" assert checkout["with"]["persist-credentials"] == "false" + wait = next(step for step in reconcile["steps"] if step.get("name", "").startswith("Wait before")) + assert wait["run"] == "sleep 900" diff --git a/tests/scripts/test_release_docker_cli.py b/tests/scripts/test_release_docker_cli.py index 99516c3094..9b4abb7620 100644 --- a/tests/scripts/test_release_docker_cli.py +++ b/tests/scripts/test_release_docker_cli.py @@ -6,6 +6,8 @@ from pathlib import Path import subprocess import sys +import pytest + ROOT = Path(__file__).resolve().parents[2] @@ -49,3 +51,26 @@ def test_cli_manifest_and_verify(tmp_path): out.write_text(json.dumps(bad) if change else 'not json', encoding='utf-8') result = cli('verify', *identity, str(out)) assert result.returncode == 1 and '::error::' in result.stderr + + +def test_promotion_reuses_the_receipt_digest_without_rebuilding(): + from scripts.releases.docker import DockerReleaseError, promote_stable + + digest = 'sha256:' + 'd' * 64 + calls = [] + + def run(argv): + calls.append(argv) + if argv[:4] == ['docker', 'buildx', 'imagetools', 'inspect']: + return digest + if argv[:4] == ['docker', 'buildx', 'imagetools', 'create']: + return '' + raise AssertionError(argv) + + promote_stable('v1.2.3', digest, run=run) + create = next(argv for argv in calls if argv[3] == 'create') + assert create[-1] == f'nousresearch/hermes-agent@{digest}' + assert all('build' not in argv for argv in calls) + + with pytest.raises(DockerReleaseError, match='versioned tag'): + promote_stable('v1.2.3', digest, run=lambda _argv: 'sha256:' + 'e' * 64) diff --git a/tests/scripts/test_release_entrypoint.py b/tests/scripts/test_release_entrypoint.py index fa437c218c..84821dc326 100644 --- a/tests/scripts/test_release_entrypoint.py +++ b/tests/scripts/test_release_entrypoint.py @@ -5,6 +5,8 @@ never starts is an error, not a warning the operator has to notice. """ import json import subprocess +import threading +import time import pytest @@ -97,7 +99,7 @@ def test_a_dispatch_that_never_starts_is_an_error(source): def test_publish_dispatches_the_sequencer_and_abandon_keeps_the_claim(source): - from scripts.releases.entrypoint import abandon, publish + from scripts.releases.entrypoint import ReleaseRefused, abandon, publish commit = git(source, "rev-parse", "HEAD") _claim(source, "0.21.5", commit) @@ -113,3 +115,74 @@ def test_publish_dispatches_the_sequencer_and_abandon_keeps_the_claim(source): assert abandoned["burned"] == "0.21.5" assert calls[-1] == ["gh", "release", "delete", "v0.21.5-rc", "--repo", "example/hermes-agent", "--yes"] assert "v0.21.5-rc" in git(source, "tag", "--list") + + with pytest.raises(ReleaseRefused, match="burned or superseded by 0\\.21\\.6"): + publish( + "0.21.5", repository="example/hermes-agent", + dispatch=lambda _command: pytest.fail("superseded publish must not dispatch"), + inspect=lambda _command: json.dumps([ + {"tagName": "v0.21.5-rc", "isDraft": True, "isPrerelease": False}, + {"tagName": "v0.21.6", "isDraft": False, "isPrerelease": False}, + ]), + ) + + +def test_concurrent_claim_loser_reports_the_remote_winner_and_the_version_stays_spent( + source, tmp_path, monkeypatch): + from scripts.releases import entrypoint + from scripts.releases.versioning import derive_next_version + + old = git(source, "rev-parse", "HEAD") + git(source, "commit", "--allow-empty", "--quiet", "-m", "later") + git(source, "push", "--quiet", "origin", "main") + new = git(source, "rev-parse", "HEAD") + origin = git(source, "remote", "get-url", "origin") + left, right = tmp_path / "left", tmp_path / "right" + git(tmp_path, "clone", "--quiet", origin, str(left)) + git(tmp_path, "clone", "--quiet", origin, str(right)) + for clone in (left, right): + git(clone, "config", "user.name", "Test") + git(clone, "config", "user.email", "test@example.test") + git(left, "checkout", "--quiet", old) + + barrier = threading.Barrier(2) + original_git = entrypoint._git + + def racing_git(repo, *args): + if args[:2] == ("push", "origin") and args[-1] == "refs/tags/v0.21.5-rc": + barrier.wait(timeout=10) + if repo == left: + time.sleep(0.1) + return original_git(repo, *args) + + monkeypatch.setattr(entrypoint, "_git", racing_git) + outcomes = {} + + def claim(name, repo, commit): + try: + outcomes[name] = entrypoint.release( + commit, bump="patch", repo=repo, remote="origin", + repository="example/hermes-agent", execute=lambda _command: None, + ) + except Exception as error: + outcomes[name] = error + + threads = [ + threading.Thread(target=claim, args=("old", left, old)), + threading.Thread(target=claim, args=("new", right, new)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + + assert outcomes["new"]["commit"] == new + assert isinstance(outcomes["old"], entrypoint.ReleaseRefused) + assert "was claimed by Test" in str(outcomes["old"]) + assert new in str(outcomes["old"]) + assert git(left, "rev-parse", "v0.21.5-rc^{commit}") == new + + fresh = tmp_path / "fresh" + git(tmp_path, "clone", "--quiet", origin, str(fresh)) + claims = git(fresh, "tag", "--list", "v*-rc").splitlines() + assert derive_next_version(published=None, claims=claims, bump="patch") == "0.21.6" diff --git a/tests/scripts/test_release_sequencer.py b/tests/scripts/test_release_sequencer.py index 18a31b659e..f988d5f906 100644 --- a/tests/scripts/test_release_sequencer.py +++ b/tests/scripts/test_release_sequencer.py @@ -40,8 +40,8 @@ def test_a_newer_green_release_flushes_the_older_waiting_draft(): ("0.21.6", "green", False), ), head="0.21.4") - assert _flips(steps) == ["0.21.5"] - assert steps[-1] == {"advance": "0.21.5"} + assert _flips(steps) == ["0.21.5", "0.21.6"] + assert steps[-2:] == [{"advance": "0.21.5"}, {"advance": "0.21.6"}] def test_green_progress_before_a_running_blocker_is_preserved(): @@ -55,6 +55,32 @@ def test_green_progress_before_a_running_blocker_is_preserved(): assert _flips(steps) == ["0.21.5"] assert steps[-1] == {"advance": "0.21.5"} + assert plan(_claims( + ("0.21.5", "running", False), + ("0.21.6", "green", True), + ), head="0.21.4") == [] + + +def test_failed_run_retries_twice_after_backoff_before_burning(): + from datetime import datetime, timezone + + from scripts.releases.sequencer import classify_runs, retry_due + + now = datetime(2026, 9, 22, 1, 30, tzinfo=timezone.utc) + failed = { + "id": 42, "status": "completed", "conclusion": "failure", + "run_attempt": 1, "updated_at": "2026-09-22T01:14:59Z", + } + state, retry = classify_runs([failed]) + assert state == "running" + assert retry_due([{"version": "0.21.5", "state": state, "retry": retry}], now=now) == [{ + "version": "0.21.5", "run_id": 42, "attempt": 2, + }] + failed["run_attempt"] = 2 + state, retry = classify_runs([failed]) + assert retry_due([{"version": "0.21.5", "state": state, "retry": retry}], now=now)[0]["attempt"] == 3 + failed["run_attempt"] = 3 + assert classify_runs([failed]) == ("burned", None) def test_a_burned_claim_is_spent_and_skipped(): @@ -125,7 +151,11 @@ def test_reconcile_discovers_custody_flips_then_advances_oldest_first(): claim_tag, tag = f"v{version}-rc", f"v{version}" claim_object, final_object = str(index) * 40, str(index + 2) * 40 claim = {"schema": 1, "version": version, "commit": commit, "autopublish": False} - final = {**claim, "claimTag": claim_tag, "claimTagObject": claim_object} + final = { + **claim, "claimTag": claim_tag, "claimTagObject": claim_object, + "candidateManifestSha256": "a" * 64, + "dockerManifestDigest": "sha256:" + "b" * 64, + } tags[claim_tag] = (claim_object, commit, claim) tags[tag] = (final_object, commit, final) releases.append({ diff --git a/tests/scripts/test_stable_release.py b/tests/scripts/test_stable_release.py index d73d5e7f00..3caa66b1bc 100644 --- a/tests/scripts/test_stable_release.py +++ b/tests/scripts/test_stable_release.py @@ -242,7 +242,11 @@ def test_claim_object_movement_and_lightweight_tags_fail_closed(tmp_path, monkey ["git", "rev-parse", ref], text=True, encoding="utf-8").strip()}) claim = check_claim(env) assert claim["commit"] == actual - final_object = ensure_final_tag("v1.2.3", actual, claim) + final_object = ensure_final_tag( + "v1.2.3", actual, claim, + candidate_manifest_sha256="c" * 64, + docker_manifest_digest="sha256:" + "d" * 64, + ) remote_final = subprocess.check_output( ["git", "ls-remote", "origin", "refs/tags/v1.2.3", "refs/tags/v1.2.3^{}"], text=True, encoding="utf-8",