hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

test(ci): drop hardcoded desktop job ids from summary, archive and canary tests

Browse Source 
These tests still named desktop jobs by id, or compared `if:` expressions as
literal strings: build-win32-x64, build-win32-x64-release,
assemble-win32-bundle, publish-canary, builds-table, termux-deb.

- tag build summary: found as the table the canary publisher waits on. Its
  gate is evaluated to run over failed builds and to refuse a rejected tag,
  commit builds, dry runs, stable phases and channel builds. Failed needs are
  the selection gates, the termux builder and the Windows updater publisher,
  all found by role.
- termux/input archive: the validate archive step, the setup-pm archive step,
  the canary prune R2 steps and the Termux native gate are evaluated, not
  compared. evaluate() now resolves `env.*` for the R2-credential gates. "No
  build needs a separate archive job" now covers every job, not three ids.
- store eligibility and protected canary: jobs come from the assembler, native
  leg and canary publisher roles. Stage scripts come from the receipt the
  step stages.
This commit is contained in:
ethernet
2026-09-24 14:04:36 -04:00
parent 8f7da10406
commit 992433bde6
5 changed files with 96 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/ci/desktop_release_roles.py
View File

@@ -129,6 +129,12 @@ def commit_summary(jobs: dict) -> str:
"commit build summary")
def tag_summary(jobs: dict) -> str:
"""The tag build table, which the canary publisher waits on."""
canary = jobs[canary_publisher(jobs)]
return _only([name for name in needs_of(canary) if _renders(jobs[name], "--tag")], "tag build summary")
def canary_publisher(jobs: dict) -> str:
return _only([name for name, job in jobs.items()
if any("scripts.releases.channel_releases canary" in step.get("run", "")
@@ -181,7 +187,7 @@ def admitted(names, *, channel=False):
return {name: {"result": "success", "outputs": dict(outputs)} for name in names}
def evaluate(expression, inputs, needs, *, cancelled=False, failed=False, job_if=True, github=None):
def evaluate(expression, inputs, needs, *, cancelled=False, failed=False, job_if=True, github=None, env=None):
"""Evaluate the workflow expression subset, including Actions' implicit success.
This is not a scheduler simulation; native Actions still owns cancellation
@@ -199,12 +205,12 @@ def evaluate(expression, inputs, needs, *, cancelled=False, failed=False, job_if
def value(match):
bits = match[0].split('.')
result = {'inputs': inputs, 'needs': needs, 'github': github or {}}
result = {'inputs': inputs, 'needs': needs, 'github': github or {}, 'env': env or {}}
for bit in bits:
result = result.get(bit, '') if isinstance(result, dict) else ''
return repr(result)
expression = re.sub(r'\b(?:inputs|needs|github)(?:\.[\w-]+)+', value, expression)
expression = re.sub(r'\b(?:inputs|needs|github|env)(?:\.[\w-]+)+', value, expression)
expression = expression.replace('&&', ' and ').replace('||', ' or ')
expression = re.sub(r'!(?!=)', ' not ', expression)
expression = re.sub(r'\btrue\b', 'True', expression)

9
tests/ci/test_desktop_store_eligibility.py
View File

@@ -8,6 +8,7 @@ import sys
import pytest
from tests.ci.desktop_release_roles import native_builds, universal_assembler
from tests.ci.test_desktop_release_tag_admission import _BASH, _child_env, _workflow
@@ -34,7 +35,7 @@ def test_bundle_only_requests_store_for_stable(tmp_path, tag, commit, store):
env = _child_env(HERMES_PAYLOAD_TAG=tag, HERMES_BUILD_COMMIT=commit,
HERMES_PAYLOAD_VERSION='0.28.0', RELEASE_PHASE='candidate' if store else '', CALL_LOG=str(log))
env['PATH'] = str(helper) + os.pathsep + env['PATH']
job = jobs['assemble-win32-bundle']
job = jobs[universal_assembler(jobs)]
script = next(step['run'] for step in job['steps']
if step.get('name') == 'Assemble the signed MSIX bundle without publishing')
result = subprocess.run([_BASH, '-e', '-o', 'pipefail', '-c', script], env=env, cwd=tmp_path,
@@ -52,9 +53,11 @@ def test_bundle_only_requests_store_for_stable(tmp_path, tag, commit, store):
@pytest.mark.platforms('windows')
def test_native_windows_build_selects_store_only_for_stable(tmp_path):
jobs = _workflow()['jobs']
legs = native_builds(jobs)
scripts = {
kind: next(step['run'] for step in jobs[job]['steps'] if step.get('name') == 'Build and package')
for kind, job in [('release', 'build-win32-x64-release'), ('commit', 'build-win32-x64-commit')]
kind: next(step['run'] for step in jobs[legs[('win32-x64', kind)]]['steps']
if step.get('name') == 'Build and package')
for kind in ('release', 'commit')
}
cases = [
('v0.28.0', '', True, 'release'),

33
tests/ci/test_protected_canary_publication.py
View File

@@ -20,6 +20,7 @@ import pytest
from hermes_cli.release_channels import ChannelReader
from scripts.releases import channel_releases, handoff
from tests.ci.desktop_release_roles import canary_publisher, native_builds, stage_step
from tests.ci.test_desktop_release_tag_admission import _git, _seed_repo
from tests.scripts.test_release_r2 import r2_server # noqa: F401
@@ -27,12 +28,23 @@ ROOT = Path(__file__).resolve().parents[2]
pytestmark = pytest.mark.platforms("posix")
def workflow_job(name):
return hermes_yaml.safe_load((ROOT / ".github/workflows/desktop-bundled-release.yml").read_text())["jobs"][name]
def workflow_jobs():
return hermes_yaml.safe_load((ROOT / ".github/workflows/desktop-bundled-release.yml").read_text())["jobs"]
def canary_job():
jobs = workflow_jobs()
return jobs[canary_publisher(jobs)]
def stage_script(target, mode):
"""The tag/commit receipt stage of the native build leg for *target*."""
jobs = workflow_jobs()
return stage_step(jobs[native_builds(jobs)[(target, mode)]])["run"]
def step_script(job, name):
return next(step["run"] for step in workflow_job(job)["steps"] if step.get("name") == name)
return next(step["run"] for step in job["steps"] if step.get("name") == name)
@pytest.fixture
@@ -95,7 +107,7 @@ def canary(tmp_path, r2_server, monkeypatch):
"RELEASE_COMMIT": commit, "RELEASE_PHASE": "", "HERMES_DESKTOP_VARIANT": "bundled",
"HERMES_BUILD_COMMIT": "", "CHANNEL_BUILD": "", "R2_DISPOSABLE_RUN": "",
"CLOUDFLARE_R2_PUBLIC_URL": base,
"RELEASE_NEEDS": json.dumps({name: {"result": "success"} for name in workflow_job("publish-canary")["needs"]})}
"RELEASE_NEEDS": json.dumps({name: {"result": "success"} for name in canary_job()["needs"]})}
def run(script, **overrides):
return subprocess.run(["bash", "-e", "-o", "pipefail", "-c", script], cwd=clone,
@@ -146,8 +158,7 @@ def test_canary_metadata_is_recorded_and_receipt_bound(canary, r2_server, platfo
env["HERMES_BUILD_COMMIT"] = env["RELEASE_COMMIT"]
root = clone / "apps/desktop/release"
native_leg(root, identity, env, platform, "x64")
name = "Stage Windows packages to R2" if platform == "win32" else "Stage macOS packages and feed inputs to R2"
script = step_script(f"build-{platform}-x64-{'commit' if commit_build else 'release'}", name)
script = stage_script(f"{platform}-x64", "commit" if commit_build else "release")
result = run(script, **env, TARGET=f"{platform}-x64")
assert result.returncode == 0, result.stdout + result.stderr
prefix = f"releases/commit/{env['RELEASE_COMMIT']}/" if commit_build else f"releases/tag/{env['RELEASE_TAG']}/"
@@ -173,8 +184,7 @@ def stage_canary(clone, identity, env, run):
if root.exists():
shutil.rmtree(root)
native_leg(root, identity, env, platform, arch)
name = "Stage Windows packages to R2" if platform == "win32" else "Stage macOS packages and feed inputs to R2"
result = run(step_script(f"build-{platform}-{arch}-release", name), **env, TARGET=f"{platform}-{arch}")
result = run(stage_script(f"{platform}-{arch}", "release"), **env, TARGET=f"{platform}-{arch}")
assert result.returncode == 0, result.stdout + result.stderr
bundle = root / f"{identity['artifactNamePascal']}-{env['FIXTURE_WINDOWS_VERSION']}-win.msixbundle"
with zipfile.ZipFile(bundle, "w") as package:
@@ -187,9 +197,10 @@ def stage_canary(clone, identity, env, run):
def test_published_canary_workflow_advances_only_after_every_gate(canary, r2_server, monkeypatch, tmp_path):
clone, identity, env, run = canary
bundle = stage_canary(clone, identity, env, run)
script = step_script("publish-canary", "Publish the admitted canary and advance its protected head")
publisher = canary_job()
script = step_script(publisher, "Publish the admitted canary and advance its protected head")
before = dict(r2_server.store)
for job in workflow_job("publish-canary")["needs"]:
for job in publisher["needs"]:
for outcome in ("failure", "skipped", "cancelled", None):
needs = json.loads(env["RELEASE_NEEDS"])
if outcome is None:
@@ -228,7 +239,7 @@ def test_published_canary_workflow_advances_only_after_every_gate(canary, r2_ser
assert r2_server.store == before
# The controller owns the draft flip and its independent custody read-back.
release.write_text(json.dumps({**published, "isDraft": True}))
for step in workflow_job("publish-canary")["steps"]:
for step in publisher["steps"]:
if "run" in step:
result = run(step["run"])
assert result.returncode == 0, result.stdout + result.stderr

52
tests/ci/test_tag_builds_summary.py
View File

@@ -6,6 +6,10 @@ import sys
import pytest
from scripts.releases import r2
from tests.ci.desktop_release_roles import (
CANARY_TAG, DOWNLOADABLE_DISPATCHES, admitted, gate, needs_of, selection_gates, tag_summary, termux_builder,
updater_publishers,
)
from tests.ci.test_commit_build_staging import shell_step
from tests.ci.test_desktop_release_tag_admission import _workflow
from tests.scripts.test_release_r2 import r2_server # noqa: F401
@@ -13,18 +17,35 @@ from tests.scripts.test_release_r2 import r2_server # noqa: F401
@pytest.mark.parametrize("gh_available", [False, True])
def test_admitted_failure_publishes_tag_info_without_promoting_channel(tmp_path, r2_server, gh_available):
job = _workflow()["jobs"]["builds-table"]
gate = job["if"]
jobs = _workflow()["jobs"]
table = tag_summary(jobs)
job = jobs[table]
gates = selection_gates(jobs)
windows_publisher = updater_publishers(jobs)["win32"]
termux = termux_builder(jobs)
# This signing-context observer must survive failed needs without admitting
# rejected tags, commit builds, dry runs, or stable release phases.
for condition in ("always()", "needs.validate.result == 'success'",
"needs.validate.outputs.sha != ''", "inputs.build_commit == ''",
"inputs.upload_release == true", "inputs.release-phase == ''"):
assert condition in gate
tag = DOWNLOADABLE_DISPATCHES["tag"]
def observes(inputs, **validate):
needs = admitted(needs_of(job))
needs["validate"]["outputs"]["all-jobs"] = "true"
needs["validate"].update(validate)
for name in needs:
if name != "validate":
needs[name]["result"] = "failure"
return gate(job["if"], inputs, needs)
assert observes(tag)
assert not observes(tag, result="failure")
assert not observes(tag, outputs={"all-jobs": "true", "sha": ""})
for rejected in ({"build_commit": "a" * 40}, {"upload_release": False}, {"release-phase": "candidate"},
{"release-phase": "publish"}, {"channel": "preview"}):
assert not observes({**tag, **rejected}), rejected
render = next(step for step in job["steps"] if step.get("name") == "Render")
assert render["env"]["RELEASE_NEEDS"] == "${{ toJSON(needs) }}"
tag = "v0.28.0+canary.20260818T101010Z"
tag = CANARY_TAG
run_url = "https://github.example/o/r/actions/runs/12345"
base = f"http://127.0.0.1:{r2_server.server_port}/hermes-releases"
channel_key = "releases/canary/index.html"
@@ -51,13 +72,10 @@ def test_admitted_failure_publishes_tag_info_without_promoting_channel(tmp_path,
gh.write_text(f'#!/bin/sh\nexec {shlex.quote(sys.executable)} {shlex.quote(str(driver))} "$@"\n')
gh.chmod(0o755)
needs = {name: {"result": "success"} for name in job["needs"]}
needs["build-win32-x64"]["result"] = "failure"
needs["build-win32-arm64"]["result"] = "failure"
needs["build-darwin-arm64"]["result"] = "failure"
needs["build-darwin-x64"]["result"] = "failure"
needs["termux-deb"]["result"] = "failure"
needs["publish-win32-updater"]["result"] = "skipped"
result = shell_step(tmp_path, r2_server, "builds-table", "Render", {
for name in [*gates.values(), termux]:
needs[name]["result"] = "failure"
needs[windows_publisher]["result"] = "skipped"
result = shell_step(tmp_path, r2_server, table, "Render", {
"HERMES_PAYLOAD_TAG": tag, "GITHUB_REPOSITORY": "o/r",
"RUN_URL": run_url,
"RELEASE_NEEDS": json.dumps(needs), "CLOUDFLARE_R2_PUBLIC_URL": base,
@@ -71,7 +89,7 @@ def test_admitted_failure_publishes_tag_info_without_promoting_channel(tmp_path,
release_url = r2.public_url_for("https://github.com/o/r/releases/tag", tag)
assert f'href="{release_url}"' in page
assert "Build incomplete" in page
assert "build-win32-x64 (failure)" in page and "publish-win32-updater (skipped)" in page
assert f"{gates['win32-x64']} (failure)" in page and f"{windows_publisher} (skipped)" in page
assert "No downloadable artifacts" in page
for name, info in needs.items():
if info["result"] != "success":
@@ -86,8 +104,8 @@ def test_admitted_failure_publishes_tag_info_without_promoting_channel(tmp_path,
# not turn a failed Termux prerequisite into permission to publish a draft.
needs = {name: {"result": "success"} for name in job["needs"]}
for state in ("failure", "skipped", "success"):
needs["termux-deb"]["result"] = state
checked = shell_step(tmp_path, r2_server, "builds-table", "Preserve release success gate", {
needs[termux]["result"] = state
checked = shell_step(tmp_path, r2_server, table, "Preserve release success gate", {
"RELEASE_NEEDS": json.dumps(needs),
})
assert (checked.returncode == 0) is (state == "success"), checked.stdout + checked.stderr

37
tests/ci/test_termux_input_archive.py
View File

@@ -6,9 +6,16 @@ import pytest
from ruamel.yaml import YAML
from tests.ci.desktop_release_roles import DOWNLOADABLE_DISPATCHES, gate, needs_of
ROOT = Path(__file__).resolve().parents[2]
R2_ENV = {"CLOUDFLARE_R2_ACCOUNT_ID", "CLOUDFLARE_R2_ACCESS_KEY_ID", "CLOUDFLARE_R2_SECRET_ACCESS_KEY", "CLOUDFLARE_R2_BUCKET"}
R2_GATE = "env.CLOUDFLARE_R2_ACCOUNT_ID != ''"
def needs_r2_credentials(step):
"""The step runs with R2 credentials configured and skips without them."""
configured = gate(step["if"], {}, {}, job_if=False, env={"CLOUDFLARE_R2_ACCOUNT_ID": "account"})
return configured and not gate(step["if"], {}, {}, job_if=False, env={"CLOUDFLARE_R2_ACCOUNT_ID": ""})
def load(name):
@@ -43,7 +50,7 @@ def test_archive_gate_uses_bootstrap_python_and_trusted_exact_revision():
assert not any(s.get("uses") == "./.github/actions/setup-pm" for s in job["steps"])
(archive,) = [s for s in job["steps"] if s.get("run") == "python3 -m scripts.ci.archive_inputs"]
# Pushes to main without R2 credentials skip the archive rather than fail.
assert archive["if"] == R2_GATE
assert needs_r2_credentials(archive)
checkout = job["steps"][0]
assert checkout["with"]["ref"] == "${{ inputs.sha || github.sha }}"
release = load("desktop-bundled-release.yml")["jobs"]
@@ -51,22 +58,23 @@ def test_archive_gate_uses_bootstrap_python_and_trusted_exact_revision():
# a skipped archive (disposable/termux runs) must not skip the builds.
validate = release["validate"]
(archive,) = [s for s in validate["steps"] if s.get("run") == "python3 -m scripts.ci.archive_inputs"]
assert archive["if"] == (
"inputs.disposable_run == '' && inputs.disposable_channel == ''"
" && (inputs.release-phase == '' || inputs.release-phase == 'candidate')"
)
for dispatch, inputs in DOWNLOADABLE_DISPATCHES.items():
assert gate(archive["if"], inputs, {}, job_if=False), dispatch
for skipped in ({"disposable_run": "98765"}, {"disposable_channel": "native-preview"},
{"release-phase": "publish"}, {"release-phase": "promote"}):
assert not gate(archive["if"], {**DOWNLOADABLE_DISPATCHES["tag"], **skipped}, {}, job_if=False), skipped
assert R2_ENV <= archive["env"].keys()
assert validate["environment"] == "release-signing"
checkout = validate["steps"][0]
assert "actions/checkout" in checkout["uses"]
for name in ("build-win32-x64", "build-darwin-arm64", "termux-deb"):
assert "archive-inputs" not in release[name].get("needs", [])
assert not [name for name, job in release.items() if "archive-inputs" in needs_of(job)]
action = YAML(typ="base").load((ROOT / ".github/actions/setup-pm/action.yml").read_text(encoding="utf-8"))
assert action["inputs"]["archive-inputs"]["default"] == "false"
steps = action["runs"]["steps"]
archive_index, archive = next((i, s) for i, s in enumerate(steps) if "setup_toolchain.py\" archive-inputs" in s.get("run", ""))
assert archive["if"] == "inputs.archive-inputs == 'true'"
assert gate(archive["if"], {"archive-inputs": "true"}, {}, job_if=False)
assert not gate(archive["if"], {"archive-inputs": "false"}, {}, job_if=False)
assert archive_index < next(i for i, s in enumerate(steps) if s.get("id") == "install")
@@ -75,12 +83,15 @@ def test_scheduled_and_main_r2_consumers_skip_without_credentials():
prune = load("canary-release.yml")["jobs"]["prune"]
assert R2_ENV <= prune["env"].keys()
r2_steps = [s for s in prune["steps"] if "scripts.releases.r2" in s.get("run", "")]
assert r2_steps and all(s["if"] == R2_GATE for s in r2_steps)
assert r2_steps and all(needs_r2_credentials(s) for s in r2_steps)
termux = load("termux-verify.yml")
assert termux["on"]["push"]["branches"] == ["main"] and "pull_request" in termux["on"]
native = termux["jobs"]["native-runtime"]
gate = termux["jobs"][native["needs"]]
assert gate["environment"] == "release-signing"
assert native["if"] == f"needs.{native['needs']}.outputs.configured == 'true' || inputs.release == true"
probe = termux["jobs"][native["needs"]]
assert probe["environment"] == "release-signing"
for configured, release, runs in (("true", False, True), ("false", True, True), ("false", False, False),
("", False, False)):
needs = {native["needs"]: {"result": "success", "outputs": {"configured": configured}}}
assert gate(native["if"], {"release": release}, needs) is runs, (configured, release)