c8bbac5c6e
uv is internal to PM (never on a user's PATH), so env_for() left it out of
the exported PATH once the toolchain roots stopped listing python itself.
The fresh-install, pm-toolchain cold and worker-contract suites drive real
uv through shutil.which("uv") and failed with "requires real uv".
240 lines
9.7 KiB
Python
240 lines
9.7 KiB
Python
"""GitHub Actions file commands around the real, stdlib-only PM bootstrap."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import re
|
|
import sys
|
|
|
|
# The runner invokes this file before the checkout has been installed.
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
from pm.lock import Lockfile
|
|
from pm.paths import lockfile_path
|
|
from pm.registry import walk
|
|
from pm.store import current_target
|
|
|
|
_ROOTS = {"python": ["uv"], "node": ["npm"], "all": ["npm", "uv"]}
|
|
|
|
|
|
def parse_package_list(value: str) -> list[str]:
|
|
"""Comma-separated extra PM tools (e.g. `ffmpeg`) beyond the toolchain roots."""
|
|
if value == "":
|
|
return []
|
|
for name in value.split(","):
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name.strip()):
|
|
raise argparse.ArgumentTypeError(f"invalid pm tool name: {name!r}")
|
|
return sorted({name.strip() for name in value.split(",")})
|
|
|
|
|
|
def packages(toolchain: str, extra: list[str] | None = None) -> list[str]:
|
|
# walk() resolves the deps-first closure, so an extra tool's dependencies
|
|
# (ffmpeg -> none) come along without listing them.
|
|
return sorted(package.name for package in walk(_ROOTS[toolchain] + (extra or [])))
|
|
|
|
|
|
def file_commands(destination: str, values: dict) -> None:
|
|
"""All exports are single-line values, including JSON-encoded arrays."""
|
|
lines = []
|
|
for key, value in values.items():
|
|
text = str(value)
|
|
if any(character in key + text for character in "\r\n\0"):
|
|
raise ValueError(f"invalid GitHub file command: {key!r}")
|
|
lines.append(f"{key}={text}\n")
|
|
with open(os.environ[destination], "a", encoding="utf-8") as stream:
|
|
stream.writelines(lines)
|
|
|
|
|
|
def parse_extras(value: str) -> list[str] | None:
|
|
if value == "":
|
|
return None
|
|
message = "extras must be a JSON array of extra names"
|
|
try:
|
|
extras = json.loads(value)
|
|
except ValueError as exc:
|
|
raise argparse.ArgumentTypeError(message) from exc
|
|
if not isinstance(extras, list) or any(
|
|
not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name)
|
|
for name in extras
|
|
):
|
|
raise argparse.ArgumentTypeError(message)
|
|
return sorted(set(extras))
|
|
|
|
|
|
def prepare(args) -> None:
|
|
home = args.home.resolve()
|
|
lock = Lockfile(lockfile_path())
|
|
target = current_target()
|
|
names = packages(args.toolchain, args.packages)
|
|
values = {
|
|
"packages": json.dumps(names), "target": target, "arch": target.split("-")[1],
|
|
"store": str(home / "tools"),
|
|
}
|
|
for name in names:
|
|
version = lock.version(name)
|
|
if not version or not lock.artifacts(name, target):
|
|
raise ValueError(f"{name} has no pinned artifact for {target}")
|
|
values[f"{name}-version"] = version.partition("+")[0] if name == "python" else version
|
|
# Cache-key fragment for the tools cache: empty when no extras, so every
|
|
# existing key stays byte-identical.
|
|
values["extra-packages"] = "".join(f"-{name}" for name in (args.packages or []))
|
|
# The OS image belongs in the uv cache identity: built wheels can link
|
|
# against its system libraries. Unlike npm's cache, these are not just JS.
|
|
values["os-version"] = platform.platform()
|
|
values["bootstrap-python"] = sys.executable
|
|
file_commands("GITHUB_OUTPUT", values)
|
|
file_commands("GITHUB_ENV", {
|
|
"HERMES_HOME": home,
|
|
"HERMES_RUNTIME_DIR": home / "tools",
|
|
"PYTHONUTF8": "1",
|
|
})
|
|
|
|
|
|
def add_path(directories: list[str]) -> None:
|
|
# The runner prepends each line, so write PM's dependent-first PATH in
|
|
# reverse. npm must shadow the different npm bundled inside Node.
|
|
with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as stream:
|
|
for directory in reversed(list(dict.fromkeys(directories))):
|
|
if any(character in directory for character in "\r\n\0"):
|
|
raise ValueError("invalid PATH directory")
|
|
stream.write(directory + "\n")
|
|
|
|
|
|
def python3_alias(python: Path) -> None:
|
|
if os.name == "nt":
|
|
import shutil
|
|
|
|
alias = python.with_name("python3.exe")
|
|
if not alias.exists():
|
|
shutil.copy2(python, alias)
|
|
|
|
|
|
def archive_inputs(args) -> None:
|
|
from pm.paths import repo_root
|
|
from pm.store import Store
|
|
from scripts.ci.archive_inputs import Archive, pinned_inputs, stage_inputs
|
|
from scripts.releases import r2
|
|
|
|
pins = pinned_inputs(repo_root(), target=current_target(), packages=set(packages(args.toolchain, args.packages)))
|
|
stage_inputs(pins, archive=Archive(*r2.credentials()), store=Store(args.home.resolve() / "tools"))
|
|
|
|
|
|
def install(args) -> None:
|
|
import subprocess
|
|
|
|
from pm import build_requirements_environment
|
|
from pm.cli import _live_progress
|
|
from pm.install import ensure, env_for
|
|
from pm.lock import Facts
|
|
from pm.package import compose_env
|
|
from pm.packages import uv_cache_dir
|
|
from pm.paths import facts_path, store_root
|
|
from pm.registry import get_package
|
|
|
|
names = packages(args.toolchain, args.packages)
|
|
for name in _ROOTS[args.toolchain] + args.packages:
|
|
ensure(name, explicit=True, progress=_live_progress(name))
|
|
facts = Facts(facts_path())
|
|
target = current_target()
|
|
public_names = [name for name in names if name != "uv"]
|
|
binaries = {
|
|
name: get_package(name).binary(store_root() / facts.get(name)["entry"], target)
|
|
for name in public_names
|
|
}
|
|
environment = env_for(*public_names, base_env={})
|
|
path = environment["PATH"].split(os.pathsep)
|
|
environment = compose_env([environment])
|
|
exported = {}
|
|
outputs = {f"{name}-path": str(binaries[name]) for name in public_names}
|
|
if "python" in names:
|
|
# uv is PM-internal (never on a user's PATH), but the test suites this
|
|
# toolchain serves drive real uv through shutil.which("uv").
|
|
uv = get_package("uv").binary(store_root() / facts.get("uv")["entry"], target)
|
|
path.append(str(uv.parent))
|
|
outputs["uv-path"] = str(uv)
|
|
# Keep third-party CI tooling out of the verified interpreter store.
|
|
# PM prepares the empty command environment through its normal builder.
|
|
commands = args.home.resolve() / "python" / facts.get("python")["entry"]
|
|
if not (commands / "pyvenv.cfg").is_file():
|
|
build_requirements_environment([], out=commands, python=binaries["python"],
|
|
env=environment, explicit=True)
|
|
python = commands / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
|
python3_alias(python)
|
|
path.insert(0, str(python.parent))
|
|
outputs["python-path"] = str(python)
|
|
exported.update({
|
|
"HERMES_PYTHON": python,
|
|
})
|
|
outputs["uv-cache-path"] = str(uv_cache_dir())
|
|
if "npm" in names:
|
|
cache = subprocess.check_output(
|
|
[str(binaries["npm"]), "config", "get", "cache"],
|
|
env=environment, text=True, encoding="utf-8", timeout=60,
|
|
).strip()
|
|
outputs["npm-cache-path"] = cache
|
|
file_commands("GITHUB_ENV", exported)
|
|
file_commands("GITHUB_OUTPUT", outputs)
|
|
add_path(path)
|
|
print("PM toolchain ready: " + ", ".join(f"{name} {facts.get(name)['version']}" for name in names))
|
|
|
|
|
|
def dependencies(args) -> None:
|
|
if args.extras is None:
|
|
return
|
|
import tomllib
|
|
|
|
from pm.environments import selected_venv
|
|
from pm import build_environment, check_project_lock, sync_venv
|
|
from pm.paths import repo_root
|
|
|
|
project = repo_root()
|
|
metadata = tomllib.loads((project / "pyproject.toml").read_text(encoding="utf-8-sig"))
|
|
unknown = set(args.extras) - metadata["project"]["optional-dependencies"].keys()
|
|
if unknown:
|
|
raise ValueError(f"unknown project extras: {sorted(unknown)}")
|
|
# Frozen sync must not turn a stale project lock into a green job.
|
|
check_project_lock(project, explicit=True)
|
|
if "dev" in args.extras:
|
|
# Test-only groups must not enter PM facts or a shipped generation.
|
|
venv = args.home.resolve() / "test-environment"
|
|
build_environment(source=project, out=venv, extras=args.extras,
|
|
groups=["test"], no_install_project=True, explicit=True)
|
|
else:
|
|
sync_venv(args.extras, explicit=True, plugin_dirs=[])
|
|
venv = selected_venv(project)
|
|
bindir = venv / ("Scripts" if os.name == "nt" else "bin")
|
|
python = bindir / ("python.exe" if os.name == "nt" else "python")
|
|
python3_alias(python)
|
|
file_commands("GITHUB_ENV", {
|
|
"HERMES_PYTHON": python,
|
|
"VIRTUAL_ENV": venv,
|
|
"PYTHONPATH": project,
|
|
})
|
|
file_commands("GITHUB_OUTPUT", {"python-path": python, "venv": venv})
|
|
add_path([str(bindir)])
|
|
print(f"PM dependencies ready: {args.extras} in {venv}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
phases = {"prepare": prepare, "archive-inputs": archive_inputs, "install": install, "dependencies": dependencies}
|
|
parser.add_argument("phase", choices=list(phases))
|
|
parser.add_argument("--toolchain", choices=["python", "node", "all"], default="python")
|
|
parser.add_argument("--home", type=Path, required=True)
|
|
parser.add_argument("--extras", type=parse_extras, default="")
|
|
parser.add_argument("--packages", type=parse_package_list, default=[], help="extra PM tools beyond the toolchain roots (e.g. ffmpeg)")
|
|
args = parser.parse_args()
|
|
if args.toolchain == "node" and args.extras is not None:
|
|
parser.error("extras require the python or all toolchain")
|
|
os.environ["HERMES_HOME"] = str(args.home.resolve())
|
|
os.environ["HERMES_RUNTIME_DIR"] = str(args.home.resolve() / "tools")
|
|
phases[args.phase](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|