diff --git a/.github/workflows/pm-bundle.yml b/.github/workflows/pm-bundle.yml index d18dca0185..0bb6014639 100644 --- a/.github/workflows/pm-bundle.yml +++ b/.github/workflows/pm-bundle.yml @@ -39,7 +39,6 @@ on: - '.github/actions/desktop-build-cache/**' - 'scripts/windows-build-deps.ps1' - 'scripts/build/windows-deps.ps1' - - 'scripts/build/windows_deps.py' - '.github/actions/setup-windows-build-deps/**' - 'uv.lock' - 'pyproject.toml' diff --git a/pm/native_build.py b/pm/native_build.py new file mode 100644 index 0000000000..5bc51d45c0 --- /dev/null +++ b/pm/native_build.py @@ -0,0 +1,61 @@ +"""Native compiler environment for dependency builds from a source checkout. + +Windows ARM64 has no wheel for parts of the locked closure (cryptography), so +every sync there compiles from sdists and needs MSVC, Clang, Rust and static +OpenSSL. PM owns the sync, so PM prepares that environment. Otherwise only +callers that remembered to (source activation) could build, and +install.ps1, `hermes update` and repair failed in openssl-sys. +""" +from __future__ import annotations + +from collections.abc import Mapping +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +_PROVIDER = Path("scripts/build/windows-deps.ps1") + + +def prepare_windows_environment(*, source: Path, state: Path, env: Mapping[str, str]) -> dict[str, str]: + """The distribution adapter decides whether this target needs ARM64 tools.""" + shell = shutil.which("powershell", path=env.get("PATH")) or shutil.which("pwsh", path=env.get("PATH")) + if shell is None: + raise FileNotFoundError("PowerShell is required to prepare Windows ARM64 build dependencies") + state.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="environment-", dir=state) as scratch: + output = Path(scratch) / "environment.json" + subprocess.run( + [shell, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", + str(source / _PROVIDER), "-StateRoot", str(state), + "-EnvironmentFile", str(output)], + cwd=source, env=dict(env), check=True, stdin=subprocess.DEVNULL, + ) + prepared = json.loads(output.read_text(encoding="utf-8-sig")) + if not isinstance(prepared, dict) or any(not isinstance(k, str) or not isinstance(v, str) for k, v in prepared.items()): + raise ValueError("Windows build dependency provider returned an invalid environment") + return prepared + + +def source_build_environment(source: Path) -> dict[str, str] | None: + """The environment a dependency build of ``source`` needs on this host. + + None means the ambient environment suffices. Only a checkout carries the + provider; a payload's dependencies are prebuilt, so it needs no compiler. + The state root is the store's parent, the one source setup has always + used, so an existing vcpkg/OpenSSL build is reused rather than repeated. + """ + from pm.paths import store_root + from pm.store import current_target + + if current_target() != "win32-arm64" or not (source / _PROVIDER).is_file(): + return None + from pm.index_config import bridged_index_settings + + prepared = prepare_windows_environment(source=source, state=store_root().parent, env=os.environ) + # managed_environment translates pip's index knobs only for the ambient + # environment; this one replaces it, so mirrors must ride along. + prepared.update(bridged_index_settings(os.environ)) + return prepared diff --git a/pm/operations.py b/pm/operations.py index 75911df16a..3307759796 100644 --- a/pm/operations.py +++ b/pm/operations.py @@ -40,6 +40,7 @@ def build_environment( Sealed builds prune only the .pth files that refer to build-time state. """ from pm.environment import _fresh_build, managed_environment + from pm.native_build import source_build_environment source, out = Path(source).absolute(), Path(out).absolute() if not (source / "pyproject.toml").is_file(): @@ -49,6 +50,10 @@ def build_environment( if out.exists() or out.is_symlink(): raise FileExistsError(f"environment destination already exists: {out}") _require_install_allowed(explicit) + # A caller-supplied env is already the build environment (bundle staging + # prepares its own, shared with its Node builds). + if env is None: + env = source_build_environment(source) environment = managed_environment( out, python=Path(python) if python is not None else None, cache=Path(cache) if cache is not None else None, env=env, diff --git a/pm/packages.py b/pm/packages.py index 5f0932e1cc..d567c18b00 100644 --- a/pm/packages.py +++ b/pm/packages.py @@ -390,12 +390,14 @@ class Venv(StatePackage): from pm.environments import install_state_dir, runtime_facts_path from pm.environment import managed_environment from pm.lock import Facts + from pm.native_build import source_build_environment from pm.workspace import enabled_member_dirs, lock_and_sync project = self.project_root() generation = install_state_dir(project) / "environments" / uuid.uuid4().hex candidate = generation / "venv" - environment = managed_environment(candidate, explicit=explicit or repair, output=sys.stderr) + environment = managed_environment(candidate, env=source_build_environment(project), + explicit=explicit or repair, output=sys.stderr) members = [] if repair else (enabled_member_dirs() if plugin_dirs is None else plugin_dirs) try: generation.mkdir(parents=True) diff --git a/scripts/build/README.md b/scripts/build/README.md index 726a3a176a..189274ce92 100644 --- a/scripts/build/README.md +++ b/scripts/build/README.md @@ -288,8 +288,10 @@ save. The packaged `uv-cache/` is a copy, not the writable build cache. ### Windows ARM64 build prerequisites `scripts/windows-build-deps.ps1` owns Visual Studio ARM64, Clang, Rust, and -static OpenSSL preparation. Source setup calls its initializer. Native build -adapters use `scripts/build/windows-deps.ps1` through `windows_deps.py`, before +static OpenSSL preparation. PM calls it through `pm/native_build.py` before any +dependency build from a checkout, so every source install path gets it. Native +build adapters use the same `scripts/build/windows-deps.ps1` entrypoint through +`pm.native_build`, before isolating HOME or compiling Node/Python dependencies. CI uses the same script through `setup-windows-build-deps`, with an OpenSSL cache outside the product. The product compilers and assembler do not install these prerequisites. diff --git a/scripts/build/windows_deps.py b/scripts/build/windows_deps.py deleted file mode 100644 index bfa603e7b4..0000000000 --- a/scripts/build/windows_deps.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Prepare native Windows build inputs without changing the caller's process.""" -from __future__ import annotations - -from collections.abc import Mapping -import json -from pathlib import Path -import shutil -import subprocess -import tempfile - - -def prepare_windows_environment(*, source: Path, state: Path, env: Mapping[str, str]) -> dict[str, str]: - """The distribution adapter decides whether this target needs ARM64 tools.""" - shell = shutil.which("powershell", path=env.get("PATH")) or shutil.which("pwsh", path=env.get("PATH")) - if shell is None: - raise FileNotFoundError("PowerShell is required to prepare Windows ARM64 build dependencies") - state.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(prefix="environment-", dir=state) as scratch: - output = Path(scratch) / "environment.json" - subprocess.run( - [shell, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", - str(source / "scripts/build/windows-deps.ps1"), "-StateRoot", str(state), - "-EnvironmentFile", str(output)], - cwd=source, env=dict(env), check=True, - ) - prepared = json.loads(output.read_text(encoding="utf-8-sig")) - if not isinstance(prepared, dict) or any(not isinstance(k, str) or not isinstance(v, str) for k, v in prepared.items()): - raise ValueError("Windows build dependency provider returned an invalid environment") - return prepared diff --git a/scripts/bundles/desktop_toolchain.py b/scripts/bundles/desktop_toolchain.py index 5940ccf824..c96555a7e0 100644 --- a/scripts/bundles/desktop_toolchain.py +++ b/scripts/bundles/desktop_toolchain.py @@ -100,7 +100,7 @@ def prepare_tools(source: Path, work: Path, cache: Path, raise ValueError("desktop tools require the isolated preparation worker") prepared = dict(env) if pm.current_target() == "win32-arm64": - from scripts.build.windows_deps import prepare_windows_environment + from pm.native_build import prepare_windows_environment prepared = prepare_windows_environment(source=source, state=cache / "native/prerequisites", env=prepared) if pm.current_target().startswith("darwin"): diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 4c9d5c4251..20fd8f1711 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -106,7 +106,7 @@ def stage_native(args) -> int: cache = Path(getattr(args, "cache", None) or os.environ.get("UV_CACHE_DIR") or uv_cache_dir()).resolve() base_env = dict(os.environ) if current_target() == "win32-arm64": - from scripts.build.windows_deps import prepare_windows_environment + from pm.native_build import prepare_windows_environment base_env = prepare_windows_environment(source=root, state=out.parent / ".build-deps", env=base_env) # Rustup resolves its installed toolchain under HOME unless these are explicit. diff --git a/scripts/ci/desktop_build_cache.py b/scripts/ci/desktop_build_cache.py index 4427c96f4d..d5d44ac9e7 100644 --- a/scripts/ci/desktop_build_cache.py +++ b/scripts/ci/desktop_build_cache.py @@ -29,7 +29,7 @@ _INPUT_FILES = ( "scripts/bundles/desktop_inputs.py", "scripts/bundles/native.py", "scripts/bundles/native_prepared.py", "scripts/bundles/native_build.py", "scripts/build/node-deps.mjs", "scripts/build/icon_environment.py", - "scripts/build/windows_deps.py", "scripts/windows-build-deps.ps1", + "pm/native_build.py", "scripts/windows-build-deps.ps1", "apps/desktop/scripts/stage-native-deps.mjs", "apps/desktop/scripts/prepare-packaging-tools.mjs", "apps/desktop/scripts/build-command-screenshot-monitor.mjs", "apps/desktop/scripts/build-hud-modifier-monitor.mjs", diff --git a/setup-hermes.ps1 b/setup-hermes.ps1 index 5f25a9d62a..0da6d53835 100644 --- a/setup-hermes.ps1 +++ b/setup-hermes.ps1 @@ -70,14 +70,13 @@ if (Test-Path $uv) { } # --------------------------------------------------------------------------- -# Tools first, then the compiler environment, then the venv sync. -# ARM64 source wheels need the native compiler and OpenSSL development -# libraries, and that setup shells out to git. The git it finds must be the -# one pm just installed, not a host install whose PATH has two git.exe files -# (cmd\ and bin\). Activation runs setup in a child, so these build variables -# do not leak into its caller. +# PM installs the tools, then the venv. On ARM64 PM prepares the compiler and +# OpenSSL environment for that sync itself (pm/native_build.py), after its own +# git is published, so every install path builds the same way. +# Activation trusts the recorded tool digest. A direct setup re-checks it. # --------------------------------------------------------------------------- -Write-Host 'Installing python + tools via pm...' -ForegroundColor Cyan +Write-Host 'Installing python + tools + dependencies via pm (hash-verified via uv.lock)...' -ForegroundColor Cyan +Write-Host '(first run on a fresh checkout can take 1-5 minutes)' Push-Location $repo try { # PM can replace its uv entry only after the bootstrap uv has exited. @@ -85,29 +84,6 @@ try { if ($LASTEXITCODE -ne 0) { throw 'bootstrap Python installation failed' } $bootPy = (& $uv python find --managed-python $pyVersion) -join "`n" if ($LASTEXITCODE -ne 0 -or -not $bootPy) { throw 'bootstrap Python lookup failed' } - # The closure pm install would provision, minus the venv. A bare - # `pm install` also syncs the venv, and that sync must not run until the - # compiler environment below is on PATH. - & $bootPy.Trim() -m pm.cli install --tools-only $(if ($RuntimeOnly) { '--trust-recorded' }) - if ($LASTEXITCODE -ne 0) { throw 'pm tool install failed - see output above.' } -} finally { - Pop-Location -} -Write-Host 'Tools installed' -ForegroundColor Green - -if ($arch -eq 'arm64') { - . (Join-Path $repo 'scripts\windows-build-deps.ps1') - Initialize-HermesArm64BuildTools -StateRoot (Split-Path $store -Parent) -} - -# The venv sync. Tools are already on PATH inside that process (pm install -# publishes them before syncing); the compiler env set above is inherited. -# Activation trusts the recorded tool digest. A direct setup re-checks it. -# --------------------------------------------------------------------------- -Write-Host 'Installing dependencies via pm (hash-verified via uv.lock)...' -ForegroundColor Cyan -Write-Host '(first run on a fresh checkout can take 1-5 minutes)' -Push-Location $repo -try { & $bootPy.Trim() -m pm.cli install $(if ($RuntimeOnly) { '--trust-recorded' }) if ($LASTEXITCODE -ne 0) { throw 'pm install failed - see output above.' } } finally { diff --git a/tests/scripts/test_bundle_native.py b/tests/scripts/test_bundle_native.py index aec837f9a8..457152efe1 100644 --- a/tests/scripts/test_bundle_native.py +++ b/tests/scripts/test_bundle_native.py @@ -552,7 +552,7 @@ def test_staged_cache_ships_full_wheel_set_and_rebuilds_offline(tmp_path): def test_native_dispatch_isolates_process_state_on_real_child_failure(tmp_path, monkeypatch): # Compiler provisioning has its own native test; this probe must stop # at the invalid revision without installing tools on a developer host. - monkeypatch.setattr("scripts.build.windows_deps.prepare_windows_environment", lambda **kwargs: dict(kwargs["env"])) + monkeypatch.setattr("pm.native_build.prepare_windows_environment", lambda **kwargs: dict(kwargs["env"])) monkeypatch.setenv("HERMES_HOME", str(tmp_path / "user-home")) monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "user-tools")) before = dict(os.environ) @@ -587,7 +587,7 @@ def test_native_dispatch_child_environment(tmp_path, monkeypatch, cache_source, from pm.packages import uv_cache_dir monkeypatch.setattr(Path, "home", lambda: tmp_path / "host") - monkeypatch.setattr("scripts.build.windows_deps.prepare_windows_environment", lambda **kwargs: dict(kwargs["env"])) + monkeypatch.setattr("pm.native_build.prepare_windows_environment", lambda **kwargs: dict(kwargs["env"])) monkeypatch.setenv("HERMES_HOME", str(tmp_path / "user")) monkeypatch.delenv("UV_CACHE_DIR", raising=False) ambient = tmp_path / "ambient" diff --git a/tests/scripts/test_desktop_toolchain.py b/tests/scripts/test_desktop_toolchain.py index 273ee12dcb..3fb12ece19 100644 --- a/tests/scripts/test_desktop_toolchain.py +++ b/tests/scripts/test_desktop_toolchain.py @@ -173,7 +173,7 @@ def test_packaging_preserves_keychain_home_without_retargeting_build_state(tmp_p def test_prepare_tools_uses_pm_native_pins_and_separate_cache(tmp_path, monkeypatch): import pm from scripts.bundles import desktop_toolchain - from scripts.build import windows_deps + from pm import native_build source, work, cache = (tmp_path / name for name in ("source", "work", "cache")) env = desktop_toolchain.bootstrap_environment(source, work, cache, os.environ) @@ -195,7 +195,7 @@ def test_prepare_tools_uses_pm_native_pins_and_separate_cache(tmp_path, monkeypa monkeypatch.setattr(pm, "ensure", ensure) monkeypatch.setattr(pm, "installed_package", lambda name: SimpleNamespace(binary=bins[name])) monkeypatch.setattr(pm, "env_for", lambda *names, base_env: {**base_env, "PATH": "pm-tools"}) - monkeypatch.setattr(windows_deps, "prepare_windows_environment", native) + monkeypatch.setattr(native_build, "prepare_windows_environment", native) monkeypatch.setattr(desktop_toolchain, "native_cache_path", lambda cache, env: cache / "python/runtime/native-identity", raising=False) before = dict(os.environ) diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.md index 7557698223..6bc2a0bba9 100644 --- a/website/docs/reference/package-management.md +++ b/website/docs/reference/package-management.md @@ -217,13 +217,14 @@ interpreter or redirect an installed desktop app to this checkout. Use an ordinary terminal outside the packaged Hermes app. Leave any existing Python virtual environment first. On Windows, use native PowerShell with Git. -For ARM64, setup checks Visual Studio C++ tools, Clang, native Rust, and static -OpenSSL development libraries before PM runs. It reuses existing installations -and installs missing prerequisites. Missing Visual Studio components require -an Administrator PowerShell. OpenSSL uses vcpkg's `arm64-windows-static-md` -triplet. A damaged shared installation produces a repair error, not automatic -deletion. Compiler and OpenSSL environment variables apply only to the setup -process when you enter through `activate.ps1`. +On ARM64, PM prepares Visual Studio C++ tools, Clang, native Rust, and static +OpenSSL development libraries before every dependency build from a checkout: +setup, `activate.ps1`, `install.ps1`, `hermes update`, and repair alike. It +reuses existing installations and installs missing prerequisites. Missing +Visual Studio components require an Administrator PowerShell. OpenSSL uses +vcpkg's `arm64-windows-static-md` triplet. A damaged shared installation +produces a repair error, not automatic deletion. Compiler and OpenSSL +environment variables apply only to PM's dependency build, never to your shell. Other platforms still require the native compiler tools and libraries needed by dependencies without compatible wheels.