From 131ad86ad3905b673dd1de071acb4c92767e34be Mon Sep 17 00:00:00 2001 From: ethernet Date: Sun, 13 Sep 2026 12:50:10 -0400 Subject: [PATCH] fix(pm): preserve complete environments and bound install work --- hermes_cli/macos_signing.py | 49 +++++++ hermes_cli/macos_tcc_anchor.py | 77 +---------- pm/cli.py | 35 ++--- pm/ensure.py | 72 +++++++++- pm/environment.py | 10 ++ pm/packages.py | 57 +------- tests/hermes_cli/test_macos_tcc_anchor.py | 2 +- tests/pm/test_bootstrap_import_closure.py | 18 +++ tests/pm/test_environment_build.py | 112 +++++++++++++-- tests/pm/test_pm_core.py | 137 +++++++++++++------ website/docs/reference/package-management.md | 4 +- 11 files changed, 375 insertions(+), 198 deletions(-) create mode 100644 hermes_cli/macos_signing.py diff --git a/hermes_cli/macos_signing.py b/hermes_cli/macos_signing.py new file mode 100644 index 0000000000..3f7017f623 --- /dev/null +++ b/hermes_cli/macos_signing.py @@ -0,0 +1,49 @@ +"""Managed interpreter identity, available before PM or application imports.""" +from __future__ import annotations + +import logging +from pathlib import Path +import platform +import shutil +import subprocess + +logger = logging.getLogger(__name__) +_IDENTIFIER = "com.nousresearch.hermes.managed-python" + + +def sign_managed_python(python: Path) -> bool: + """Pin the designated requirement across downloaded Python generations. + + PBS's ad-hoc cdhash changes on upgrades. A stable identifier preserves + TCC identity without a Developer ID certificate. Signing remains best + effort so unavailable codesign cannot prevent a bootable interpreter. + """ + if platform.system() != "Darwin": + return False + codesign = shutil.which("codesign") + if not codesign: + logger.info("macOS codesign is unavailable; using the downloaded Python signature") + return False + try: + signed = subprocess.run( + [codesign, "--force", "--deep", "--sign", "-", "--timestamp=none", + "--identifier", _IDENTIFIER, "--requirements", + f'=designated => identifier "{_IDENTIFIER}"', str(python)], + check=False, capture_output=True, text=True, + ) + if signed.returncode != 0: + logger.warning("could not stably sign managed Python %s: %s", python, + (signed.stderr or signed.stdout or "codesign failed").strip()) + return False + verified = subprocess.run( + [codesign, "--verify", "--deep", "--strict", str(python)], + check=False, capture_output=True, text=True, + ) + if verified.returncode != 0: + logger.warning("macOS signature verification failed for managed Python %s: %s", python, + (verified.stderr or verified.stdout or "verification failed").strip()) + return False + return True + except Exception as exc: + logger.warning("could not sign managed Python %s: %s", python, exc) + return False diff --git a/hermes_cli/macos_tcc_anchor.py b/hermes_cli/macos_tcc_anchor.py index 1e4fec635e..0113315a17 100644 --- a/hermes_cli/macos_tcc_anchor.py +++ b/hermes_cli/macos_tcc_anchor.py @@ -33,84 +33,11 @@ _MARKER_NAME = ".tcc-anchor-source" _STORE_COMMON_MARKERS = ("cpython-", "-macos-") # Recognize both historical interpreter locations during upgrades. _STORE_ROOT_MARKERS = ("/uv/python/", "/.hermes-runtime/python/") -_MACOS_MANAGED_PYTHON_IDENTIFIER = "com.nousresearch.hermes.managed-python" _ALIAS_NAMES = ("python3", f"python3.{sys.version_info.minor}") _STORE_BIN_NAMES = (f"python3.{sys.version_info.minor}", "python3", "python") -def _macos_sign_managed_python(python: Path) -> bool: - """Give a newly downloaded managed Python a stable macOS code identity. - - python-build-standalone binaries are ad-hoc signed, which leaves macOS - TCC with a cdhash-only identity that changes whenever Hermes provisions a - new runtime generation. An identifier-pinned designated requirement - gives those generations a stable identity even when no Developer ID - certificate is available locally. - - Signing is deliberately best effort. An unavailable or incompatible ``codesign`` - must not prevent a bootable interpreter from being anchored. - """ - if platform.system() != "Darwin": - return False - - codesign = shutil.which("codesign") - if not codesign: - logger.info( - "macOS codesign is unavailable; using the downloaded Python signature" - ) - return False - - requirement = ( - "=designated => identifier " - f'"{_MACOS_MANAGED_PYTHON_IDENTIFIER}"' - ) - try: - signed = subprocess.run( - [ - codesign, - "--force", - "--deep", - "--sign", - "-", - "--timestamp=none", - "--identifier", - _MACOS_MANAGED_PYTHON_IDENTIFIER, - "--requirements", - requirement, - str(python), - ], - check=False, - capture_output=True, - text=True, - ) - if signed.returncode != 0: - logger.warning( - "could not stably sign managed Python %s: %s", - python, - (signed.stderr or signed.stdout or "codesign failed").strip(), - ) - return False - - verified = subprocess.run( - [codesign, "--verify", "--deep", "--strict", str(python)], - check=False, - capture_output=True, - text=True, - ) - if verified.returncode != 0: - logger.warning( - "macOS signature verification failed for managed Python %s: %s", - python, - (verified.stderr or verified.stdout or "verification failed").strip(), - ) - return False - return True - except Exception as exc: - logger.warning("could not sign managed Python %s: %s", python, exc) - return False - - class _BootGateFailed(Exception): """Staged copy refused to boot; the live venv must stay untouched.""" @@ -354,6 +281,8 @@ def _passes_boot_gate(staged: Path, venv_dir: Path) -> bool: def _install_anchor(venv_dir: Path, source_file: Path) -> None: """Replace ``bin/python`` with a signed copy, gated on a real boot.""" + from hermes_cli.macos_signing import sign_managed_python + venv_py = venv_python_path(venv_dir) venv_bin = venv_py.parent venv_bin.mkdir(parents=True, exist_ok=True) @@ -363,7 +292,7 @@ def _install_anchor(venv_dir: Path, source_file: Path) -> None: tmp_path = _stage_copy(venv_bin, ".python-tcc-", source_file) try: try: - _macos_sign_managed_python(tmp_path) + sign_managed_python(tmp_path) except Exception: # pragma: no cover - never block the anchor logger.debug("anchor copy signing skipped", exc_info=True) if not _passes_boot_gate(tmp_path, venv_dir): diff --git a/pm/cli.py b/pm/cli.py index 8c52ebb65e..cd666e8ff5 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -84,23 +84,26 @@ def _live_progress(name: str): def _install_names(names: list[str], target: str | None = None) -> int: - failed = 0 - for name in names: - try: - if target is not None: - # Cross-target staging: publish the entry, touch no facts. - entry = stage_only(name, target) - print(f"✓ {name} (staged for {target}: {entry.name})") - else: - ensure(name, explicit=True, progress=_live_progress(name)) - if name == "python": - from hermes_cli.venv_sync import publish_launchers + from pm.ensure import _install_operation - publish_launchers(repo_root()) - print(f"✓ {name}", flush=True) - except InstallError as e: - print(f"✗ {e}", flush=True) - failed += 1 + failed = 0 + with _install_operation() as operation: + for name in names: + try: + if target is not None: + # Cross-target staging: publish the entry, touch no facts. + entry = stage_only(name, target) + print(f"✓ {name} (staged for {target}: {entry.name})") + else: + ensure(name, explicit=True, progress=_live_progress(name), _operation=operation) + if name == "python": + from hermes_cli.venv_sync import publish_launchers + + publish_launchers(repo_root()) + print(f"✓ {name}", flush=True) + except InstallError as e: + print(f"✗ {e}", flush=True) + failed += 1 return failed diff --git a/pm/ensure.py b/pm/ensure.py index 303f55a3b8..67666a4854 100644 --- a/pm/ensure.py +++ b/pm/ensure.py @@ -7,7 +7,7 @@ import json import logging import shutil import threading -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager, nullcontext from dataclasses import dataclass from pathlib import Path from typing import Optional @@ -271,6 +271,7 @@ def _install( download_progress: ProgressFn | None = None, *, copy_from: tuple[Facts, Store] | None = None, + _lock_held: bool = False, ) -> Path: """Realize one pin. Host installs commit facts; cross-target stages carry a marker.""" version = lockfile.version(package.name) @@ -290,7 +291,7 @@ def _install( artifacts = lockfile.artifacts(package.name, target) pin = json.dumps({"target": target, "sha256": [a["sha256"] for a in artifacts]}) - with store.install_lock(): + with nullcontext() if _lock_held else store.install_lock(): if pause_event is not None and pause_event.is_set(): raise DownloadPaused("install paused") if facts is not None: @@ -387,6 +388,35 @@ def stage_only( progress=progress, pause_event=pause_event, download_progress=download_progress) +class _InstallOperation: + """Validity lasts only while this operation holds the publication lock.""" + + def __init__(self) -> None: + self.stack = ExitStack() + self.store: Store | None = None + self.checked: set[tuple[str, str | None, str, str]] = set() + + def lock(self) -> Store: + if self.store is None: + self.store = Store(paths.writable_store_root()) + self.stack.enter_context(self.store.install_lock()) + return self.store + + def close(self) -> None: + self.checked.clear() + self.store = None + self.stack.close() + + +@contextmanager +def _install_operation(): + operation = _InstallOperation() + try: + yield operation + finally: + operation.close() + + def ensure( name: str, *, @@ -395,6 +425,7 @@ def ensure( progress=None, pause_event: threading.Event | None = None, download_progress: ProgressFn | None = None, + _operation: _InstallOperation | None = None, ) -> Runner: """``explicit`` marks a deliberate install command (`hermes pm install`, `hermes pm bundle`) — those ARE the remedy the lazy-install @@ -404,21 +435,45 @@ def ensure( install to a UI, including ordered multi-archive labels. """ if isinstance(get_package(name), StatePackage): + if _operation is not None: + # Python construction can provision tools itself. Drop both the + # lock and its validity before entering that independent operation. + _operation.close() sync_venv(explicit=explicit) return Runner(name, compose_env([], base=base_env)) + if explicit and _operation is None: + with _install_operation() as operation: + return ensure(name, base_env=base_env, explicit=True, progress=progress, + pause_event=pause_event, download_progress=download_progress, + _operation=operation) lockfile = _lockfile() target = current_target() chain = walk([name]) - missing = [p for p in chain if _installed_location(p, lockfile, target, verify=explicit) is None] + checked = _operation.checked if _operation is not None else set() + if _operation is not None: + _operation.lock() + missing = [] + for package in chain: + identity = (package.name, lockfile.version(package.name), target, + json.dumps(_identity(lockfile, package.name, target), sort_keys=True)) + if identity in checked: + continue + if _installed_location(package, lockfile, target, verify=explicit) is None: + missing.append(package) + else: + checked.add(identity) if missing and not explicit and not lazy_installs_allowed(): raise _refuse_lazy(name, ", ".join(p.name for p in missing)) if missing: - store = Store(paths.writable_store_root()) + store = _operation.lock() if _operation is not None else Store(paths.writable_store_root()) facts = _facts() if store.root == paths.store_root() else Facts(store.root / "facts.json") for package in missing: + # Publication may change entries; do not carry observations across it. + checked.clear() _install(package, lockfile, facts, store, target, progress=progress, - pause_event=pause_event, download_progress=download_progress) + pause_event=pause_event, download_progress=download_progress, + _lock_held=_operation is not None) return Runner(name, env_for(name, base_env=base_env)) @@ -521,7 +576,8 @@ def sync_venv(extras: Optional[list[str]] = None, *, explicit: bool = False, plu if unsupported: raise InstallError("venv", f"extras {unsupported} are not supported by this Python/platform", "choose a supported provider; no dependency environment was changed") - frozen = read_features() + shipped = read_features() + frozen = shipped # Without a frozen declaration, explicit source setup needs no policy # read: the config loader initializes/chmods unrelated user state. if frozen is not None and not repair and lazy_installs_allowed(): @@ -568,7 +624,9 @@ def sync_venv(extras: Optional[list[str]] = None, *, explicit: bool = False, plu stamp = fact.get("stamp") or package.expected_stamp(enabled, plugin_dirs=[]) inputs = {"repair": True} else: - enabled = sorted(set(fact.get("extras", [])) | set(extras or [])) + # The first writable generation replaces, rather than layers on, + # the payload. Retain its extras until a recorded selection owns them. + enabled = sorted(set(fact.get("extras", shipped or [])) | set(extras or [])) stamp = package.expected_stamp(enabled, **inputs) if not repair and not explicit and not lazy_installs_allowed() and not _runtime_state_matches(fact, stamp): raise _refuse_lazy("venv", str(extras) if extras else "venv out of sync") diff --git a/pm/environment.py b/pm/environment.py index 8608038369..ac15354b5e 100644 --- a/pm/environment.py +++ b/pm/environment.py @@ -51,12 +51,15 @@ def prune_site_pth(venv_dir: Path) -> None: def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int, output: TextIO) -> subprocess.CompletedProcess: """Keep CI progress live, a bounded diagnostic tail, and a wall-clock timeout.""" + from pm.workspace import _RESOLVER_MARKERS + deadline = time.monotonic() + timeout proc = subprocess.Popen(command, cwd=str(cwd), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", bufsize=0) pipe = proc.stdout assert isinstance(pipe, io.TextIOWrapper) # Popen was given stdout=PIPE and text=True. tail = "" + conflict = "" try: # A descendant can keep stdout open after proc exits. Nonblocking reads # bound that drain without leaving a thread stuck in readline()/close(). @@ -76,6 +79,11 @@ def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], continue text = decoder.decode(data, final=not data) if text: + # Preserve an observed resolver marker even after verbose output + # evicts it. Scan across read boundaries, never retain the full log. + if not conflict: + lowered = (tail + text).lower() + conflict = next((marker for marker in _RESOLVER_MARKERS if marker in lowered), "") tail = (tail + text)[-2000:] output.write(text) output.flush() @@ -92,6 +100,8 @@ def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], raise finally: pipe.close() + if conflict and conflict not in tail.lower(): + tail = conflict + "\n" + tail[-(2000 - len(conflict) - 1):] return subprocess.CompletedProcess(command, code, "", tail) diff --git a/pm/packages.py b/pm/packages.py index 5edb348cf9..e488d98607 100644 --- a/pm/packages.py +++ b/pm/packages.py @@ -7,6 +7,7 @@ import os import platform import shutil import subprocess +import sys from pathlib import Path from typing import Optional @@ -180,54 +181,6 @@ class Uv(_BionicDebArm, BinaryPackage, DebPackage): return github_release_tags("astral-sh/uv") -_MACOS_MANAGED_PYTHON_IDENTIFIER = "com.nousresearch.hermes.managed-python" - - -def _macos_sign_managed_python(python: Path) -> bool: - """Give a downloaded Python a stable macOS code identity.""" - if platform.system() != "Darwin": - return False - - codesign = shutil.which("codesign") - if not codesign: - return False - - requirement = ( - "=designated => identifier " - f'"{_MACOS_MANAGED_PYTHON_IDENTIFIER}"' - ) - try: - signed = subprocess.run( - [ - codesign, - "--force", - "--deep", - "--sign", - "-", - "--timestamp=none", - "--identifier", - _MACOS_MANAGED_PYTHON_IDENTIFIER, - "--requirements", - requirement, - str(python), - ], - check=False, - capture_output=True, - text=True, - ) - if signed.returncode != 0: - return False - verified = subprocess.run( - [codesign, "--verify", "--deep", "--strict", str(python)], - check=False, - capture_output=True, - text=True, - ) - return verified.returncode == 0 - except Exception: - return False - - @register class Python(_BionicDebArm, BinaryPackage, DebPackage): """The pinned interpreter for launchers and every PM-managed uv command. @@ -258,8 +211,10 @@ class Python(_BionicDebArm, BinaryPackage, DebPackage): def stage(self, store: Store, staged: Path, version: str, target: str) -> None: super().stage(store, staged, version, target) binary = self.binary(staged, target) - if binary is not None: - _macos_sign_managed_python(binary) + if binary is not None and sys.platform == "darwin": + from hermes_cli.macos_signing import sign_managed_python + + sign_managed_python(binary) # python-build-standalone ships the x64 VC runtime (vcruntime140_1.dll) # beside ARM64 Windows Python; it cannot load on ARM64 and would fail # the arch guard. Drop it HERE, before publish: the tree digest is @@ -408,7 +363,7 @@ class Venv(StatePackage): project = self.project_root() generation = install_state_dir(project) / "environments" / uuid.uuid4().hex candidate = generation / "venv" - environment = managed_environment(candidate, explicit=explicit or repair) + environment = managed_environment(candidate, explicit=explicit or repair, output=sys.stderr) members = [] if repair else (enabled_member_dirs() if plugin_dirs is None else plugin_dirs) try: generation.mkdir(parents=True) diff --git a/tests/hermes_cli/test_macos_tcc_anchor.py b/tests/hermes_cli/test_macos_tcc_anchor.py index 21c93e9e5f..5a422d4a1e 100644 --- a/tests/hermes_cli/test_macos_tcc_anchor.py +++ b/tests/hermes_cli/test_macos_tcc_anchor.py @@ -144,7 +144,7 @@ class TestEnsureTccAnchor: signed = [] monkeypatch.setattr( - tcc, "_macos_sign_managed_python", lambda p: signed.append(Path(p)) or True + "hermes_cli.macos_signing.sign_managed_python", lambda p: signed.append(Path(p)) or True ) store_bin = _build_store(tmp_path) root = _build_checkout(tmp_path, store_bin=store_bin) diff --git a/tests/pm/test_bootstrap_import_closure.py b/tests/pm/test_bootstrap_import_closure.py index 7cf8636b2a..fac2976257 100644 --- a/tests/pm/test_bootstrap_import_closure.py +++ b/tests/pm/test_bootstrap_import_closure.py @@ -34,3 +34,21 @@ print(root) capture_output=True, text=True, timeout=15) assert result.returncode == 0, result.stdout + result.stderr assert Path(result.stdout.strip()) == store + + +def test_managed_python_signing_import_needs_no_pm_or_application(tmp_path): + repo = Path(__file__).resolve().parents[2] + script = """ +import sys +class NoApplication: + def find_spec(self, fullname, path=None, target=None): + if fullname == 'pm' or fullname == 'utils' or fullname.startswith(('pm.', 'agent.')): + raise AssertionError('signing imported ' + fullname) +sys.meta_path.insert(0, NoApplication()) +from hermes_cli.macos_signing import sign_managed_python +assert callable(sign_managed_python) +""" + result = subprocess.run([sys.executable, "-S", "-c", script], cwd=repo, + env=dict(os.environ, HERMES_HOME=str(tmp_path / "home")), + capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr diff --git a/tests/pm/test_environment_build.py b/tests/pm/test_environment_build.py index e5a19a1f94..aa3cac30af 100644 --- a/tests/pm/test_environment_build.py +++ b/tests/pm/test_environment_build.py @@ -195,6 +195,62 @@ def test_group_only_build_excludes_application_dependencies(locked_project, tmp_ cwd=tmp_path, env=env) == "1.0" +@pytest.mark.parametrize("lazy", [False, True]) +def test_first_bundle_extension_preserves_shipped_extras(locked_project, build_worker, tmp_path, monkeypatch, lazy): + import pm + from hermes_cli.runtime_paths import selected_venv, runtime_facts_path + from pm import paths + from pm.features import write_features + from pm.lock import Facts + + source, _, env = locked_project + manifest = source / "pyproject.toml" + manifest.write_text(manifest.read_text().replace('[tool.uv.workspace]\nmembers=["member"]\n', ""), encoding="utf-8") + monkeypatch.setattr(paths, "repo_root", lambda: source) + pm.lock_project(source, offline=True, explicit=True) + base = tmp_path / "shipped" + pm.build_environment(source=source, out=base, extras=["chosen"], env=env, + cache=tmp_path / "cache", offline=True, explicit=True) + (tmp_path / "manifest.json").write_text(json.dumps({"repo": source.name, "venv": base.name}), encoding="utf-8") + write_features(["chosen"], tmp_path) + home = Path(os.environ["HERMES_HOME"]) + home.mkdir(exist_ok=True) + (home / "config.yaml").write_text(f"security:\n allow_lazy_installs: {str(lazy).lower()}\n", encoding="utf-8") + assert selected_venv(source) == base + assert Facts(runtime_facts_path(source)).get("venv") is None + python_relative = "Scripts/python.exe" if os.name == "nt" else "bin/python" + assert _run([str(base / python_relative), "-I", "-c", + "import chosen_dep; print(chosen_dep.__version__)"], cwd=tmp_path, env=env) == "1.0" + locked = (source / "uv.lock").read_bytes() + + # Admission adds only a plugin, not a list of the bundle's optional extras. + pm.sync_venv(explicit=True, plugin_dirs=[source / "member"]) + first = selected_venv(source) + assert first != base + executable = first / python_relative + assert _run([str(executable), "-I", "-c", + "import base_dep, chosen_dep, member_dep; print(chosen_dep.__version__)"], + cwd=tmp_path, env=env) == "1.0" + first_fact = Facts(runtime_facts_path(source)).get("venv") + assert first_fact is not None and first_fact["extras"] == ["chosen"] + + # Once recorded, the selection owns the baseline, even if inventory changes. + write_features(["other"], tmp_path) + _wheel(tmp_path / "wheels", "member_dep", "1.1") + member = source / "member" / "pyproject.toml" + member.write_text(member.read_text().replace("member-dep==1.0", "member-dep==1.1"), encoding="utf-8") + pm.sync_venv(explicit=True, plugin_dirs=[source / "member"]) + second = selected_venv(source) + assert second != first + second_fact = Facts(runtime_facts_path(source)).get("venv") + assert second_fact is not None and second_fact["extras"] == ["chosen"] + assert _run([str(second / executable.relative_to(first)), "-I", "-c", + "import chosen_dep, member_dep, importlib.util; " + "assert importlib.util.find_spec('other_dep') is None; print(member_dep.__version__)"], + cwd=tmp_path, env=env) == "1.1" + assert (source / "uv.lock").read_bytes() == locked + + def test_worker_sync_reuses_unions_and_reports_real_lock_drift(locked_project, build_worker, tmp_path, monkeypatch): import pm from hermes_cli.runtime_paths import selected_venv, runtime_facts_path @@ -244,8 +300,8 @@ def test_worker_sync_reuses_unions_and_reports_real_lock_drift(locked_project, b assert not {"base_dep", "chosen_dep", "other_dep"} & sys.modules.keys() -@pytest.mark.parametrize("operation", ["sync", "requirements"]) -def test_build_backend_output_is_streamed_before_build_finishes(installable_project, tmp_path, operation): +@pytest.mark.parametrize("operation", ["sync", "requirements", "application"]) +def test_build_backend_output_is_streamed_before_build_finishes(installable_project, tmp_path, monkeypatch, operation): import io from pm.environment import PythonEnvironment @@ -286,17 +342,57 @@ build_editable = build_wheel cache=tmp_path / "build-cache", offline=True, output=output, env=dict(env, UV_NO_INDEX="1", UV_FIND_LINKS=str(tmp_path / "wheels")), ) - environment.create() - if operation == "sync": - environment.sync(source, timeout=30) + if operation == "application": + from pm.packages import Venv + + monkeypatch.setattr("pm._uv._toolchain", lambda **kwargs: (uv, Path(sys.executable))) + monkeypatch.setattr(sys, "stderr", output) + result = Venv(source).apply([], plugin_dirs=[], explicit=True) + executable = result["environment"] / ("Scripts/python.exe" if os.name == "nt" else "bin/python") else: - environment.install_requirements([source.as_uri()]) - environment.check() + environment.create() + if operation == "sync": + environment.sync(source, timeout=30) + else: + environment.install_requirements([source.as_uri()]) + environment.check() + executable = environment.executable assert release.is_file(), "both backend streams must arrive during the build" - assert _run([str(environment.executable), "-I", "-c", "import root_app; print(root_app.VALUE)"], + assert _run([str(executable), "-I", "-c", "import root_app; print(root_app.VALUE)"], cwd=tmp_path, env=env) == "installed from the explicit source" +@pytest.mark.parametrize("diagnostic", ["No solution found", "Connection timed out", "Failed to build wheel"]) +def test_streaming_bounds_memory_without_losing_failure_class(tmp_path, diagnostic): + import tracemalloc + from pm.environment import PythonEnvironment + from pm.workspace import classify_uv_failure + + # Separate writes split the resolver marker across pipe reads; subsequent + # verbose output evicts it from the retained tail without changing its class. + script = ( + f"import os, time; os.write(2, {diagnostic[:4].encode()!r}); time.sleep(.15); " + f"os.write(2, {diagnostic[4:].encode()!r}); " + "[os.write(2, b'x' * 65536) for _ in range(128)]; " + "os.write(2, b'final diagnostic'); raise SystemExit(1)" + ) + with open(os.devnull, "w") as output: + environment = PythonEnvironment(uv=Path(sys.executable), python=Path(sys.executable), + destination=tmp_path / "venv", cache=tmp_path / "cache", env=dict(os.environ), output=output) + tracemalloc.start() + try: + result = environment._run(["-c", script], cwd=tmp_path, timeout=30) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert result.returncode == 1 + assert len(result.stderr) <= 2000 + assert peak < 2 * 1024 * 1024, f"stream capture retained {peak} bytes" + actual = classify_uv_failure("sync", result.returncode, result.stderr) + assert type(actual) is type(classify_uv_failure("sync", 1, diagnostic)) + assert "final diagnostic" in str(actual) + + def test_child_output_is_live_and_keeps_explicit_index_credentials(tmp_path): import io from pm.environment import PythonEnvironment diff --git a/tests/pm/test_pm_core.py b/tests/pm/test_pm_core.py index 2ad21cdb2c..a24d2e8101 100644 --- a/tests/pm/test_pm_core.py +++ b/tests/pm/test_pm_core.py @@ -273,6 +273,80 @@ def test_deps_compose_dependents_win(pm_env): assert path.index("toptool-1.0") < path.index("deptool-1.0") +def test_warm_install_verifies_shared_dependencies_once_under_lock(pm_env, monkeypatch): + import importlib + import os + from collections import Counter + from hermes_cli.runtime_state import _lock + from pm.cli import _install_names + + ensure = importlib.import_module("pm.ensure") + lockfile_path, runtime, docroot, _ = pm_env + for name in ("deptool", "toptool"): + _, digest = make_tar(docroot, f"{name}-1.0.tar.gz", {"bin/faketool": name}) + _pin(lockfile_path, name, "1.0", digest) + assert _install_names(["deptool", "toptool"]) == 0 + checked = Counter() + locked = [] + original = ensure._entry_verified + + def verify(package, fact, store, target): + fd = os.open(store.root / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600) + try: + locked.append(not _lock(fd, wait=False)) + finally: + os.close(fd) + checked[package.name] += 1 + return original(package, fact, store, target) + + monkeypatch.setattr(ensure, "_entry_verified", verify) + assert _install_names(["deptool", "toptool"]) == 0 + assert checked == {"deptool": 1, "toptool": 1} + assert all(locked), "validation must share the publication lock" + + # The next operation must not reuse validity across a writer's mutation. + fact = Facts(runtime / "facts.json").get("deptool") + assert fact is not None + binary = runtime / fact["entry"] / "bin/faketool" + with Store(runtime).install_lock(): + binary.write_text("corrupt", encoding="utf-8") + assert _install_names(["toptool"]) == 0 + assert binary.read_text(encoding="utf-8") == "deptool" + + +def test_install_forgets_verification_when_state_operation_releases_lock(pm_env, monkeypatch): + import importlib + import os + from hermes_cli.runtime_state import _lock + from pm.cli import _install_names + from pm.packages import Venv + + ensure = importlib.import_module("pm.ensure") + lockfile_path, runtime, docroot, _ = pm_env + for name in ("deptool", "toptool"): + _, digest = make_tar(docroot, f"{name}-1.0.tar.gz", {"bin/faketool": name}) + _pin(lockfile_path, name, "1.0", digest) + assert _install_names(["toptool"]) == 0 + fact = Facts(runtime / "facts.json").get("deptool") + assert fact is not None + binary = runtime / fact["entry"] / "bin/faketool" + + def sync(**kwargs): + # State operations provision their own tools. They must be able to + # acquire the lock independently, and invalidate prior observations. + fd = os.open(runtime / ".install.lock", os.O_CREAT | os.O_RDWR, 0o600) + try: + assert _lock(fd, wait=False), "tool lock leaked into the state operation" + binary.write_text("corrupt", encoding="utf-8") + finally: + os.close(fd) + + monkeypatch.setattr(ensure, "sync_venv", sync) + monkeypatch.setitem(registry._packages, "venv", Venv()) + assert _install_names(["deptool", "venv", "toptool"]) == 0 + assert binary.read_text(encoding="utf-8") == "deptool" + + def test_version_bump_selects_the_new_tool(pm_env): from pm.ensure import ensure @@ -472,54 +546,39 @@ def test_python_package_url_carries_release_tag(): pass -def test_python_package_stably_signs_macos_runtime(monkeypatch, tmp_path): - import pm.packages as packages +@pytest.mark.platforms("macos") +def test_python_package_stably_signs_macos_runtime(tmp_path): + import shutil + import subprocess + import sys from pm.registry import get_package python = get_package("python") - binary = tmp_path / "bin" / "python3" - binary.parent.mkdir() - binary.touch() - calls = [] - - monkeypatch.setattr(packages.platform, "system", lambda: "Darwin") - monkeypatch.setattr(packages.shutil, "which", lambda name: "/usr/bin/codesign") - monkeypatch.setattr( - packages.subprocess, - "run", - lambda cmd, **kwargs: calls.append((cmd, kwargs)) or type("Result", (), {"returncode": 0})(), - ) - monkeypatch.setattr(python, "binary", lambda entry, target: binary) - - python.stage(Store(tmp_path / "store"), tmp_path, "3.11", "darwin-arm64") - - assert calls[0][0] == [ - "/usr/bin/codesign", - "--force", - "--deep", - "--sign", - "-", - "--timestamp=none", - "--identifier", - "com.nousresearch.hermes.managed-python", - "--requirements", - '=designated => identifier "com.nousresearch.hermes.managed-python"', - str(binary), - ] - assert calls[1][0] == ["/usr/bin/codesign", "--verify", "--deep", "--strict", str(binary)] + staged = tmp_path / "staged" + binary = staged / "python" / "bin" / "python3" + binary.parent.mkdir(parents=True) + (staged / "python" / "lib").mkdir() + shutil.copy2(Path(sys._base_executable).resolve(), binary) + python.stage(Store(tmp_path / "store"), staged, "fixture", current_target()) + binary = python.binary(staged, current_target()) + subprocess.run(["codesign", "--verify", "--deep", "--strict", str(binary)], + check=True, capture_output=True, timeout=30) + identity = subprocess.run(["codesign", "-d", "-r-", str(binary)], + check=True, capture_output=True, text=True, timeout=30) + assert 'designated => identifier "com.nousresearch.hermes.managed-python"' in identity.stdout + identity.stderr +@pytest.mark.platforms("not macos") def test_python_package_does_not_sign_non_macos_runtime(monkeypatch, tmp_path): - import pm.packages as packages + import hermes_cli.macos_signing as signing - monkeypatch.setattr(packages.platform, "system", lambda: "Linux") monkeypatch.setattr( - packages.subprocess, + signing.subprocess, "run", lambda *args, **kwargs: pytest.fail("codesign must not run outside macOS"), ) - assert packages._macos_sign_managed_python(tmp_path / "python") is False + assert signing.sign_managed_python(tmp_path / "python") is False def test_machine_matches_binary_pe_headers(tmp_path): @@ -642,7 +701,6 @@ def test_arch_guard_allows_emulated_x64_on_win32_arm64(monkeypatch, tmp_path): def test_python_stage_drops_unloadable_x64_vc_runtime_on_arm64(monkeypatch, tmp_path): - import pm.packages as packages from pm.registry import get_package staged = tmp_path / "staged" @@ -650,7 +708,7 @@ def test_python_stage_drops_unloadable_x64_vc_runtime_on_arm64(monkeypatch, tmp_ (staged / "vcruntime140_1.dll").write_bytes(b"x64") (staged / "vcruntime140.dll").write_bytes(b"arm64") - monkeypatch.setattr(packages, "_macos_sign_managed_python", lambda p: False) + monkeypatch.setattr("hermes_cli.macos_signing.sign_managed_python", lambda p: False) get_package("python").stage(None, staged, "3.14.7", "win32-arm64") assert not (staged / "vcruntime140_1.dll").exists() @@ -658,14 +716,13 @@ def test_python_stage_drops_unloadable_x64_vc_runtime_on_arm64(monkeypatch, tmp_ def test_python_stage_keeps_vc_runtimes_on_other_targets(monkeypatch, tmp_path): - import pm.packages as packages from pm.registry import get_package staged = tmp_path / "staged" staged.mkdir() (staged / "vcruntime140_1.dll").write_bytes(b"x64") - monkeypatch.setattr(packages, "_macos_sign_managed_python", lambda p: False) + monkeypatch.setattr("hermes_cli.macos_signing.sign_managed_python", lambda p: False) get_package("python").stage(None, staged, "3.14.7", "win32-x64") assert (staged / "vcruntime140_1.dll").is_file() diff --git a/website/docs/reference/package-management.md b/website/docs/reference/package-management.md index c075fa6ef0..df886fba53 100644 --- a/website/docs/reference/package-management.md +++ b/website/docs/reference/package-management.md @@ -71,7 +71,9 @@ from that payload, rather than copying a source checkout on first launch. The bundle builder checks its files and writes the launch paths into the desktop build stamp. Electron uses those paths without probing or repairing the payload. Additional pinned tools can use the writable tool store. Python additions use -a complete writable environment outside the signed package. +a complete writable environment outside the signed package. Its first generation +retains the shipped extras along with the new requirements. Later generations +use the recorded extra selection as their baseline. Termux uses a separate bionic build and a sealed APT package. Docker bakes its runtime into the image and disables on-demand dependency installation. Nix