diff --git a/pm/cli.py b/pm/cli.py index 350cc1ed45..51ea0c4677 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -177,6 +177,12 @@ def cmd_install(args) -> int: print("✗ --extra syncs this install's venv and cannot combine with --target") return 1 names = args.names if args.names or extras else source_install_packages(_lockfile().names()) + # Only the whole default closure verifies everything an activated shell + # composes from, so only it advances the prologue's input stamps. + full_closure = not (args.names or tools_only or cross_target) + from pm.environments import activation_input_mtimes + + input_mtimes = activation_input_mtimes(repo_root()) if full_closure else {} # Tools before the venv. A bare `pm install` used to install tools and # sync the venv in one breath, so a native build (Windows ARM64 source # wheels) resolved compilers and git from the host PATH. Publish every @@ -208,6 +214,10 @@ def cmd_install(args) -> int: except InstallError as e: print(f"✗ {e}") failed += 1 + if full_closure and not failed: + from pm.environments import activation_inputs_dir, record_activation_inputs + + record_activation_inputs(activation_inputs_dir(repo_root()), input_mtimes) return 1 if failed else 0 diff --git a/pm/environments.py b/pm/environments.py index 6945dc4bf7..214bfede39 100644 --- a/pm/environments.py +++ b/pm/environments.py @@ -39,6 +39,41 @@ def runtime_facts_path(project_root: Path) -> Path: return install_state_dir(project_root) / "facts.json" +# The files that decide the dependency set. `scripts/_hermes-python` re-activates +# when any of them differs in mtime from its stamp under activation_inputs_dir. +ACTIVATION_INPUTS = ("uv.lock", "pyproject.toml", "pm/lock.json") + + +def activation_inputs_dir(project_root: Path) -> Path: + """Beside facts.json, so the prologue finds it from ``$__HERMES_ACTIVATED``.""" + return install_state_dir(project_root) / "inputs" + + +def activation_input_mtimes(project_root: Path) -> dict[str, int]: + """Snapshot before installing, so an input edited mid-install records its + pre-install mtime and the next run re-activates.""" + root = Path(project_root) + return {name: (root / name).stat().st_mtime_ns for name in ACTIVATION_INPUTS if (root / name).is_file()} + + +def record_activation_inputs(stamps: Path, mtimes: dict[str, int]) -> None: + """Give each stamp the exact mtime of the input the install was verified against. + + Recorded on every successful install, including no-op syncs: a checkout that + rewrites an input without changing it moves the mtime, and only this record + brings the stamp back to equal. The prologue compares for equality, not order, + because switching branches can move an input's mtime in either direction. + """ + import shutil + + shutil.rmtree(stamps, ignore_errors=True) + for name, mtime in mtimes.items(): + stamp = stamps / name + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.touch() + os.utime(stamp, ns=(mtime, mtime)) + + def base_venv(project_root: Path) -> Path: root = Path(project_root).resolve() manifest_path = root.parent / "manifest.json" @@ -138,7 +173,7 @@ def venv_python_version(venv: Path) -> tuple[int, int] | None: tree, and failed *after* a successful update. """ try: - for line in (venv / "pyvenv.cfg").read_text(encoding="utf-8").splitlines(): + for line in (venv / "pyvenv.cfg").read_text(encoding="utf-8-sig").splitlines(): key, _, value = line.partition("=") if key.strip() != "version": continue @@ -253,12 +288,10 @@ def activation_environment(project_root: Path) -> dict[str, str]: env.pop("VIRTUAL_ENV", None) env["PYTHONPATH"] = os.pathsep.join([str(project_root.resolve()), str(selected)]) # The child-process sentinel. Its VALUE is the installed-state file this - # environment was composed against, so a consumer gets three things for - # free: that it inherited an activated shell, which checkout/profile that - # shell came from, and a staleness stamp — uv.lock / pyproject.toml / - # pm/lock.json newer than this file means the shell's environment predates - # its inputs. pm rewrites it on every real sync and no-ops otherwise, so a - # `-nt` comparison settles back to "current" after one re-activation. + # environment was composed against, so a consumer learns that it inherited + # an activated shell and which checkout/profile that shell came from. Its + # directory also holds activation_inputs_dir, the input-mtime stamps + # `scripts/_hermes-python` compares against to decide staleness. env["__HERMES_ACTIVATED"] = str(runtime_facts_path(project_root)) return env diff --git a/scripts/_activation.py b/scripts/_activation.py index d732edcbc9..9abda84ae2 100644 --- a/scripts/_activation.py +++ b/scripts/_activation.py @@ -33,20 +33,22 @@ binds it to ``$0`` — so the prologue finds the target relative to itself and the shebang stays independent of the cwd. ``$BASH`` for the second hop keeps it free of both the exec bit and a ``PATH`` lookup. -Activation is not re-run when the inherited environment is still current: the -sentinel's value is the installed-state file the environment was composed -against (see ``pm.environments.activation_environment``), so the -prologue compares it against the inputs that decide the dependency set. Any of -``uv.lock``, ``pyproject.toml`` or ``pm/lock.json`` being newer than that file -means the inherited environment predates its inputs, and the prologue -re-activates once. ``[ -nt ]`` is a bash builtin, so the check costs no process -spawn; pm rewrites that file on every real sync and no-ops otherwise, so it -settles back to current rather than re-syncing on every run. +Activation is not re-run when the inherited environment is still current. pm +records one stamp per dependency input (``uv.lock``, ``pyproject.toml``, +``pm/lock.json``) beside the installed-state file that ``__HERMES_ACTIVATED`` +names, each carrying the exact mtime that input had when the install was last +verified against it (``pm.environments.record_activation_inputs``). The +prologue re-activates when any input's mtime *differs* from its stamp: newer +or older, since switching branches can move it either way. ``[ -nt ]`` and +``[ -ot ]`` are bash builtins, so the check costs no process spawn. Every +successful activation records again, including no-op syncs, so a checkout that +touches an input without changing it costs one re-activation and then settles. Two constraints worth knowing: the line is 83 bytes (a shebang must stay under 127), and ``/usr/bin/env -S`` is GNU and newer-BSD — verify it if an older -macOS or BSD is a target. Windows keeps using ``python scripts\\foo.py`` with -the guard. +macOS or BSD is a target. macOS's stock bash 3.2 compares whole seconds, so an +input rewritten within the same second as its recorded mtime goes unnoticed +there. Windows keeps using ``python scripts\\foo.py`` with the guard. """ from __future__ import annotations diff --git a/scripts/_hermes-python b/scripts/_hermes-python index 237e0278a6..625dc048d5 100755 --- a/scripts/_hermes-python +++ b/scripts/_hermes-python @@ -24,22 +24,30 @@ if [ ! -f "$_here/activate" ]; then exit 1 fi -# __HERMES_ACTIVATED holds the installed-state file the environment was built -# against, so `-nt` against the inputs that decide the dependency set answers -# "is the inherited environment stale". `[ -nt ]` is a bash builtin — no -# process spawn — and pm rewrites that file on every real sync while no-oping -# otherwise, so one re-activation settles this back to current rather than -# re-syncing on every run. A path that no longer exists (or a legacy literal -# "1") fails `-e` and activates. +# pm records, beside the installed-state file named by __HERMES_ACTIVATED, one +# stamp per dependency input carrying the exact mtime that input had when the +# install was last verified against it (pm.environments.record_activation_inputs). +# Any input whose mtime DIFFERS from its stamp — newer or older, since a branch +# switch can move it either way — means the inherited environment may not match +# its inputs. `-nt`/`-ot` are bash builtins, so this costs no process spawn, and +# every successful activation re-records, so one re-activation settles it. No +# stamps (an install from before stamps existed), a missing input, a dangling +# sentinel or a legacy literal "1" all activate. _needs_activation=1 if [ -n "${__HERMES_ACTIVATED:-}" ] && [ -e "${__HERMES_ACTIVATED}" ]; then + _stamps="${__HERMES_ACTIVATED%/*}/inputs" _needs_activation=0 - for _input in uv.lock pyproject.toml pm/lock.json; do - if [ "$_here/$_input" -nt "$__HERMES_ACTIVATED" ]; then + _stamped=0 + for _stamp in "$_stamps"/* "$_stamps"/*/*; do + [ -f "$_stamp" ] || continue + _stamped=1 + _input="$_here/${_stamp#"$_stamps"/}" + if [ ! -e "$_input" ] || [ "$_input" -nt "$_stamp" ] || [ "$_input" -ot "$_stamp" ]; then _needs_activation=1 break fi done + [ "$_stamped" = 1 ] || _needs_activation=1 fi if [ "$_needs_activation" = 1 ]; then diff --git a/tests/scripts/test_hermes_python_prologue.py b/tests/scripts/test_hermes_python_prologue.py index 982a9108d4..ad72f12e10 100644 --- a/tests/scripts/test_hermes_python_prologue.py +++ b/tests/scripts/test_hermes_python_prologue.py @@ -1,11 +1,14 @@ """The self-activating shebang prologue: when does it re-activate? ``scripts/_hermes-python`` is the POSIX shebang target. Its one decision worth -pinning is staleness: ``__HERMES_ACTIVATED`` holds the installed-state file the -environment was built against, so any of ``uv.lock`` / ``pyproject.toml`` / -``pm/lock.json`` being newer than that file means the inherited environment -predates its inputs. Invert that either way and the cost is invisible — a -re-sync on every run, or a stale environment that looks fine. +pinning is staleness: pm stamps each dependency input's mtime beside the +installed-state file ``__HERMES_ACTIVATED`` names, and any input whose mtime +differs from its stamp means the inherited environment may not match its +inputs. Invert that either way and the cost is invisible — a re-sync on every +run, or a stale environment that looks fine. + +The stamps come from the real ``pm.environments`` writer, so these tests pin +the contract between what pm records and what the prologue reads. A ``python3`` shim goes on PATH because the prologue execs ``python3`` by name, which does not exist on stock Windows; the subject here is the staleness @@ -23,13 +26,15 @@ from pathlib import Path import pytest +from pm.environments import ACTIVATION_INPUTS, activation_input_mtimes, record_activation_inputs + REPO_ROOT = Path(__file__).resolve().parents[2] PROLOGUE = REPO_ROOT / "scripts" / "_hermes-python" +EARLIER = "2018-01-01 00:00:00" LONG_AGO = "2019-01-01 00:00:00" STAMP_TIME = "2020-06-01 00:00:00" JUST_AFTER = "2021-01-01 00:00:00" -INPUTS = ("uv.lock", "pyproject.toml", "pm/lock.json") def _posix(path: Path) -> str: @@ -55,16 +60,19 @@ def _set_mtime(path: Path, stamp: str) -> None: @pytest.fixture def checkout(tmp_path: Path) -> Path: - """A repo-shaped tree whose stub activate announces each sourcing.""" + """A repo-shaped tree whose stub activate announces each sourcing; its + installed state lives in ``state/`` (facts.json beside the input stamps).""" root = tmp_path / "checkout" (root / "scripts").mkdir(parents=True) - (root / "pm").mkdir() (root / "shim").mkdir() + (root / "state").mkdir() shutil.copy2(PROLOGUE, root / "scripts" / PROLOGUE.name) - for name in ("uv.lock", "pyproject.toml"): + for name in ACTIVATION_INPUTS: + (root / name).parent.mkdir(parents=True, exist_ok=True) (root / name).touch() - (root / "pm" / "lock.json").touch() - (root / "stamp").touch() + _set_mtime(root / name, LONG_AGO) + (root / "state" / "facts.json").touch() + _set_mtime(root / "state" / "facts.json", STAMP_TIME) # Quote in posix form: a /bin/sh script treats backslashes in an unquoted # word as escapes (same pattern as tests/pm/test_activate_scripts.py). @@ -79,14 +87,19 @@ def checkout(tmp_path: Path) -> Path: ) (root / "activate").write_text( "echo 'ACTIVATED' >&2\n" - "export __HERMES_ACTIVATED=\"%s/stamp\"\n" + "export __HERMES_ACTIVATED=\"%s/state/facts.json\"\n" "export PATH=\"%s/shim:$PATH\"\n" % (_posix(root), _posix(root)), encoding="utf-8", ) return root -def _run(root: Path, sentinel: str | None) -> str: +def _record(root: Path) -> None: + """What a successful ``pm install`` leaves behind.""" + record_activation_inputs(root / "state" / "inputs", activation_input_mtimes(root)) + + +def _run(root: Path, sentinel: str | None = "{root}/state/facts.json") -> str: """Drive the prologue as the kernel would; return stderr, assert it ran.""" env = {**os.environ, "PATH": f"{_posix(root / 'shim')}{os.pathsep}{os.environ.get('PATH', '')}"} env.pop("__HERMES_ACTIVATED", None) @@ -101,28 +114,30 @@ def _run(root: Path, sentinel: str | None) -> str: return result.stderr -def test_current_environment_is_left_alone(checkout: Path): - """Inputs older than the stamp: re-syncing on every run is the cost of - getting this wrong, so the prologue must stay out of the way.""" - _set_mtime(checkout / "stamp", STAMP_TIME) - for name in INPUTS: - _set_mtime(checkout / name, LONG_AGO) - assert "ACTIVATED" not in _run(checkout, "{root}/stamp") +def test_recorded_inputs_are_left_alone(checkout: Path): + """A checkout rewrote every input after facts.json was last written, then a + no-op sync recorded them: the environment is current. Re-syncing on every + run is the cost of getting this wrong.""" + for name in ACTIVATION_INPUTS: + _set_mtime(checkout / name, JUST_AFTER) + _record(checkout) + assert "ACTIVATED" not in _run(checkout) -@pytest.mark.parametrize("input_name", INPUTS) -def test_input_newer_than_stamp_reactivates(checkout: Path, input_name: str): - _set_mtime(checkout / "stamp", STAMP_TIME) - for name in INPUTS: - _set_mtime(checkout / name, LONG_AGO) - _set_mtime(checkout / input_name, JUST_AFTER) - assert "ACTIVATED" in _run(checkout, "{root}/stamp") +@pytest.mark.parametrize("moved_to", [JUST_AFTER, EARLIER], ids=["newer", "older"]) +@pytest.mark.parametrize("input_name", ACTIVATION_INPUTS) +def test_input_mtime_differing_from_its_stamp_reactivates(checkout: Path, input_name: str, moved_to: str): + """A branch switch can move an input's mtime either way; both mean the + recorded install was not verified against what is on disk now.""" + _record(checkout) + _set_mtime(checkout / input_name, moved_to) + assert "ACTIVATED" in _run(checkout) -@pytest.mark.parametrize("sentinel", [None, "{root}/gone", "1"]) +@pytest.mark.parametrize("sentinel", [None, "{root}/gone", "1", "{root}/state/facts.json"], + ids=["cold", "dangling", "legacy-1", "no-stamps"]) def test_unusable_sentinel_activates(checkout: Path, sentinel: str | None): - """Cold, dangling stamp, or the bare ``1`` an older activate exported — - each must activate rather than read as current.""" - for name in INPUTS: - _set_mtime(checkout / name, LONG_AGO) - assert "ACTIVATED" in _run(checkout, sentinel) \ No newline at end of file + """Cold, dangling stamp, the bare ``1`` an older activate exported, or an + install from before stamps existed — each must activate rather than read + as current.""" + assert "ACTIVATED" in _run(checkout, sentinel)