hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

fix(pm): re-activate from the shebang only when an input's mtime differs

Browse Source 
The _hermes-python prologue compared uv.lock / pyproject.toml / pm/lock.json
against facts.json with `-nt`. facts.json is only rewritten on a real sync,
so a checkout that rewrites an input without changing it left the input
newer forever: every run from an activated shell re-sourced activate
(~370ms instead of ~13ms).

pm now records, after every successful full-closure install (no-op syncs
included), a stamp per input under installs/<key>/inputs carrying the exact
mtime the install was verified against, snapshotted before installing so
an input edited mid-install still reads as stale. The prologue re-activates
when any input's mtime differs from its stamp in either direction (branch
switches can move it backwards), when an input is missing, or when there
are no stamps. It globs the stamp dir, so pm owns the only input list.

Also read pyvenv.cfg as utf-8-sig (footgun lane, same file).
This commit is contained in:
ethernet
2026-09-23 13:38:54 -04:00
parent 95b005fde3
commit b16f953f7c
5 changed files with 128 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pm/cli.py
View File

@@ -177,6 +177,12 @@ def cmd_install(args) -> int:
print("✗ --extra syncs this install's venv and cannot combine with --target")
return 1
names = args.names if args.names or extras else source_install_packages(_lockfile().names())
# Only the whole default closure verifies everything an activated shell
# composes from, so only it advances the prologue's input stamps.
full_closure = not (args.names or tools_only or cross_target)
from pm.environments import activation_input_mtimes
input_mtimes = activation_input_mtimes(repo_root()) if full_closure else {}
# Tools before the venv. A bare `pm install` used to install tools and
# sync the venv in one breath, so a native build (Windows ARM64 source
# wheels) resolved compilers and git from the host PATH. Publish every
@@ -208,6 +214,10 @@ def cmd_install(args) -> int:
except InstallError as e:
print(f"✗ {e}")
failed += 1
if full_closure and not failed:
from pm.environments import activation_inputs_dir, record_activation_inputs
record_activation_inputs(activation_inputs_dir(repo_root()), input_mtimes)
return 1 if failed else 0

47
pm/environments.py
View File

@@ -39,6 +39,41 @@ def runtime_facts_path(project_root: Path) -> Path:
return install_state_dir(project_root) / "facts.json"
# The files that decide the dependency set. `scripts/_hermes-python` re-activates
# when any of them differs in mtime from its stamp under activation_inputs_dir.
ACTIVATION_INPUTS = ("uv.lock", "pyproject.toml", "pm/lock.json")
def activation_inputs_dir(project_root: Path) -> Path:
"""Beside facts.json, so the prologue finds it from ``$__HERMES_ACTIVATED``."""
return install_state_dir(project_root) / "inputs"
def activation_input_mtimes(project_root: Path) -> dict[str, int]:
"""Snapshot before installing, so an input edited mid-install records its
pre-install mtime and the next run re-activates."""
root = Path(project_root)
return {name: (root / name).stat().st_mtime_ns for name in ACTIVATION_INPUTS if (root / name).is_file()}
def record_activation_inputs(stamps: Path, mtimes: dict[str, int]) -> None:
"""Give each stamp the exact mtime of the input the install was verified against.
Recorded on every successful install, including no-op syncs: a checkout that
rewrites an input without changing it moves the mtime, and only this record
brings the stamp back to equal. The prologue compares for equality, not order,
because switching branches can move an input's mtime in either direction.
"""
import shutil
shutil.rmtree(stamps, ignore_errors=True)
for name, mtime in mtimes.items():
stamp = stamps / name
stamp.parent.mkdir(parents=True, exist_ok=True)
stamp.touch()
os.utime(stamp, ns=(mtime, mtime))
def base_venv(project_root: Path) -> Path:
root = Path(project_root).resolve()
manifest_path = root.parent / "manifest.json"
@@ -138,7 +173,7 @@ def venv_python_version(venv: Path) -> tuple[int, int] | None:
tree, and failed *after* a successful update.
"""
try:
for line in (venv / "pyvenv.cfg").read_text(encoding="utf-8").splitlines():
for line in (venv / "pyvenv.cfg").read_text(encoding="utf-8-sig").splitlines():
key, _, value = line.partition("=")
if key.strip() != "version":
continue
@@ -253,12 +288,10 @@ def activation_environment(project_root: Path) -> dict[str, str]:
env.pop("VIRTUAL_ENV", None)
env["PYTHONPATH"] = os.pathsep.join([str(project_root.resolve()), str(selected)])
# The child-process sentinel. Its VALUE is the installed-state file this
# environment was composed against, so a consumer gets three things for
# free: that it inherited an activated shell, which checkout/profile that
# shell came from, and a staleness stamp — uv.lock / pyproject.toml /
# pm/lock.json newer than this file means the shell's environment predates
# its inputs. pm rewrites it on every real sync and no-ops otherwise, so a
# `-nt` comparison settles back to "current" after one re-activation.
# environment was composed against, so a consumer learns that it inherited
# an activated shell and which checkout/profile that shell came from. Its
# directory also holds activation_inputs_dir, the input-mtime stamps
# `scripts/_hermes-python` compares against to decide staleness.
env["__HERMES_ACTIVATED"] = str(runtime_facts_path(project_root))
return env

24
scripts/_activation.py
View File

@@ -33,20 +33,22 @@ binds it to ``$0`` — so the prologue finds the target relative to itself and
the shebang stays independent of the cwd. ``$BASH`` for the second hop keeps it
free of both the exec bit and a ``PATH`` lookup.
Activation is not re-run when the inherited environment is still current: the
sentinel's value is the installed-state file the environment was composed
against (see ``pm.environments.activation_environment``), so the
prologue compares it against the inputs that decide the dependency set. Any of
``uv.lock``, ``pyproject.toml`` or ``pm/lock.json`` being newer than that file
means the inherited environment predates its inputs, and the prologue
re-activates once. ``[ -nt ]`` is a bash builtin, so the check costs no process
spawn; pm rewrites that file on every real sync and no-ops otherwise, so it
settles back to current rather than re-syncing on every run.
Activation is not re-run when the inherited environment is still current. pm
records one stamp per dependency input (``uv.lock``, ``pyproject.toml``,
``pm/lock.json``) beside the installed-state file that ``__HERMES_ACTIVATED``
names, each carrying the exact mtime that input had when the install was last
verified against it (``pm.environments.record_activation_inputs``). The
prologue re-activates when any input's mtime *differs* from its stamp: newer
or older, since switching branches can move it either way. ``[ -nt ]`` and
``[ -ot ]`` are bash builtins, so the check costs no process spawn. Every
successful activation records again, including no-op syncs, so a checkout that
touches an input without changing it costs one re-activation and then settles.
Two constraints worth knowing: the line is 83 bytes (a shebang must stay under
127), and ``/usr/bin/env -S`` is GNU and newer-BSD — verify it if an older
macOS or BSD is a target. Windows keeps using ``python scripts\\foo.py`` with
the guard.
macOS or BSD is a target. macOS's stock bash 3.2 compares whole seconds, so an
input rewritten within the same second as its recorded mtime goes unnoticed
there. Windows keeps using ``python scripts\\foo.py`` with the guard.
"""
from __future__ import annotations

26
scripts/_hermes-python
View File

@@ -24,22 +24,30 @@ if [ ! -f "$_here/activate" ]; then
exit 1
fi
# __HERMES_ACTIVATED holds the installed-state file the environment was built
# against, so `-nt` against the inputs that decide the dependency set answers
# "is the inherited environment stale". `[ -nt ]` is a bash builtin — no
# process spawn — and pm rewrites that file on every real sync while no-oping
# otherwise, so one re-activation settles this back to current rather than
# re-syncing on every run. A path that no longer exists (or a legacy literal
# "1") fails `-e` and activates.
# pm records, beside the installed-state file named by __HERMES_ACTIVATED, one
# stamp per dependency input carrying the exact mtime that input had when the
# install was last verified against it (pm.environments.record_activation_inputs).
# Any input whose mtime DIFFERS from its stamp — newer or older, since a branch
# switch can move it either way — means the inherited environment may not match
# its inputs. `-nt`/`-ot` are bash builtins, so this costs no process spawn, and
# every successful activation re-records, so one re-activation settles it. No
# stamps (an install from before stamps existed), a missing input, a dangling
# sentinel or a legacy literal "1" all activate.
_needs_activation=1
if [ -n "${__HERMES_ACTIVATED:-}" ] && [ -e "${__HERMES_ACTIVATED}" ]; then
_stamps="${__HERMES_ACTIVATED%/*}/inputs"
_needs_activation=0
for _input in uv.lock pyproject.toml pm/lock.json; do
if [ "$_here/$_input" -nt "$__HERMES_ACTIVATED" ]; then
_stamped=0
for _stamp in "$_stamps"/* "$_stamps"/*/*; do
[ -f "$_stamp" ] || continue
_stamped=1
_input="$_here/${_stamp#"$_stamps"/}"
if [ ! -e "$_input" ] || [ "$_input" -nt "$_stamp" ] || [ "$_input" -ot "$_stamp" ]; then
_needs_activation=1
break
fi
done
[ "$_stamped" = 1 ] || _needs_activation=1
fi
if [ "$_needs_activation" = 1 ]; then

81
tests/scripts/test_hermes_python_prologue.py
View File

@@ -1,11 +1,14 @@
"""The self-activating shebang prologue: when does it re-activate?
``scripts/_hermes-python`` is the POSIX shebang target. Its one decision worth
pinning is staleness: ``__HERMES_ACTIVATED`` holds the installed-state file the
environment was built against, so any of ``uv.lock`` / ``pyproject.toml`` /
``pm/lock.json`` being newer than that file means the inherited environment
predates its inputs. Invert that either way and the cost is invisible — a
re-sync on every run, or a stale environment that looks fine.
pinning is staleness: pm stamps each dependency input's mtime beside the
installed-state file ``__HERMES_ACTIVATED`` names, and any input whose mtime
differs from its stamp means the inherited environment may not match its
inputs. Invert that either way and the cost is invisible — a re-sync on every
run, or a stale environment that looks fine.
The stamps come from the real ``pm.environments`` writer, so these tests pin
the contract between what pm records and what the prologue reads.
A ``python3`` shim goes on PATH because the prologue execs ``python3`` by name,
which does not exist on stock Windows; the subject here is the staleness
@@ -23,13 +26,15 @@ from pathlib import Path
import pytest
from pm.environments import ACTIVATION_INPUTS, activation_input_mtimes, record_activation_inputs
REPO_ROOT = Path(__file__).resolve().parents[2]
PROLOGUE = REPO_ROOT / "scripts" / "_hermes-python"
EARLIER = "2018-01-01 00:00:00"
LONG_AGO = "2019-01-01 00:00:00"
STAMP_TIME = "2020-06-01 00:00:00"
JUST_AFTER = "2021-01-01 00:00:00"
INPUTS = ("uv.lock", "pyproject.toml", "pm/lock.json")
def _posix(path: Path) -> str:
@@ -55,16 +60,19 @@ def _set_mtime(path: Path, stamp: str) -> None:
@pytest.fixture
def checkout(tmp_path: Path) -> Path:
"""A repo-shaped tree whose stub activate announces each sourcing."""
"""A repo-shaped tree whose stub activate announces each sourcing; its
installed state lives in ``state/`` (facts.json beside the input stamps)."""
root = tmp_path / "checkout"
(root / "scripts").mkdir(parents=True)
(root / "pm").mkdir()
(root / "shim").mkdir()
(root / "state").mkdir()
shutil.copy2(PROLOGUE, root / "scripts" / PROLOGUE.name)
for name in ("uv.lock", "pyproject.toml"):
for name in ACTIVATION_INPUTS:
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).touch()
(root / "pm" / "lock.json").touch()
(root / "stamp").touch()
_set_mtime(root / name, LONG_AGO)
(root / "state" / "facts.json").touch()
_set_mtime(root / "state" / "facts.json", STAMP_TIME)
# Quote in posix form: a /bin/sh script treats backslashes in an unquoted
# word as escapes (same pattern as tests/pm/test_activate_scripts.py).
@@ -79,14 +87,19 @@ def checkout(tmp_path: Path) -> Path:
)
(root / "activate").write_text(
"echo 'ACTIVATED' >&2\n"
"export __HERMES_ACTIVATED=\"%s/stamp\"\n"
"export __HERMES_ACTIVATED=\"%s/state/facts.json\"\n"
"export PATH=\"%s/shim:$PATH\"\n" % (_posix(root), _posix(root)),
encoding="utf-8",
)
return root
def _run(root: Path, sentinel: str | None) -> str:
def _record(root: Path) -> None:
"""What a successful ``pm install`` leaves behind."""
record_activation_inputs(root / "state" / "inputs", activation_input_mtimes(root))
def _run(root: Path, sentinel: str | None = "{root}/state/facts.json") -> str:
"""Drive the prologue as the kernel would; return stderr, assert it ran."""
env = {**os.environ, "PATH": f"{_posix(root / 'shim')}{os.pathsep}{os.environ.get('PATH', '')}"}
env.pop("__HERMES_ACTIVATED", None)
@@ -101,28 +114,30 @@ def _run(root: Path, sentinel: str | None) -> str:
return result.stderr
def test_current_environment_is_left_alone(checkout: Path):
"""Inputs older than the stamp: re-syncing on every run is the cost of
getting this wrong, so the prologue must stay out of the way."""
_set_mtime(checkout / "stamp", STAMP_TIME)
for name in INPUTS:
_set_mtime(checkout / name, LONG_AGO)
assert "ACTIVATED" not in _run(checkout, "{root}/stamp")
def test_recorded_inputs_are_left_alone(checkout: Path):
"""A checkout rewrote every input after facts.json was last written, then a
no-op sync recorded them: the environment is current. Re-syncing on every
run is the cost of getting this wrong."""
for name in ACTIVATION_INPUTS:
_set_mtime(checkout / name, JUST_AFTER)
_record(checkout)
assert "ACTIVATED" not in _run(checkout)
@pytest.mark.parametrize("input_name", INPUTS)
def test_input_newer_than_stamp_reactivates(checkout: Path, input_name: str):
_set_mtime(checkout / "stamp", STAMP_TIME)
for name in INPUTS:
_set_mtime(checkout / name, LONG_AGO)
_set_mtime(checkout / input_name, JUST_AFTER)
assert "ACTIVATED" in _run(checkout, "{root}/stamp")
@pytest.mark.parametrize("moved_to", [JUST_AFTER, EARLIER], ids=["newer", "older"])
@pytest.mark.parametrize("input_name", ACTIVATION_INPUTS)
def test_input_mtime_differing_from_its_stamp_reactivates(checkout: Path, input_name: str, moved_to: str):
"""A branch switch can move an input's mtime either way; both mean the
recorded install was not verified against what is on disk now."""
_record(checkout)
_set_mtime(checkout / input_name, moved_to)
assert "ACTIVATED" in _run(checkout)
@pytest.mark.parametrize("sentinel", [None, "{root}/gone", "1"])
@pytest.mark.parametrize("sentinel", [None, "{root}/gone", "1", "{root}/state/facts.json"],
ids=["cold", "dangling", "legacy-1", "no-stamps"])
def test_unusable_sentinel_activates(checkout: Path, sentinel: str | None):
"""Cold, dangling stamp, or the bare ``1`` an older activate exported —
each must activate rather than read as current."""
for name in INPUTS:
_set_mtime(checkout / name, LONG_AGO)
assert "ACTIVATED" in _run(checkout, sentinel)
"""Cold, dangling stamp, the bare ``1`` an older activate exported, or an
install from before stamps existed — each must activate rather than read
as current."""
assert "ACTIVATED" in _run(checkout, sentinel)