diff --git a/Dockerfile b/Dockerfile index a245d55f39..f93bfbe7dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -270,11 +270,18 @@ FROM runtime_base AS python_deps # is baked so hosted/immutable images can enable the adapter without writing # the sealed venv. # +# --record-selection writes these extras to the store's facts.json as the venv +# selection, as a bundle build does. Without it, the first opt-in extra a +# container installs on use (FAL, a plugin's deps) records a selection of +# just that extra, and every later boot runs a generation without the extras +# baked here (aiohttp, so no API server). stage2's dependency refresh +# restores this baseline into volumes that already lost it. +# # Source binding is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md RUN python3 -m pm.build_env --source /opt/hermes --python /usr/local/bin/python3 \ - --out /opt/hermes/.venv --no-install-project --sealed \ + --out /opt/hermes/.venv --no-install-project --sealed --record-selection \ --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock \ --extra azure-identity --extra matrix --extra google-chat diff --git a/pm/build_env.py b/pm/build_env.py index 55f9b0afa0..10e29fd30d 100644 --- a/pm/build_env.py +++ b/pm/build_env.py @@ -7,6 +7,33 @@ from pathlib import Path import sys +def _record_selection(source: Path, extras: list[str]) -> None: + """Publish the built extras as the store's venv selection, as a bundle build does. + + A sync that finds no runtime selection starts from this one, so the first opt-in + extra installed on use extends the shipped environment instead of replacing it + with a generation that carries only that extra. The frozen build installed exactly + this list from the lock, so it is recorded as given; an anchor inventory + (``installed_extras``) misses meta extras such as ``all`` and claims extras whose + anchors other packages happen to provide. + """ + from pm.lock import Facts + from pm.packages import Venv + from pm.paths import facts_path + + selected = sorted(set(extras)) + stamp = Venv(source.resolve()).expected_stamp(selected, plugin_dirs=[]) + Facts(facts_path()).record_state("venv", stamp, selected) + + +def _check_record_selection(parser: argparse.ArgumentParser, args: argparse.Namespace, *, + build: bool, requirements: list[str]) -> None: + """Only a project build from an explicit --extra list has a selection to record.""" + if args.record_selection and (not build or requirements or args.manager_runtime + or args.all_extras or args.groups or not args.extras): + parser.error("--record-selection records the --extra list of a project environment build") + + def main(argv: Sequence[str] | None = None) -> int: import pm @@ -35,6 +62,8 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--sealed", action="store_true") parser.add_argument("--offline", action="store_true") parser.add_argument("--explicit", action="store_true", help="CLI builds are always explicit requests") + parser.add_argument("--record-selection", action="store_true", + help="record the --extra list as this install's baseline dependency selection") parser.add_argument("--wheelhouse", type=Path) parser.add_argument("--requirements", type=Path) parser.add_argument("--requirement", action="append", default=[]) @@ -61,6 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error("--out is required when building an environment") if args.manager_runtime and args.python is None: parser.error("--manager-runtime requires the target --python") + _check_record_selection(parser, args, build=build, requirements=requirements) try: if args.exact_lock: from pm.uv_cache_prune import prune_uv_cache_to_lock @@ -99,6 +129,8 @@ def main(argv: Sequence[str] | None = None) -> int: no_install_project=args.no_install_project, frozen=not args.resolve, sealed=args.sealed, offline=args.offline, explicit=True, ) + if args.record_selection: + _record_selection(args.source, args.extras) except (pm.InstallError, OSError, ValueError) as exc: print(f"python environment: {exc}", file=sys.stderr) return 1 diff --git a/pm/install.py b/pm/install.py index 224d6eaca5..7ea39a8fe0 100644 --- a/pm/install.py +++ b/pm/install.py @@ -624,6 +624,13 @@ def _member_inputs(plugins: PluginInput | None) -> dict: raise TypeError(f"{type(plugins).__name__} changes plugin state; only a sync may carry it") +def _extra_key(name: str) -> str: + """The PEP 685 name uv matches an extra by (``foo_bar`` is ``foo-bar``).""" + import re + + return re.sub(r"[-_.]+", "-", name).lower() + + def _still_declared(package, recorded: list[str]) -> list[str]: """The recorded extras this tree still declares. @@ -633,17 +640,13 @@ def _still_declared(package, recorded: list[str]) -> list[str]: Membership uses PEP 685 names (uv matches ``foo_bar`` to ``foo-bar``); the recorded spelling is what reaches uv. """ - import re from pm.features import declared_extras - def normalized(name: str) -> str: - return re.sub(r"[-_.]+", "-", name).lower() - root = package.project_root() if not (root / "pyproject.toml").is_file(): return list(recorded) - declared = {normalized(extra) for extra in declared_extras(root)} - return [extra for extra in recorded if normalized(extra) in declared] + declared = {_extra_key(extra) for extra in declared_extras(root)} + return [extra for extra in recorded if _extra_key(extra) in declared] def venv_is_current(*, extras: list[str] | None = None, plugins: Members | Candidates | None = None, diff --git a/pm/recovery.py b/pm/recovery.py index d1bb8f94e5..f05e7d460a 100644 --- a/pm/recovery.py +++ b/pm/recovery.py @@ -63,13 +63,40 @@ def repair_dependencies(project_root: Path) -> None: collect_superseded_generations(project_root) +def _baseline_gaps(root: Path) -> list[str]: + """Extras the store's shipped selection carries that the recorded selection lacks. + + A volume whose first generation was built before the image recorded its selection holds + only what was installed on use (``["fal"]``, or ``[]``), and that generation replaces the + image's environment with one missing everything the image ships. + """ + from pm.environments import runtime_facts_path + from pm.install import _extra_key, _facts, _still_declared + from pm.lock import Facts + from pm.packages import Venv + + fact = Facts(runtime_facts_path(root), strict=True).get("venv") + if fact is None: + return [] + # The same checks venv_is_current applies; a gap must not skip them. + recorded = fact.get("extras") if isinstance(fact, dict) else None + stamp = fact.get("stamp") if isinstance(fact, dict) else None + if (not isinstance(recorded, list) or any(not isinstance(extra, str) for extra in recorded) + or not isinstance(stamp, str) or not stamp): + raise ValueError("invalid recorded dependency state") + shipped = (_facts().get("venv") or {}).get("extras") or [] + have = {_extra_key(extra) for extra in recorded} + return sorted({extra for extra in _still_declared(Venv(root), shipped) if _extra_key(extra) not in have}) + + def refresh_dependencies(project_root: Path) -> str: """Re-resolve the durable selection against inputs an external update replaced. A container image swaps the code and lock under a selection recorded on the data volume; a generation resolved against the previous lock must never boot the new code. Rebuilds the recorded extras and plugins, or on failure boots the image's own environment while keeping - them recorded, so the next boot or install rebuilds them. Returns what happened. + them recorded, so the next boot or install rebuilds them. A recorded selection missing + extras the image ships gets them back. Returns what happened. """ from hermes_cli.runtime_state import runtime_lock from pm.client import sync_venv @@ -83,16 +110,21 @@ def refresh_dependencies(project_root: Path) -> str: raise InstallError("venv", "refresh root does not match this PM installation") if not runtime_facts_path(root).is_file(): return "base" - if venv_is_current(project_root=root): + missing = _baseline_gaps(root) + if not missing and venv_is_current(project_root=root): return "current" try: with contextlib.redirect_stdout(sys.stderr): - sync_venv(explicit=True) - return "rebuilt" + sync_venv(missing or None, explicit=True) + return f"restored {', '.join(missing)}" if missing else "rebuilt" except Exception as exc: print(f"dependency refresh failed: {exc}", file=sys.stderr) with runtime_lock(root, timeout=None): facts = Facts(runtime_facts_path(root), strict=True) fact = facts.get("venv") or {} - facts.record_state("venv", fact.get("stamp") or "stale", list(fact.get("extras") or [])) + # Keep the restored extras recorded too: a lazy install before the next boot + # unions onto this selection and must not publish a generation without them. + recorded = list(fact.get("extras") or []) + facts.record_state("venv", fact.get("stamp") or "stale", + recorded + [extra for extra in missing if extra not in recorded]) return "fallback" diff --git a/tests/pm/test_baked_selection.py b/tests/pm/test_baked_selection.py new file mode 100644 index 0000000000..639f72e930 --- /dev/null +++ b/tests/pm/test_baked_selection.py @@ -0,0 +1,192 @@ +"""A Docker image records the extras it bakes, and boot recovery restores them. + +Without a recorded selection, the first extra a container installs on use +(``fal``) became the whole selection: the generation built from it replaced the +image environment and dropped every baked extra (aiohttp, so no API server). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +PYPROJECT = ( + '[project]\nname="baked"\nversion="1"\n' + '[project.optional-dependencies]\nall=[]\nmessaging=[]\notlp=[]\nfal=[]\nazure-identity=[]\n' +) + + +@pytest.fixture +def install(tmp_path, monkeypatch): + from pm import paths + from pm.packages import Venv + + root = tmp_path / "hermes" + root.mkdir() + (root / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8") + store = tmp_path / "tools" + store.mkdir() + monkeypatch.setattr(paths, "repo_root", lambda: root) + monkeypatch.setattr(paths, "store_root", lambda: store) + monkeypatch.setattr(Venv, "expected_stamp", + lambda self, extras, plugin_dirs=None: "stamp:" + ",".join(sorted(extras))) + return root, store + + +def test_build_records_the_extra_list_as_the_store_selection(install, monkeypatch): + import pm + import pm.features + from pm import build_env + from pm.lock import Facts + + root, store = install + monkeypatch.setattr(pm, "build_environment", lambda **kwargs: root / "venv/bin/python") + # The frozen build's list is the graph; an anchor inventory would drop meta extras like ``all``. + monkeypatch.setattr(pm.features, "installed_extras", lambda *a, **k: pytest.fail("no inventory expected")) + assert build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection", + "--extra", "all", "--extra", "messaging", "--extra", "otlp", "--extra", "all"]) == 0 + fact = Facts(store / "facts.json").get("venv") + assert fact == {"stamp": "stamp:all,messaging,otlp", "extras": ["all", "messaging", "otlp"]} + + +def test_record_selection_refuses_builds_without_an_extra_list(install): + from pm import build_env + + root, _ = install + with pytest.raises(SystemExit): + build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection", "--all-extras"]) + + +@pytest.mark.parametrize("store_extras, expected", [ + (None, ["fal"]), # main before the fix: the lazy extra becomes the whole selection + (["all", "messaging", "otlp"], ["all", "fal", "messaging", "otlp"]), +]) +def test_first_lazy_extra_extends_the_recorded_image_selection(install, store_extras, expected): + from pm.install import _facts, _target_selection + from pm.lock import Facts + from pm.packages import Venv + + root, store = install + if store_extras is not None: + Facts(store / "facts.json").record_state("venv", "image", store_extras) + fact = _facts().get("venv") or {} # sync_venv's fallback when the volume records nothing + enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []}, + repair=False, shipped=None, frozen=None) + assert enabled == expected + + +@pytest.mark.parametrize("recorded", [["fal"], []]) +def test_refresh_restores_baked_extras_a_volume_lost(install, monkeypatch, recorded): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", recorded) + calls = [] + monkeypatch.setattr(pm.client, "sync_venv", lambda extras=None, **kwargs: calls.append((extras, kwargs))) + # The broken generation matches its own stamp: a plain refresh would call it current. + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + + assert refresh_dependencies(root) == "restored all, messaging, otlp" + assert calls == [(["all", "messaging", "otlp"], {"explicit": True})] + + +def test_refresh_leaves_a_complete_selection_alone(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "fal", "messaging"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current" + + +def test_offline_refresh_keeps_baked_extras_for_the_next_lazy_install(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.install import _target_selection + from pm.lock import Facts + from pm.packages import Venv + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["fal"]) + + def offline(*args, **kwargs): + raise RuntimeError("uv sync exited 1: no network") + + monkeypatch.setattr(pm.client, "sync_venv", offline) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + + assert refresh_dependencies(root) == "fallback" + fact = Facts(runtime_facts_path(root)).get("venv") + assert fact is not None and fact["extras"] == ["fal", "all", "messaging", "otlp"] + assert "environment" not in fact # the image environment boots + # A lazy install before the next boot must not drop the baked extras again. + enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []}, + repair=False, shipped=None, frozen=None) + assert enabled == ["all", "fal", "messaging", "otlp"] + + +@pytest.mark.parametrize("venv", [{"stamp": "s", "extras": "fal"}, {"stamp": "s", "extras": None}, + {"stamp": "s", "extras": [1]}, {"stamp": "s"}, + {"extras": ["fal"]}, {"stamp": "", "extras": ["fal"]}, + {"stamp": 7, "extras": ["fal"]}]) +def test_refresh_refuses_an_invalid_recorded_selection(install, monkeypatch, venv): + import json + + import pm.client + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"]) + path = runtime_facts_path(root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"schema": 1, "packages": {"venv": venv}}), encoding="utf-8") + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("must not sync over a corrupt fact")) + with pytest.raises(ValueError, match="invalid recorded dependency state"): + refresh_dependencies(root) + + +def test_refresh_matches_recorded_extras_by_pep685_name(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "azure-identity"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "Azure_Identity", "fal"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current" + + +def test_refresh_ignores_baked_extras_the_source_no_longer_declares(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "hindsight"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current"