From fe2b130b8b1c21e14dbaf524ba285eb2d5c54b41 Mon Sep 17 00:00:00 2001 From: unsupportedpastels Date: Fri, 9 Oct 2026 17:12:14 +0000 Subject: [PATCH] fix(pm): record the Docker image's baked extras and restore them on boot The image bakes eight extras into /opt/hermes/.venv but never recorded them as PM's dependency selection. The first generation a container built (a lazy extra such as fal, a plugin's dependencies) started from an empty selection, so it carried only what was requested and dropped aiohttp, the messaging SDKs and mcp. After the next restart the API server, platform adapters and MCP servers failed to start. - pm/build_env.py, Dockerfile: --record-selection writes the frozen build's --extra list to the store facts as the venv selection, as a bundle build does. enabled-features.json is not written, so the frozen lazy-install policy stays out of Docker. - pm/recovery.py: stage2's refresh restores extras the image ships that a volume's recorded selection lacks, through the normal explicit sync. On failure the image venv boots and the missing extras stay recorded for the next attempt. A malformed recorded fact is refused, as venv_is_current does. - pm/install.py: share the PEP 685 extra-name key between _still_declared and the boot reconciliation. Fixes #135329 --- Dockerfile | 9 +- pm/build_env.py | 32 ++++++ pm/install.py | 15 ++- pm/recovery.py | 42 ++++++- tests/pm/test_baked_selection.py | 192 +++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 12 deletions(-) create mode 100644 tests/pm/test_baked_selection.py diff --git a/Dockerfile b/Dockerfile index a245d55f39..f93bfbe7dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -270,11 +270,18 @@ FROM runtime_base AS python_deps # is baked so hosted/immutable images can enable the adapter without writing # the sealed venv. # +# --record-selection writes these extras to the store's facts.json as the venv +# selection, as a bundle build does. Without it, the first opt-in extra a +# container installs on use (FAL, a plugin's deps) records a selection of +# just that extra, and every later boot runs a generation without the extras +# baked here (aiohttp, so no API server). stage2's dependency refresh +# restores this baseline into volumes that already lost it. +# # Source binding is created after the source copy below. COPY pyproject.toml uv.lock ./ RUN touch ./README.md RUN python3 -m pm.build_env --source /opt/hermes --python /usr/local/bin/python3 \ - --out /opt/hermes/.venv --no-install-project --sealed \ + --out /opt/hermes/.venv --no-install-project --sealed --record-selection \ --extra all --extra messaging --extra otlp --extra anthropic --extra bedrock \ --extra azure-identity --extra matrix --extra google-chat diff --git a/pm/build_env.py b/pm/build_env.py index 55f9b0afa0..10e29fd30d 100644 --- a/pm/build_env.py +++ b/pm/build_env.py @@ -7,6 +7,33 @@ from pathlib import Path import sys +def _record_selection(source: Path, extras: list[str]) -> None: + """Publish the built extras as the store's venv selection, as a bundle build does. + + A sync that finds no runtime selection starts from this one, so the first opt-in + extra installed on use extends the shipped environment instead of replacing it + with a generation that carries only that extra. The frozen build installed exactly + this list from the lock, so it is recorded as given; an anchor inventory + (``installed_extras``) misses meta extras such as ``all`` and claims extras whose + anchors other packages happen to provide. + """ + from pm.lock import Facts + from pm.packages import Venv + from pm.paths import facts_path + + selected = sorted(set(extras)) + stamp = Venv(source.resolve()).expected_stamp(selected, plugin_dirs=[]) + Facts(facts_path()).record_state("venv", stamp, selected) + + +def _check_record_selection(parser: argparse.ArgumentParser, args: argparse.Namespace, *, + build: bool, requirements: list[str]) -> None: + """Only a project build from an explicit --extra list has a selection to record.""" + if args.record_selection and (not build or requirements or args.manager_runtime + or args.all_extras or args.groups or not args.extras): + parser.error("--record-selection records the --extra list of a project environment build") + + def main(argv: Sequence[str] | None = None) -> int: import pm @@ -35,6 +62,8 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--sealed", action="store_true") parser.add_argument("--offline", action="store_true") parser.add_argument("--explicit", action="store_true", help="CLI builds are always explicit requests") + parser.add_argument("--record-selection", action="store_true", + help="record the --extra list as this install's baseline dependency selection") parser.add_argument("--wheelhouse", type=Path) parser.add_argument("--requirements", type=Path) parser.add_argument("--requirement", action="append", default=[]) @@ -61,6 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int: parser.error("--out is required when building an environment") if args.manager_runtime and args.python is None: parser.error("--manager-runtime requires the target --python") + _check_record_selection(parser, args, build=build, requirements=requirements) try: if args.exact_lock: from pm.uv_cache_prune import prune_uv_cache_to_lock @@ -99,6 +129,8 @@ def main(argv: Sequence[str] | None = None) -> int: no_install_project=args.no_install_project, frozen=not args.resolve, sealed=args.sealed, offline=args.offline, explicit=True, ) + if args.record_selection: + _record_selection(args.source, args.extras) except (pm.InstallError, OSError, ValueError) as exc: print(f"python environment: {exc}", file=sys.stderr) return 1 diff --git a/pm/install.py b/pm/install.py index 224d6eaca5..7ea39a8fe0 100644 --- a/pm/install.py +++ b/pm/install.py @@ -624,6 +624,13 @@ def _member_inputs(plugins: PluginInput | None) -> dict: raise TypeError(f"{type(plugins).__name__} changes plugin state; only a sync may carry it") +def _extra_key(name: str) -> str: + """The PEP 685 name uv matches an extra by (``foo_bar`` is ``foo-bar``).""" + import re + + return re.sub(r"[-_.]+", "-", name).lower() + + def _still_declared(package, recorded: list[str]) -> list[str]: """The recorded extras this tree still declares. @@ -633,17 +640,13 @@ def _still_declared(package, recorded: list[str]) -> list[str]: Membership uses PEP 685 names (uv matches ``foo_bar`` to ``foo-bar``); the recorded spelling is what reaches uv. """ - import re from pm.features import declared_extras - def normalized(name: str) -> str: - return re.sub(r"[-_.]+", "-", name).lower() - root = package.project_root() if not (root / "pyproject.toml").is_file(): return list(recorded) - declared = {normalized(extra) for extra in declared_extras(root)} - return [extra for extra in recorded if normalized(extra) in declared] + declared = {_extra_key(extra) for extra in declared_extras(root)} + return [extra for extra in recorded if _extra_key(extra) in declared] def venv_is_current(*, extras: list[str] | None = None, plugins: Members | Candidates | None = None, diff --git a/pm/recovery.py b/pm/recovery.py index d1bb8f94e5..f05e7d460a 100644 --- a/pm/recovery.py +++ b/pm/recovery.py @@ -63,13 +63,40 @@ def repair_dependencies(project_root: Path) -> None: collect_superseded_generations(project_root) +def _baseline_gaps(root: Path) -> list[str]: + """Extras the store's shipped selection carries that the recorded selection lacks. + + A volume whose first generation was built before the image recorded its selection holds + only what was installed on use (``["fal"]``, or ``[]``), and that generation replaces the + image's environment with one missing everything the image ships. + """ + from pm.environments import runtime_facts_path + from pm.install import _extra_key, _facts, _still_declared + from pm.lock import Facts + from pm.packages import Venv + + fact = Facts(runtime_facts_path(root), strict=True).get("venv") + if fact is None: + return [] + # The same checks venv_is_current applies; a gap must not skip them. + recorded = fact.get("extras") if isinstance(fact, dict) else None + stamp = fact.get("stamp") if isinstance(fact, dict) else None + if (not isinstance(recorded, list) or any(not isinstance(extra, str) for extra in recorded) + or not isinstance(stamp, str) or not stamp): + raise ValueError("invalid recorded dependency state") + shipped = (_facts().get("venv") or {}).get("extras") or [] + have = {_extra_key(extra) for extra in recorded} + return sorted({extra for extra in _still_declared(Venv(root), shipped) if _extra_key(extra) not in have}) + + def refresh_dependencies(project_root: Path) -> str: """Re-resolve the durable selection against inputs an external update replaced. A container image swaps the code and lock under a selection recorded on the data volume; a generation resolved against the previous lock must never boot the new code. Rebuilds the recorded extras and plugins, or on failure boots the image's own environment while keeping - them recorded, so the next boot or install rebuilds them. Returns what happened. + them recorded, so the next boot or install rebuilds them. A recorded selection missing + extras the image ships gets them back. Returns what happened. """ from hermes_cli.runtime_state import runtime_lock from pm.client import sync_venv @@ -83,16 +110,21 @@ def refresh_dependencies(project_root: Path) -> str: raise InstallError("venv", "refresh root does not match this PM installation") if not runtime_facts_path(root).is_file(): return "base" - if venv_is_current(project_root=root): + missing = _baseline_gaps(root) + if not missing and venv_is_current(project_root=root): return "current" try: with contextlib.redirect_stdout(sys.stderr): - sync_venv(explicit=True) - return "rebuilt" + sync_venv(missing or None, explicit=True) + return f"restored {', '.join(missing)}" if missing else "rebuilt" except Exception as exc: print(f"dependency refresh failed: {exc}", file=sys.stderr) with runtime_lock(root, timeout=None): facts = Facts(runtime_facts_path(root), strict=True) fact = facts.get("venv") or {} - facts.record_state("venv", fact.get("stamp") or "stale", list(fact.get("extras") or [])) + # Keep the restored extras recorded too: a lazy install before the next boot + # unions onto this selection and must not publish a generation without them. + recorded = list(fact.get("extras") or []) + facts.record_state("venv", fact.get("stamp") or "stale", + recorded + [extra for extra in missing if extra not in recorded]) return "fallback" diff --git a/tests/pm/test_baked_selection.py b/tests/pm/test_baked_selection.py new file mode 100644 index 0000000000..639f72e930 --- /dev/null +++ b/tests/pm/test_baked_selection.py @@ -0,0 +1,192 @@ +"""A Docker image records the extras it bakes, and boot recovery restores them. + +Without a recorded selection, the first extra a container installs on use +(``fal``) became the whole selection: the generation built from it replaced the +image environment and dropped every baked extra (aiohttp, so no API server). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +PYPROJECT = ( + '[project]\nname="baked"\nversion="1"\n' + '[project.optional-dependencies]\nall=[]\nmessaging=[]\notlp=[]\nfal=[]\nazure-identity=[]\n' +) + + +@pytest.fixture +def install(tmp_path, monkeypatch): + from pm import paths + from pm.packages import Venv + + root = tmp_path / "hermes" + root.mkdir() + (root / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8") + store = tmp_path / "tools" + store.mkdir() + monkeypatch.setattr(paths, "repo_root", lambda: root) + monkeypatch.setattr(paths, "store_root", lambda: store) + monkeypatch.setattr(Venv, "expected_stamp", + lambda self, extras, plugin_dirs=None: "stamp:" + ",".join(sorted(extras))) + return root, store + + +def test_build_records_the_extra_list_as_the_store_selection(install, monkeypatch): + import pm + import pm.features + from pm import build_env + from pm.lock import Facts + + root, store = install + monkeypatch.setattr(pm, "build_environment", lambda **kwargs: root / "venv/bin/python") + # The frozen build's list is the graph; an anchor inventory would drop meta extras like ``all``. + monkeypatch.setattr(pm.features, "installed_extras", lambda *a, **k: pytest.fail("no inventory expected")) + assert build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection", + "--extra", "all", "--extra", "messaging", "--extra", "otlp", "--extra", "all"]) == 0 + fact = Facts(store / "facts.json").get("venv") + assert fact == {"stamp": "stamp:all,messaging,otlp", "extras": ["all", "messaging", "otlp"]} + + +def test_record_selection_refuses_builds_without_an_extra_list(install): + from pm import build_env + + root, _ = install + with pytest.raises(SystemExit): + build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection", "--all-extras"]) + + +@pytest.mark.parametrize("store_extras, expected", [ + (None, ["fal"]), # main before the fix: the lazy extra becomes the whole selection + (["all", "messaging", "otlp"], ["all", "fal", "messaging", "otlp"]), +]) +def test_first_lazy_extra_extends_the_recorded_image_selection(install, store_extras, expected): + from pm.install import _facts, _target_selection + from pm.lock import Facts + from pm.packages import Venv + + root, store = install + if store_extras is not None: + Facts(store / "facts.json").record_state("venv", "image", store_extras) + fact = _facts().get("venv") or {} # sync_venv's fallback when the volume records nothing + enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []}, + repair=False, shipped=None, frozen=None) + assert enabled == expected + + +@pytest.mark.parametrize("recorded", [["fal"], []]) +def test_refresh_restores_baked_extras_a_volume_lost(install, monkeypatch, recorded): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", recorded) + calls = [] + monkeypatch.setattr(pm.client, "sync_venv", lambda extras=None, **kwargs: calls.append((extras, kwargs))) + # The broken generation matches its own stamp: a plain refresh would call it current. + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + + assert refresh_dependencies(root) == "restored all, messaging, otlp" + assert calls == [(["all", "messaging", "otlp"], {"explicit": True})] + + +def test_refresh_leaves_a_complete_selection_alone(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "fal", "messaging"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current" + + +def test_offline_refresh_keeps_baked_extras_for_the_next_lazy_install(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.install import _target_selection + from pm.lock import Facts + from pm.packages import Venv + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["fal"]) + + def offline(*args, **kwargs): + raise RuntimeError("uv sync exited 1: no network") + + monkeypatch.setattr(pm.client, "sync_venv", offline) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + + assert refresh_dependencies(root) == "fallback" + fact = Facts(runtime_facts_path(root)).get("venv") + assert fact is not None and fact["extras"] == ["fal", "all", "messaging", "otlp"] + assert "environment" not in fact # the image environment boots + # A lazy install before the next boot must not drop the baked extras again. + enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []}, + repair=False, shipped=None, frozen=None) + assert enabled == ["all", "fal", "messaging", "otlp"] + + +@pytest.mark.parametrize("venv", [{"stamp": "s", "extras": "fal"}, {"stamp": "s", "extras": None}, + {"stamp": "s", "extras": [1]}, {"stamp": "s"}, + {"extras": ["fal"]}, {"stamp": "", "extras": ["fal"]}, + {"stamp": 7, "extras": ["fal"]}]) +def test_refresh_refuses_an_invalid_recorded_selection(install, monkeypatch, venv): + import json + + import pm.client + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"]) + path = runtime_facts_path(root) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"schema": 1, "packages": {"venv": venv}}), encoding="utf-8") + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("must not sync over a corrupt fact")) + with pytest.raises(ValueError, match="invalid recorded dependency state"): + refresh_dependencies(root) + + +def test_refresh_matches_recorded_extras_by_pep685_name(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "azure-identity"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "Azure_Identity", "fal"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current" + + +def test_refresh_ignores_baked_extras_the_source_no_longer_declares(install, monkeypatch): + import pm.client + import pm.install + from pm.environments import runtime_facts_path + from pm.lock import Facts + from pm.recovery import refresh_dependencies + + root, store = install + Facts(store / "facts.json").record_state("venv", "image", ["all", "hindsight"]) + Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all"]) + monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected")) + monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True) + assert refresh_dependencies(root) == "current"