hermes-admin/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #135726 from unsupportedpastels/fix/pm-docker-baked-baseline

Browse Source 
fix(pm): record the Docker image's baked extras and restore them on boot
This commit is contained in:
ethernet
2026-10-09 13:46:18 -04:00
committed by GitHub
parent 32b77f51c1 fe2b130b8b
commit 26610b09a5
5 changed files with 278 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
Dockerfile
View File

@@ -270,11 +270,18 @@ FROM runtime_base AS python_deps
# is baked so hosted/immutable images can enable the adapter without writing
# the sealed venv.
#
# --record-selection writes these extras to the store's facts.json as the venv
# selection, as a bundle build does. Without it, the first opt-in extra a
# container installs on use (FAL, a plugin's deps) records a selection of
# just that extra, and every later boot runs a generation without the extras
# baked here (aiohttp, so no API server). stage2's dependency refresh
# restores this baseline into volumes that already lost it.
#
# Source binding is created after the source copy below.
COPY pyproject.toml uv.lock ./
RUN touch ./README.md
RUN python3 -m pm.build_env --source /opt/hermes --python /usr/local/bin/python3 \
--out /opt/hermes/.venv --no-install-project --sealed \
--out /opt/hermes/.venv --no-install-project --sealed --record-selection \
--extra all --extra messaging --extra otlp --extra anthropic --extra bedrock \
--extra azure-identity --extra matrix --extra google-chat

32
pm/build_env.py
View File

@@ -7,6 +7,33 @@ from pathlib import Path
import sys
def _record_selection(source: Path, extras: list[str]) -> None:
"""Publish the built extras as the store's venv selection, as a bundle build does.
A sync that finds no runtime selection starts from this one, so the first opt-in
extra installed on use extends the shipped environment instead of replacing it
with a generation that carries only that extra. The frozen build installed exactly
this list from the lock, so it is recorded as given; an anchor inventory
(``installed_extras``) misses meta extras such as ``all`` and claims extras whose
anchors other packages happen to provide.
"""
from pm.lock import Facts
from pm.packages import Venv
from pm.paths import facts_path
selected = sorted(set(extras))
stamp = Venv(source.resolve()).expected_stamp(selected, plugin_dirs=[])
Facts(facts_path()).record_state("venv", stamp, selected)
def _check_record_selection(parser: argparse.ArgumentParser, args: argparse.Namespace, *,
build: bool, requirements: list[str]) -> None:
"""Only a project build from an explicit --extra list has a selection to record."""
if args.record_selection and (not build or requirements or args.manager_runtime
or args.all_extras or args.groups or not args.extras):
parser.error("--record-selection records the --extra list of a project environment build")
def main(argv: Sequence[str] | None = None) -> int:
import pm
@@ -35,6 +62,8 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.add_argument("--sealed", action="store_true")
parser.add_argument("--offline", action="store_true")
parser.add_argument("--explicit", action="store_true", help="CLI builds are always explicit requests")
parser.add_argument("--record-selection", action="store_true",
help="record the --extra list as this install's baseline dependency selection")
parser.add_argument("--wheelhouse", type=Path)
parser.add_argument("--requirements", type=Path)
parser.add_argument("--requirement", action="append", default=[])
@@ -61,6 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int:
parser.error("--out is required when building an environment")
if args.manager_runtime and args.python is None:
parser.error("--manager-runtime requires the target --python")
_check_record_selection(parser, args, build=build, requirements=requirements)
try:
if args.exact_lock:
from pm.uv_cache_prune import prune_uv_cache_to_lock
@@ -99,6 +129,8 @@ def main(argv: Sequence[str] | None = None) -> int:
no_install_project=args.no_install_project, frozen=not args.resolve,
sealed=args.sealed, offline=args.offline, explicit=True,
)
if args.record_selection:
_record_selection(args.source, args.extras)
except (pm.InstallError, OSError, ValueError) as exc:
print(f"python environment: {exc}", file=sys.stderr)
return 1

15
pm/install.py
View File

@@ -624,6 +624,13 @@ def _member_inputs(plugins: PluginInput | None) -> dict:
raise TypeError(f"{type(plugins).__name__} changes plugin state; only a sync may carry it")
def _extra_key(name: str) -> str:
"""The PEP 685 name uv matches an extra by (``foo_bar`` is ``foo-bar``)."""
import re
return re.sub(r"[-_.]+", "-", name).lower()
def _still_declared(package, recorded: list[str]) -> list[str]:
"""The recorded extras this tree still declares.
@@ -633,17 +640,13 @@ def _still_declared(package, recorded: list[str]) -> list[str]:
Membership uses PEP 685 names (uv matches ``foo_bar`` to ``foo-bar``); the
recorded spelling is what reaches uv.
"""
import re
from pm.features import declared_extras
def normalized(name: str) -> str:
return re.sub(r"[-_.]+", "-", name).lower()
root = package.project_root()
if not (root / "pyproject.toml").is_file():
return list(recorded)
declared = {normalized(extra) for extra in declared_extras(root)}
return [extra for extra in recorded if normalized(extra) in declared]
declared = {_extra_key(extra) for extra in declared_extras(root)}
return [extra for extra in recorded if _extra_key(extra) in declared]
def venv_is_current(*, extras: list[str] | None = None, plugins: Members | Candidates | None = None,

42
pm/recovery.py
View File

@@ -63,13 +63,40 @@ def repair_dependencies(project_root: Path) -> None:
collect_superseded_generations(project_root)
def _baseline_gaps(root: Path) -> list[str]:
"""Extras the store's shipped selection carries that the recorded selection lacks.
A volume whose first generation was built before the image recorded its selection holds
only what was installed on use (``["fal"]``, or ``[]``), and that generation replaces the
image's environment with one missing everything the image ships.
"""
from pm.environments import runtime_facts_path
from pm.install import _extra_key, _facts, _still_declared
from pm.lock import Facts
from pm.packages import Venv
fact = Facts(runtime_facts_path(root), strict=True).get("venv")
if fact is None:
return []
# The same checks venv_is_current applies; a gap must not skip them.
recorded = fact.get("extras") if isinstance(fact, dict) else None
stamp = fact.get("stamp") if isinstance(fact, dict) else None
if (not isinstance(recorded, list) or any(not isinstance(extra, str) for extra in recorded)
or not isinstance(stamp, str) or not stamp):
raise ValueError("invalid recorded dependency state")
shipped = (_facts().get("venv") or {}).get("extras") or []
have = {_extra_key(extra) for extra in recorded}
return sorted({extra for extra in _still_declared(Venv(root), shipped) if _extra_key(extra) not in have})
def refresh_dependencies(project_root: Path) -> str:
"""Re-resolve the durable selection against inputs an external update replaced.
A container image swaps the code and lock under a selection recorded on the data volume; a
generation resolved against the previous lock must never boot the new code. Rebuilds the
recorded extras and plugins, or on failure boots the image's own environment while keeping
them recorded, so the next boot or install rebuilds them. Returns what happened.
them recorded, so the next boot or install rebuilds them. A recorded selection missing
extras the image ships gets them back. Returns what happened.
"""
from hermes_cli.runtime_state import runtime_lock
from pm.client import sync_venv
@@ -83,16 +110,21 @@ def refresh_dependencies(project_root: Path) -> str:
raise InstallError("venv", "refresh root does not match this PM installation")
if not runtime_facts_path(root).is_file():
return "base"
if venv_is_current(project_root=root):
missing = _baseline_gaps(root)
if not missing and venv_is_current(project_root=root):
return "current"
try:
with contextlib.redirect_stdout(sys.stderr):
sync_venv(explicit=True)
return "rebuilt"
sync_venv(missing or None, explicit=True)
return f"restored {', '.join(missing)}" if missing else "rebuilt"
except Exception as exc:
print(f"dependency refresh failed: {exc}", file=sys.stderr)
with runtime_lock(root, timeout=None):
facts = Facts(runtime_facts_path(root), strict=True)
fact = facts.get("venv") or {}
facts.record_state("venv", fact.get("stamp") or "stale", list(fact.get("extras") or []))
# Keep the restored extras recorded too: a lazy install before the next boot
# unions onto this selection and must not publish a generation without them.
recorded = list(fact.get("extras") or [])
facts.record_state("venv", fact.get("stamp") or "stale",
recorded + [extra for extra in missing if extra not in recorded])
return "fallback"

192
tests/pm/test_baked_selection.py Normal file
View File

@@ -0,0 +1,192 @@
"""A Docker image records the extras it bakes, and boot recovery restores them.
Without a recorded selection, the first extra a container installs on use
(``fal``) became the whole selection: the generation built from it replaced the
image environment and dropped every baked extra (aiohttp, so no API server).
"""
from __future__ import annotations
from pathlib import Path
import pytest
PYPROJECT = (
'[project]\nname="baked"\nversion="1"\n'
'[project.optional-dependencies]\nall=[]\nmessaging=[]\notlp=[]\nfal=[]\nazure-identity=[]\n'
)
@pytest.fixture
def install(tmp_path, monkeypatch):
from pm import paths
from pm.packages import Venv
root = tmp_path / "hermes"
root.mkdir()
(root / "pyproject.toml").write_text(PYPROJECT, encoding="utf-8")
store = tmp_path / "tools"
store.mkdir()
monkeypatch.setattr(paths, "repo_root", lambda: root)
monkeypatch.setattr(paths, "store_root", lambda: store)
monkeypatch.setattr(Venv, "expected_stamp",
lambda self, extras, plugin_dirs=None: "stamp:" + ",".join(sorted(extras)))
return root, store
def test_build_records_the_extra_list_as_the_store_selection(install, monkeypatch):
import pm
import pm.features
from pm import build_env
from pm.lock import Facts
root, store = install
monkeypatch.setattr(pm, "build_environment", lambda **kwargs: root / "venv/bin/python")
# The frozen build's list is the graph; an anchor inventory would drop meta extras like ``all``.
monkeypatch.setattr(pm.features, "installed_extras", lambda *a, **k: pytest.fail("no inventory expected"))
assert build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection",
"--extra", "all", "--extra", "messaging", "--extra", "otlp", "--extra", "all"]) == 0
fact = Facts(store / "facts.json").get("venv")
assert fact == {"stamp": "stamp:all,messaging,otlp", "extras": ["all", "messaging", "otlp"]}
def test_record_selection_refuses_builds_without_an_extra_list(install):
from pm import build_env
root, _ = install
with pytest.raises(SystemExit):
build_env.main(["--source", str(root), "--out", str(root / "venv"), "--record-selection", "--all-extras"])
@pytest.mark.parametrize("store_extras, expected", [
(None, ["fal"]), # main before the fix: the lazy extra becomes the whole selection
(["all", "messaging", "otlp"], ["all", "fal", "messaging", "otlp"]),
])
def test_first_lazy_extra_extends_the_recorded_image_selection(install, store_extras, expected):
from pm.install import _facts, _target_selection
from pm.lock import Facts
from pm.packages import Venv
root, store = install
if store_extras is not None:
Facts(store / "facts.json").record_state("venv", "image", store_extras)
fact = _facts().get("venv") or {} # sync_venv's fallback when the volume records nothing
enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []},
repair=False, shipped=None, frozen=None)
assert enabled == expected
@pytest.mark.parametrize("recorded", [["fal"], []])
def test_refresh_restores_baked_extras_a_volume_lost(install, monkeypatch, recorded):
import pm.client
import pm.install
from pm.environments import runtime_facts_path
from pm.lock import Facts
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"])
Facts(runtime_facts_path(root)).record_state("venv", "lazy", recorded)
calls = []
monkeypatch.setattr(pm.client, "sync_venv", lambda extras=None, **kwargs: calls.append((extras, kwargs)))
# The broken generation matches its own stamp: a plain refresh would call it current.
monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True)
assert refresh_dependencies(root) == "restored all, messaging, otlp"
assert calls == [(["all", "messaging", "otlp"], {"explicit": True})]
def test_refresh_leaves_a_complete_selection_alone(install, monkeypatch):
import pm.client
import pm.install
from pm.environments import runtime_facts_path
from pm.lock import Facts
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"])
Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "fal", "messaging"])
monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected"))
monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True)
assert refresh_dependencies(root) == "current"
def test_offline_refresh_keeps_baked_extras_for_the_next_lazy_install(install, monkeypatch):
import pm.client
import pm.install
from pm.environments import runtime_facts_path
from pm.install import _target_selection
from pm.lock import Facts
from pm.packages import Venv
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging", "otlp"])
Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["fal"])
def offline(*args, **kwargs):
raise RuntimeError("uv sync exited 1: no network")
monkeypatch.setattr(pm.client, "sync_venv", offline)
monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True)
assert refresh_dependencies(root) == "fallback"
fact = Facts(runtime_facts_path(root)).get("venv")
assert fact is not None and fact["extras"] == ["fal", "all", "messaging", "otlp"]
assert "environment" not in fact # the image environment boots
# A lazy install before the next boot must not drop the baked extras again.
enabled, _, _ = _target_selection(Venv(root), fact, extras=["fal"], inputs={"plugin_dirs": []},
repair=False, shipped=None, frozen=None)
assert enabled == ["all", "fal", "messaging", "otlp"]
@pytest.mark.parametrize("venv", [{"stamp": "s", "extras": "fal"}, {"stamp": "s", "extras": None},
{"stamp": "s", "extras": [1]}, {"stamp": "s"},
{"extras": ["fal"]}, {"stamp": "", "extras": ["fal"]},
{"stamp": 7, "extras": ["fal"]}])
def test_refresh_refuses_an_invalid_recorded_selection(install, monkeypatch, venv):
import json
import pm.client
from pm.environments import runtime_facts_path
from pm.lock import Facts
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "messaging"])
path = runtime_facts_path(root)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps({"schema": 1, "packages": {"venv": venv}}), encoding="utf-8")
monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("must not sync over a corrupt fact"))
with pytest.raises(ValueError, match="invalid recorded dependency state"):
refresh_dependencies(root)
def test_refresh_matches_recorded_extras_by_pep685_name(install, monkeypatch):
import pm.client
import pm.install
from pm.environments import runtime_facts_path
from pm.lock import Facts
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "azure-identity"])
Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all", "Azure_Identity", "fal"])
monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected"))
monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True)
assert refresh_dependencies(root) == "current"
def test_refresh_ignores_baked_extras_the_source_no_longer_declares(install, monkeypatch):
import pm.client
import pm.install
from pm.environments import runtime_facts_path
from pm.lock import Facts
from pm.recovery import refresh_dependencies
root, store = install
Facts(store / "facts.json").record_state("venv", "image", ["all", "hindsight"])
Facts(runtime_facts_path(root)).record_state("venv", "lazy", ["all"])
monkeypatch.setattr(pm.client, "sync_venv", lambda *a, **k: pytest.fail("no rebuild expected"))
monkeypatch.setattr(pm.install, "venv_is_current", lambda **kwargs: True)
assert refresh_dependencies(root) == "current"