fix(pm): inventory features in the staged interpreter
The builder's imported modules could mark an empty dependency tree as installed. Probe every anchor in one isolated target process. Require all anchors for a multi-module extra, and process only the selected tree's .pth files so editable packages retain their launch behavior. The native builder passes its staged Python explicitly and stops before manifest publication when the inventory fails. Correct the Hindsight and Teams anchors using the namespaces in their locked wheels. Verified: 31 tests passed, 3 host skips. A real target child records its identity, and a caller mutation back to the builder Python fails the regression. Both downloaded SDK wheels match uv.lock and pass inventory and availability checks. Ruff and added-comment checks passed. No full native package or signing run is claimed.
This commit is contained in:
ethernet
@@ -34,7 +34,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = {
|
||||
"wake-porcupine": "pvporcupine",
|
||||
"fal": "fal_client",
|
||||
"honcho": "honcho",
|
||||
"hindsight": "hindsight",
|
||||
"hindsight": "hindsight_client",
|
||||
"supermemory": "supermemory",
|
||||
"mem0": "mem0",
|
||||
"messaging": "telegram",
|
||||
@@ -45,7 +45,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = {
|
||||
"dingtalk": "dingtalk_stream",
|
||||
"feishu": "lark_oapi",
|
||||
"wecom": "defusedxml",
|
||||
"teams": "microsoft.teams.apps",
|
||||
"teams": "microsoft_teams.apps",
|
||||
"modal": "modal",
|
||||
"daytona": "daytona",
|
||||
"vercel": "vercel",
|
||||
|
||||
@@ -20,6 +20,10 @@ from typing import Optional
|
||||
FEATURES_FILENAME = "enabled-features.json"
|
||||
|
||||
|
||||
class FeatureProbeError(RuntimeError):
|
||||
"""A failed inventory must not become an empty feature list."""
|
||||
|
||||
|
||||
def features_path(base_dir: Optional[Path] = None) -> Path:
|
||||
"""Where the enabled-features file lives. In a bundle, the payload
|
||||
root (beside manifest.json — bundle-written, sealed-shipped). At
|
||||
@@ -68,44 +72,51 @@ def declared_extras(repo_dir: Path) -> list[str]:
|
||||
return sorted(data.get("project", {}).get("optional-dependencies", {}))
|
||||
|
||||
|
||||
def installed_extras(repo_dir: Path, venv_dir: Path) -> list[str]:
|
||||
"""The extras that ACTUALLY installed on this target: every declared
|
||||
extra whose pm anchor import resolves in the venv. Markers-gated
|
||||
extras show up as missing anchors — the honest per-target record of
|
||||
what `uv sync --all-extras` put on THIS machine."""
|
||||
from pm.extras import ANCHORS
|
||||
def installed_extras(repo_dir: Path, venv_dir: Path, *, python_exe: Path) -> list[str]:
|
||||
"""Inventory every required anchor in one isolated target process.
|
||||
|
||||
installed: list[str] = []
|
||||
for extra in declared_extras(repo_dir):
|
||||
anchor = ANCHORS.get(extra, extra.replace("-", "_"))
|
||||
if isinstance(anchor, str):
|
||||
anchor = (anchor,)
|
||||
if all(_importable_in(anchor_member, venv_dir) for anchor_member in anchor):
|
||||
installed.append(extra)
|
||||
return sorted(installed)
|
||||
The staged interpreter owns the site layout. Only the selected tree's
|
||||
.pth files are processed, so editable packages keep their launch behavior.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
from pm.extras import _anchors
|
||||
|
||||
def _importable_in(anchor: str, venv_dir: Path) -> bool:
|
||||
"""Does the anchor import resolve inside venv_dir's site-packages?"""
|
||||
import importlib.util
|
||||
import sys
|
||||
import sysconfig
|
||||
from pathlib import Path as _P
|
||||
|
||||
sites = set()
|
||||
pure = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("purelib")
|
||||
if pure:
|
||||
sites.add(_P(pure))
|
||||
plat = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("platlib")
|
||||
if plat:
|
||||
sites.add(_P(plat))
|
||||
|
||||
saved = list(sys.path)
|
||||
try:
|
||||
sys.path = [str(s) for s in sites]
|
||||
required = {extra: _anchors(extra) for extra in declared_extras(repo_dir)}
|
||||
anchors = sorted({anchor for group in required.values() for anchor in group})
|
||||
probe = """
|
||||
import contextlib, importlib.util, json, os, site, sys, sysconfig
|
||||
base, anchors = sys.argv[1], json.loads(sys.argv[2])
|
||||
paths = sysconfig.get_paths(vars={"base": base, "platbase": base})
|
||||
sites = dict.fromkeys(paths[key] for key in ("purelib", "platlib"))
|
||||
if not any(os.path.isdir(path) for path in sites):
|
||||
raise RuntimeError("target dependency tree has no site-packages")
|
||||
result = {}
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
for path in sites:
|
||||
site.addsitedir(path)
|
||||
for anchor in anchors:
|
||||
try:
|
||||
return importlib.util.find_spec(anchor) is not None
|
||||
result[anchor] = importlib.util.find_spec(anchor) is not None
|
||||
except (ImportError, ValueError):
|
||||
return False
|
||||
finally:
|
||||
sys.path = saved
|
||||
result[anchor] = False
|
||||
print(json.dumps(result))
|
||||
"""
|
||||
try:
|
||||
child = subprocess.run(
|
||||
[str(python_exe), "-B", "-I", "-S", "-c", probe,
|
||||
str(venv_dir.resolve()), json.dumps(anchors)],
|
||||
cwd=repo_dir, capture_output=True, text=True, encoding="utf-8",
|
||||
errors="replace", timeout=60, check=True,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
detail = (getattr(exc, "stderr", None) or str(exc)).strip()
|
||||
raise FeatureProbeError(f"feature inventory failed on {python_exe}: {detail}") from exc
|
||||
try:
|
||||
resolved = json.loads(child.stdout)
|
||||
except ValueError as exc:
|
||||
raise FeatureProbeError("feature inventory returned invalid JSON") from exc
|
||||
if (not isinstance(resolved, dict) or set(resolved) != set(anchors)
|
||||
or any(type(value) is not bool for value in resolved.values())):
|
||||
raise FeatureProbeError("feature inventory returned incomplete anchor results")
|
||||
return sorted(extra for extra, group in required.items() if all(resolved[anchor] for anchor in group))
|
||||
|
||||
@@ -155,12 +155,14 @@ def _stage_native(args) -> int:
|
||||
return 1
|
||||
print("✓ venv (relocatable, all extras, on the staged interpreter)")
|
||||
|
||||
# The frozen feature set: the EXACT extras that installed on this
|
||||
# target (markers gate some off per-platform). This file is the
|
||||
# lazy-off contract — pm sync never deviates from it.
|
||||
from pm.features import installed_extras, write_features
|
||||
# Inventory the staged interpreter before publishing the bundle contract.
|
||||
from pm.features import FeatureProbeError, installed_extras, write_features
|
||||
|
||||
features = installed_extras(repo_dir, venv_dir)
|
||||
try:
|
||||
features = installed_extras(repo_dir, venv_dir, python_exe=python_bin)
|
||||
except FeatureProbeError as exc:
|
||||
print(f"✗ features: {exc}")
|
||||
return 1
|
||||
write_features(features, out)
|
||||
print(f"✓ enabled-features.json ({len(features)} extras recorded)")
|
||||
|
||||
|
||||
@@ -76,9 +76,13 @@ def test_available_missing_module():
|
||||
assert extras.available("no-such-extra-anywhere") is False
|
||||
|
||||
|
||||
def test_available_counts_sys_modules_fakes(monkeypatch):
|
||||
monkeypatch.setitem(sys.modules, "hindsight", SimpleNamespace())
|
||||
assert extras.available("hindsight") is True
|
||||
@pytest.mark.parametrize(("extra", "module"), [
|
||||
("hindsight", "hindsight_client"),
|
||||
("teams", "microsoft_teams.apps"),
|
||||
])
|
||||
def test_available_counts_sys_modules_fakes(monkeypatch, extra, module):
|
||||
monkeypatch.setitem(sys.modules, module, SimpleNamespace())
|
||||
assert extras.available(extra) is True
|
||||
|
||||
|
||||
def test_available_unknown_extra_uses_underscore_guess(monkeypatch):
|
||||
|
||||
74
tests/pm/test_feature_inventory.py
Normal file
74
tests/pm/test_feature_inventory.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""Feature inventory reads the selected dependency tree, not the builder."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import packaging
|
||||
import pytest
|
||||
|
||||
import pm.extras as extras
|
||||
import pm.features as features
|
||||
from hermes_cli.runtime_paths import site_packages
|
||||
|
||||
|
||||
def test_inventory_uses_the_target_and_requires_every_anchor(tmp_path, monkeypatch):
|
||||
target_root = tmp_path / "target"
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "venv", "--without-pip", str(target_root)],
|
||||
capture_output=True, check=True, timeout=60,
|
||||
)
|
||||
target = target_root / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
||||
dependencies = tmp_path / "dependencies"
|
||||
site = site_packages(dependencies)
|
||||
site.mkdir(parents=True)
|
||||
witness = tmp_path / "child.json"
|
||||
package = site / "inventory_anchor"
|
||||
package.mkdir()
|
||||
(package / "__init__.py").write_text(
|
||||
"import json, pathlib, sys\n"
|
||||
f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(package / "present.py").write_text("VALUE = 'target'\n", encoding="utf-8")
|
||||
(site / "one_part.py").write_text("", encoding="utf-8")
|
||||
editable = tmp_path / "editable"
|
||||
editable.mkdir()
|
||||
(editable / "editable_anchor.py").write_text("", encoding="utf-8")
|
||||
(site / "editable.pth").write_text(str(editable) + "\n", encoding="utf-8")
|
||||
launches = tmp_path / "launches.jsonl"
|
||||
(site / "launches.pth").write_text(
|
||||
f"import json, pathlib, sys; p = pathlib.Path({str(launches)!r}); "
|
||||
"text = p.read_text(encoding='utf-8') if p.exists() else ''; "
|
||||
"p.write_text(text + json.dumps(sys.executable) + '\\n', encoding='utf-8')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
mapping = {
|
||||
"complete": "inventory_anchor.present",
|
||||
"partial": ("one_part", "absent_part"),
|
||||
"host-only": "packaging",
|
||||
"editable": "editable_anchor",
|
||||
}
|
||||
repo = tmp_path / "source"
|
||||
repo.mkdir()
|
||||
(repo / "pyproject.toml").write_text(
|
||||
"[project.optional-dependencies]\n" + "".join(f'"{name}"=[]\n' for name in mapping),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(extras, "ANCHORS", mapping)
|
||||
before_path = list(sys.path)
|
||||
before_env = dict(os.environ)
|
||||
assert packaging.__file__ and "packaging" in sys.modules
|
||||
result = features.installed_extras(repo, dependencies, python_exe=target)
|
||||
assert result == ["complete", "editable"]
|
||||
assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target
|
||||
assert [Path(json.loads(line)) for line in launches.read_text(encoding="utf-8").splitlines()] == [target]
|
||||
assert "inventory_anchor" not in sys.modules
|
||||
assert sys.path == before_path
|
||||
assert dict(os.environ) == before_env
|
||||
|
||||
(site / "absent_part.py").write_text("", encoding="utf-8")
|
||||
assert features.installed_extras(repo, dependencies, python_exe=target) == ["complete", "editable", "partial"]
|
||||
with pytest.raises(features.FeatureProbeError, match="feature inventory failed"):
|
||||
features.installed_extras(repo, dependencies, python_exe=tmp_path / "missing-python")
|
||||
@@ -46,6 +46,8 @@ def test_features_path_in_bundle_uses_payload_root(rooted):
|
||||
|
||||
|
||||
def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch):
|
||||
import sys
|
||||
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "pyproject.toml").write_text(
|
||||
@@ -70,7 +72,7 @@ def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch):
|
||||
"ANCHORS",
|
||||
{**extras_mod.ANCHORS, "present": "somepkg", "absent": "missingmod"},
|
||||
)
|
||||
got = feats.installed_extras(repo, venv)
|
||||
got = feats.installed_extras(repo, venv, python_exe=Path(sys.executable))
|
||||
assert "present" in got
|
||||
assert "absent" not in got
|
||||
|
||||
|
||||
@@ -17,9 +17,15 @@ from scripts.bundles import native
|
||||
|
||||
|
||||
def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_path, monkeypatch):
|
||||
from hermes_cli.runtime_paths import site_packages
|
||||
|
||||
interpreter = tmp_path / "staged-python"
|
||||
subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(interpreter)],
|
||||
capture_output=True, check=True, timeout=60)
|
||||
target_python = interpreter / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
(repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[tool.uv]\npackage=false\n', encoding="utf-8")
|
||||
(repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[project.optional-dependencies]\npayloadtest=[]\n[tool.uv]\npackage=false\n', encoding="utf-8")
|
||||
uv = shutil.which("uv")
|
||||
assert uv, "native bundle test requires uv"
|
||||
env = {**os.environ, "UV_OFFLINE": "1", "UV_PYTHON_DOWNLOADS": "never", "UV_CACHE_DIR": str(tmp_path / "cache")}
|
||||
@@ -31,21 +37,37 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat
|
||||
monkeypatch.setattr("pm.paths.repo_root", lambda: repo)
|
||||
monkeypatch.setattr(native, "_bundle_package_names", lambda: [])
|
||||
monkeypatch.setattr(native, "_install_names", lambda names: 0)
|
||||
monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: Path(sys.executable).parent))
|
||||
monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: target_python.parent))
|
||||
monkeypatch.setattr(native, "_facts", lambda: SimpleNamespace(get=lambda _: {"entry": "python"}, entries_in_use=lambda: []))
|
||||
monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: Path(sys.executable)))
|
||||
monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: target_python))
|
||||
monkeypatch.setattr(native, "pm_uv", lambda: (uv, dict(env)))
|
||||
monkeypatch.setattr(native, "_arch_guard", lambda store: [])
|
||||
monkeypatch.setattr("scripts.bundles.payload.relativize_links", lambda root: 0)
|
||||
monkeypatch.setattr("pm.features.installed_extras", lambda *args: [])
|
||||
monkeypatch.setattr("pm.extras.ANCHORS", {"payloadtest": "bundle_probe.present"})
|
||||
monkeypatch.setattr("pm.packages.uv_cache_dir", lambda: tmp_path / "empty-cache")
|
||||
real_run = native._run_live
|
||||
calls = []
|
||||
witness = tmp_path / "inventory-python.json"
|
||||
fail_inventory = False
|
||||
|
||||
def child(argv, *, cwd, env):
|
||||
calls.append(argv[1])
|
||||
assert not (output / "manifest.json").exists()
|
||||
return real_run(argv, cwd=cwd, env=env)
|
||||
result = real_run(argv, cwd=cwd, env=env)
|
||||
if argv[1] == "sync" and result[0] == 0:
|
||||
site = site_packages(output / "venv")
|
||||
if fail_inventory:
|
||||
shutil.rmtree(site)
|
||||
else:
|
||||
package = site / "bundle_probe"
|
||||
package.mkdir()
|
||||
(package / "__init__.py").write_text(
|
||||
"import json, pathlib, sys\n"
|
||||
f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(package / "present.py").write_text("", encoding="utf-8")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(native, "_run_live", child)
|
||||
monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "original"))
|
||||
@@ -53,6 +75,16 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat
|
||||
assert calls == ["venv", "sync"]
|
||||
assert (output / "hermes-agent/pyproject.toml").is_file()
|
||||
assert json.loads((output / "manifest.json").read_text())["repo"] == "hermes-agent"
|
||||
feature_file = output / "enabled-features.json"
|
||||
assert json.loads(feature_file.read_text(encoding="utf-8"))["extras"] == ["payloadtest"]
|
||||
assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target_python
|
||||
assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original")
|
||||
|
||||
before = feature_file.read_bytes()
|
||||
fail_inventory = True
|
||||
assert native.stage_native(SimpleNamespace(out=str(output), ref="HEAD")) == 1
|
||||
assert not (output / "manifest.json").exists()
|
||||
assert feature_file.read_bytes() == before
|
||||
assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original")
|
||||
|
||||
monkeypatch.setattr(native, "_run_live", lambda *a, **kw: (1, "injected failure"))
|
||||
|
||||
Reference in New Issue
Block a user