From f2db4349e873224806d4652c21f1051b668f2dd3 Mon Sep 17 00:00:00 2001 From: ethernet Date: Wed, 9 Sep 2026 21:47:54 -0400 Subject: [PATCH] fix(pm): inventory features in the staged interpreter The builder's imported modules could mark an empty dependency tree as installed. Probe every anchor in one isolated target process. Require all anchors for a multi-module extra, and process only the selected tree's .pth files so editable packages retain their launch behavior. The native builder passes its staged Python explicitly and stops before manifest publication when the inventory fails. Correct the Hindsight and Teams anchors using the namespaces in their locked wheels. Verified: 31 tests passed, 3 host skips. A real target child records its identity, and a caller mutation back to the builder Python fails the regression. Both downloaded SDK wheels match uv.lock and pass inventory and availability checks. Ruff and added-comment checks passed. No full native package or signing run is claimed. --- pm/extras.py | 4 +- pm/features.py | 83 ++++++++++++++++------------- scripts/bundles/native.py | 12 +++-- tests/pm/test_extras.py | 10 ++-- tests/pm/test_feature_inventory.py | 74 +++++++++++++++++++++++++ tests/pm/test_features.py | 4 +- tests/scripts/test_bundle_native.py | 42 +++++++++++++-- 7 files changed, 177 insertions(+), 52 deletions(-) create mode 100644 tests/pm/test_feature_inventory.py diff --git a/pm/extras.py b/pm/extras.py index 448e0c7120..c86d11a084 100644 --- a/pm/extras.py +++ b/pm/extras.py @@ -34,7 +34,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = { "wake-porcupine": "pvporcupine", "fal": "fal_client", "honcho": "honcho", - "hindsight": "hindsight", + "hindsight": "hindsight_client", "supermemory": "supermemory", "mem0": "mem0", "messaging": "telegram", @@ -45,7 +45,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = { "dingtalk": "dingtalk_stream", "feishu": "lark_oapi", "wecom": "defusedxml", - "teams": "microsoft.teams.apps", + "teams": "microsoft_teams.apps", "modal": "modal", "daytona": "daytona", "vercel": "vercel", diff --git a/pm/features.py b/pm/features.py index 51b274f293..2539d3dc29 100644 --- a/pm/features.py +++ b/pm/features.py @@ -20,6 +20,10 @@ from typing import Optional FEATURES_FILENAME = "enabled-features.json" +class FeatureProbeError(RuntimeError): + """A failed inventory must not become an empty feature list.""" + + def features_path(base_dir: Optional[Path] = None) -> Path: """Where the enabled-features file lives. In a bundle, the payload root (beside manifest.json — bundle-written, sealed-shipped). At @@ -68,44 +72,51 @@ def declared_extras(repo_dir: Path) -> list[str]: return sorted(data.get("project", {}).get("optional-dependencies", {})) -def installed_extras(repo_dir: Path, venv_dir: Path) -> list[str]: - """The extras that ACTUALLY installed on this target: every declared - extra whose pm anchor import resolves in the venv. Markers-gated - extras show up as missing anchors — the honest per-target record of - what `uv sync --all-extras` put on THIS machine.""" - from pm.extras import ANCHORS +def installed_extras(repo_dir: Path, venv_dir: Path, *, python_exe: Path) -> list[str]: + """Inventory every required anchor in one isolated target process. - installed: list[str] = [] - for extra in declared_extras(repo_dir): - anchor = ANCHORS.get(extra, extra.replace("-", "_")) - if isinstance(anchor, str): - anchor = (anchor,) - if all(_importable_in(anchor_member, venv_dir) for anchor_member in anchor): - installed.append(extra) - return sorted(installed) + The staged interpreter owns the site layout. Only the selected tree's + .pth files are processed, so editable packages keep their launch behavior. + """ + import subprocess + from pm.extras import _anchors -def _importable_in(anchor: str, venv_dir: Path) -> bool: - """Does the anchor import resolve inside venv_dir's site-packages?""" - import importlib.util - import sys - import sysconfig - from pathlib import Path as _P - - sites = set() - pure = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("purelib") - if pure: - sites.add(_P(pure)) - plat = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("platlib") - if plat: - sites.add(_P(plat)) - - saved = list(sys.path) - try: - sys.path = [str(s) for s in sites] + required = {extra: _anchors(extra) for extra in declared_extras(repo_dir)} + anchors = sorted({anchor for group in required.values() for anchor in group}) + probe = """ +import contextlib, importlib.util, json, os, site, sys, sysconfig +base, anchors = sys.argv[1], json.loads(sys.argv[2]) +paths = sysconfig.get_paths(vars={"base": base, "platbase": base}) +sites = dict.fromkeys(paths[key] for key in ("purelib", "platlib")) +if not any(os.path.isdir(path) for path in sites): + raise RuntimeError("target dependency tree has no site-packages") +result = {} +with contextlib.redirect_stdout(sys.stderr): + for path in sites: + site.addsitedir(path) + for anchor in anchors: try: - return importlib.util.find_spec(anchor) is not None + result[anchor] = importlib.util.find_spec(anchor) is not None except (ImportError, ValueError): - return False - finally: - sys.path = saved + result[anchor] = False +print(json.dumps(result)) +""" + try: + child = subprocess.run( + [str(python_exe), "-B", "-I", "-S", "-c", probe, + str(venv_dir.resolve()), json.dumps(anchors)], + cwd=repo_dir, capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=60, check=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + detail = (getattr(exc, "stderr", None) or str(exc)).strip() + raise FeatureProbeError(f"feature inventory failed on {python_exe}: {detail}") from exc + try: + resolved = json.loads(child.stdout) + except ValueError as exc: + raise FeatureProbeError("feature inventory returned invalid JSON") from exc + if (not isinstance(resolved, dict) or set(resolved) != set(anchors) + or any(type(value) is not bool for value in resolved.values())): + raise FeatureProbeError("feature inventory returned incomplete anchor results") + return sorted(extra for extra, group in required.items() if all(resolved[anchor] for anchor in group)) diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 36b5b792d9..bd4d00c40e 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -155,12 +155,14 @@ def _stage_native(args) -> int: return 1 print("✓ venv (relocatable, all extras, on the staged interpreter)") - # The frozen feature set: the EXACT extras that installed on this - # target (markers gate some off per-platform). This file is the - # lazy-off contract — pm sync never deviates from it. - from pm.features import installed_extras, write_features + # Inventory the staged interpreter before publishing the bundle contract. + from pm.features import FeatureProbeError, installed_extras, write_features - features = installed_extras(repo_dir, venv_dir) + try: + features = installed_extras(repo_dir, venv_dir, python_exe=python_bin) + except FeatureProbeError as exc: + print(f"✗ features: {exc}") + return 1 write_features(features, out) print(f"✓ enabled-features.json ({len(features)} extras recorded)") diff --git a/tests/pm/test_extras.py b/tests/pm/test_extras.py index 1bd3a87a5c..8c5d9e0886 100644 --- a/tests/pm/test_extras.py +++ b/tests/pm/test_extras.py @@ -76,9 +76,13 @@ def test_available_missing_module(): assert extras.available("no-such-extra-anywhere") is False -def test_available_counts_sys_modules_fakes(monkeypatch): - monkeypatch.setitem(sys.modules, "hindsight", SimpleNamespace()) - assert extras.available("hindsight") is True +@pytest.mark.parametrize(("extra", "module"), [ + ("hindsight", "hindsight_client"), + ("teams", "microsoft_teams.apps"), +]) +def test_available_counts_sys_modules_fakes(monkeypatch, extra, module): + monkeypatch.setitem(sys.modules, module, SimpleNamespace()) + assert extras.available(extra) is True def test_available_unknown_extra_uses_underscore_guess(monkeypatch): diff --git a/tests/pm/test_feature_inventory.py b/tests/pm/test_feature_inventory.py new file mode 100644 index 0000000000..4e33887117 --- /dev/null +++ b/tests/pm/test_feature_inventory.py @@ -0,0 +1,74 @@ +"""Feature inventory reads the selected dependency tree, not the builder.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import packaging +import pytest + +import pm.extras as extras +import pm.features as features +from hermes_cli.runtime_paths import site_packages + + +def test_inventory_uses_the_target_and_requires_every_anchor(tmp_path, monkeypatch): + target_root = tmp_path / "target" + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(target_root)], + capture_output=True, check=True, timeout=60, + ) + target = target_root / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + dependencies = tmp_path / "dependencies" + site = site_packages(dependencies) + site.mkdir(parents=True) + witness = tmp_path / "child.json" + package = site / "inventory_anchor" + package.mkdir() + (package / "__init__.py").write_text( + "import json, pathlib, sys\n" + f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n", + encoding="utf-8", + ) + (package / "present.py").write_text("VALUE = 'target'\n", encoding="utf-8") + (site / "one_part.py").write_text("", encoding="utf-8") + editable = tmp_path / "editable" + editable.mkdir() + (editable / "editable_anchor.py").write_text("", encoding="utf-8") + (site / "editable.pth").write_text(str(editable) + "\n", encoding="utf-8") + launches = tmp_path / "launches.jsonl" + (site / "launches.pth").write_text( + f"import json, pathlib, sys; p = pathlib.Path({str(launches)!r}); " + "text = p.read_text(encoding='utf-8') if p.exists() else ''; " + "p.write_text(text + json.dumps(sys.executable) + '\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + mapping = { + "complete": "inventory_anchor.present", + "partial": ("one_part", "absent_part"), + "host-only": "packaging", + "editable": "editable_anchor", + } + repo = tmp_path / "source" + repo.mkdir() + (repo / "pyproject.toml").write_text( + "[project.optional-dependencies]\n" + "".join(f'"{name}"=[]\n' for name in mapping), + encoding="utf-8", + ) + monkeypatch.setattr(extras, "ANCHORS", mapping) + before_path = list(sys.path) + before_env = dict(os.environ) + assert packaging.__file__ and "packaging" in sys.modules + result = features.installed_extras(repo, dependencies, python_exe=target) + assert result == ["complete", "editable"] + assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target + assert [Path(json.loads(line)) for line in launches.read_text(encoding="utf-8").splitlines()] == [target] + assert "inventory_anchor" not in sys.modules + assert sys.path == before_path + assert dict(os.environ) == before_env + + (site / "absent_part.py").write_text("", encoding="utf-8") + assert features.installed_extras(repo, dependencies, python_exe=target) == ["complete", "editable", "partial"] + with pytest.raises(features.FeatureProbeError, match="feature inventory failed"): + features.installed_extras(repo, dependencies, python_exe=tmp_path / "missing-python") diff --git a/tests/pm/test_features.py b/tests/pm/test_features.py index a347d46c05..be73001d69 100644 --- a/tests/pm/test_features.py +++ b/tests/pm/test_features.py @@ -46,6 +46,8 @@ def test_features_path_in_bundle_uses_payload_root(rooted): def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch): + import sys + repo = tmp_path / "repo" repo.mkdir() (repo / "pyproject.toml").write_text( @@ -70,7 +72,7 @@ def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch): "ANCHORS", {**extras_mod.ANCHORS, "present": "somepkg", "absent": "missingmod"}, ) - got = feats.installed_extras(repo, venv) + got = feats.installed_extras(repo, venv, python_exe=Path(sys.executable)) assert "present" in got assert "absent" not in got diff --git a/tests/scripts/test_bundle_native.py b/tests/scripts/test_bundle_native.py index 854187b579..0ec8cb113c 100644 --- a/tests/scripts/test_bundle_native.py +++ b/tests/scripts/test_bundle_native.py @@ -17,9 +17,15 @@ from scripts.bundles import native def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_path, monkeypatch): + from hermes_cli.runtime_paths import site_packages + + interpreter = tmp_path / "staged-python" + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(interpreter)], + capture_output=True, check=True, timeout=60) + target_python = interpreter / ("Scripts/python.exe" if os.name == "nt" else "bin/python") repo = tmp_path / "repo" repo.mkdir() - (repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[tool.uv]\npackage=false\n', encoding="utf-8") + (repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[project.optional-dependencies]\npayloadtest=[]\n[tool.uv]\npackage=false\n', encoding="utf-8") uv = shutil.which("uv") assert uv, "native bundle test requires uv" env = {**os.environ, "UV_OFFLINE": "1", "UV_PYTHON_DOWNLOADS": "never", "UV_CACHE_DIR": str(tmp_path / "cache")} @@ -31,21 +37,37 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat monkeypatch.setattr("pm.paths.repo_root", lambda: repo) monkeypatch.setattr(native, "_bundle_package_names", lambda: []) monkeypatch.setattr(native, "_install_names", lambda names: 0) - monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: Path(sys.executable).parent)) + monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: target_python.parent)) monkeypatch.setattr(native, "_facts", lambda: SimpleNamespace(get=lambda _: {"entry": "python"}, entries_in_use=lambda: [])) - monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: Path(sys.executable))) + monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: target_python)) monkeypatch.setattr(native, "pm_uv", lambda: (uv, dict(env))) monkeypatch.setattr(native, "_arch_guard", lambda store: []) monkeypatch.setattr("scripts.bundles.payload.relativize_links", lambda root: 0) - monkeypatch.setattr("pm.features.installed_extras", lambda *args: []) + monkeypatch.setattr("pm.extras.ANCHORS", {"payloadtest": "bundle_probe.present"}) monkeypatch.setattr("pm.packages.uv_cache_dir", lambda: tmp_path / "empty-cache") real_run = native._run_live calls = [] + witness = tmp_path / "inventory-python.json" + fail_inventory = False def child(argv, *, cwd, env): calls.append(argv[1]) assert not (output / "manifest.json").exists() - return real_run(argv, cwd=cwd, env=env) + result = real_run(argv, cwd=cwd, env=env) + if argv[1] == "sync" and result[0] == 0: + site = site_packages(output / "venv") + if fail_inventory: + shutil.rmtree(site) + else: + package = site / "bundle_probe" + package.mkdir() + (package / "__init__.py").write_text( + "import json, pathlib, sys\n" + f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n", + encoding="utf-8", + ) + (package / "present.py").write_text("", encoding="utf-8") + return result monkeypatch.setattr(native, "_run_live", child) monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "original")) @@ -53,6 +75,16 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat assert calls == ["venv", "sync"] assert (output / "hermes-agent/pyproject.toml").is_file() assert json.loads((output / "manifest.json").read_text())["repo"] == "hermes-agent" + feature_file = output / "enabled-features.json" + assert json.loads(feature_file.read_text(encoding="utf-8"))["extras"] == ["payloadtest"] + assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target_python + assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") + + before = feature_file.read_bytes() + fail_inventory = True + assert native.stage_native(SimpleNamespace(out=str(output), ref="HEAD")) == 1 + assert not (output / "manifest.json").exists() + assert feature_file.read_bytes() == before assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") monkeypatch.setattr(native, "_run_live", lambda *a, **kw: (1, "injected failure"))