diff --git a/pm/extras.py b/pm/extras.py index 448e0c7120..c86d11a084 100644 --- a/pm/extras.py +++ b/pm/extras.py @@ -34,7 +34,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = { "wake-porcupine": "pvporcupine", "fal": "fal_client", "honcho": "honcho", - "hindsight": "hindsight", + "hindsight": "hindsight_client", "supermemory": "supermemory", "mem0": "mem0", "messaging": "telegram", @@ -45,7 +45,7 @@ ANCHORS: dict[str, str | tuple[str, ...]] = { "dingtalk": "dingtalk_stream", "feishu": "lark_oapi", "wecom": "defusedxml", - "teams": "microsoft.teams.apps", + "teams": "microsoft_teams.apps", "modal": "modal", "daytona": "daytona", "vercel": "vercel", diff --git a/pm/features.py b/pm/features.py index 51b274f293..2539d3dc29 100644 --- a/pm/features.py +++ b/pm/features.py @@ -20,6 +20,10 @@ from typing import Optional FEATURES_FILENAME = "enabled-features.json" +class FeatureProbeError(RuntimeError): + """A failed inventory must not become an empty feature list.""" + + def features_path(base_dir: Optional[Path] = None) -> Path: """Where the enabled-features file lives. In a bundle, the payload root (beside manifest.json — bundle-written, sealed-shipped). At @@ -68,44 +72,51 @@ def declared_extras(repo_dir: Path) -> list[str]: return sorted(data.get("project", {}).get("optional-dependencies", {})) -def installed_extras(repo_dir: Path, venv_dir: Path) -> list[str]: - """The extras that ACTUALLY installed on this target: every declared - extra whose pm anchor import resolves in the venv. Markers-gated - extras show up as missing anchors — the honest per-target record of - what `uv sync --all-extras` put on THIS machine.""" - from pm.extras import ANCHORS +def installed_extras(repo_dir: Path, venv_dir: Path, *, python_exe: Path) -> list[str]: + """Inventory every required anchor in one isolated target process. - installed: list[str] = [] - for extra in declared_extras(repo_dir): - anchor = ANCHORS.get(extra, extra.replace("-", "_")) - if isinstance(anchor, str): - anchor = (anchor,) - if all(_importable_in(anchor_member, venv_dir) for anchor_member in anchor): - installed.append(extra) - return sorted(installed) + The staged interpreter owns the site layout. Only the selected tree's + .pth files are processed, so editable packages keep their launch behavior. + """ + import subprocess + from pm.extras import _anchors -def _importable_in(anchor: str, venv_dir: Path) -> bool: - """Does the anchor import resolve inside venv_dir's site-packages?""" - import importlib.util - import sys - import sysconfig - from pathlib import Path as _P - - sites = set() - pure = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("purelib") - if pure: - sites.add(_P(pure)) - plat = sysconfig.get_paths(vars={"base": str(venv_dir)}).get("platlib") - if plat: - sites.add(_P(plat)) - - saved = list(sys.path) - try: - sys.path = [str(s) for s in sites] + required = {extra: _anchors(extra) for extra in declared_extras(repo_dir)} + anchors = sorted({anchor for group in required.values() for anchor in group}) + probe = """ +import contextlib, importlib.util, json, os, site, sys, sysconfig +base, anchors = sys.argv[1], json.loads(sys.argv[2]) +paths = sysconfig.get_paths(vars={"base": base, "platbase": base}) +sites = dict.fromkeys(paths[key] for key in ("purelib", "platlib")) +if not any(os.path.isdir(path) for path in sites): + raise RuntimeError("target dependency tree has no site-packages") +result = {} +with contextlib.redirect_stdout(sys.stderr): + for path in sites: + site.addsitedir(path) + for anchor in anchors: try: - return importlib.util.find_spec(anchor) is not None + result[anchor] = importlib.util.find_spec(anchor) is not None except (ImportError, ValueError): - return False - finally: - sys.path = saved + result[anchor] = False +print(json.dumps(result)) +""" + try: + child = subprocess.run( + [str(python_exe), "-B", "-I", "-S", "-c", probe, + str(venv_dir.resolve()), json.dumps(anchors)], + cwd=repo_dir, capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=60, check=True, + ) + except (OSError, subprocess.SubprocessError) as exc: + detail = (getattr(exc, "stderr", None) or str(exc)).strip() + raise FeatureProbeError(f"feature inventory failed on {python_exe}: {detail}") from exc + try: + resolved = json.loads(child.stdout) + except ValueError as exc: + raise FeatureProbeError("feature inventory returned invalid JSON") from exc + if (not isinstance(resolved, dict) or set(resolved) != set(anchors) + or any(type(value) is not bool for value in resolved.values())): + raise FeatureProbeError("feature inventory returned incomplete anchor results") + return sorted(extra for extra, group in required.items() if all(resolved[anchor] for anchor in group)) diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 36b5b792d9..bd4d00c40e 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -155,12 +155,14 @@ def _stage_native(args) -> int: return 1 print("✓ venv (relocatable, all extras, on the staged interpreter)") - # The frozen feature set: the EXACT extras that installed on this - # target (markers gate some off per-platform). This file is the - # lazy-off contract — pm sync never deviates from it. - from pm.features import installed_extras, write_features + # Inventory the staged interpreter before publishing the bundle contract. + from pm.features import FeatureProbeError, installed_extras, write_features - features = installed_extras(repo_dir, venv_dir) + try: + features = installed_extras(repo_dir, venv_dir, python_exe=python_bin) + except FeatureProbeError as exc: + print(f"✗ features: {exc}") + return 1 write_features(features, out) print(f"✓ enabled-features.json ({len(features)} extras recorded)") diff --git a/tests/pm/test_extras.py b/tests/pm/test_extras.py index 1bd3a87a5c..8c5d9e0886 100644 --- a/tests/pm/test_extras.py +++ b/tests/pm/test_extras.py @@ -76,9 +76,13 @@ def test_available_missing_module(): assert extras.available("no-such-extra-anywhere") is False -def test_available_counts_sys_modules_fakes(monkeypatch): - monkeypatch.setitem(sys.modules, "hindsight", SimpleNamespace()) - assert extras.available("hindsight") is True +@pytest.mark.parametrize(("extra", "module"), [ + ("hindsight", "hindsight_client"), + ("teams", "microsoft_teams.apps"), +]) +def test_available_counts_sys_modules_fakes(monkeypatch, extra, module): + monkeypatch.setitem(sys.modules, module, SimpleNamespace()) + assert extras.available(extra) is True def test_available_unknown_extra_uses_underscore_guess(monkeypatch): diff --git a/tests/pm/test_feature_inventory.py b/tests/pm/test_feature_inventory.py new file mode 100644 index 0000000000..4e33887117 --- /dev/null +++ b/tests/pm/test_feature_inventory.py @@ -0,0 +1,74 @@ +"""Feature inventory reads the selected dependency tree, not the builder.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import packaging +import pytest + +import pm.extras as extras +import pm.features as features +from hermes_cli.runtime_paths import site_packages + + +def test_inventory_uses_the_target_and_requires_every_anchor(tmp_path, monkeypatch): + target_root = tmp_path / "target" + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(target_root)], + capture_output=True, check=True, timeout=60, + ) + target = target_root / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + dependencies = tmp_path / "dependencies" + site = site_packages(dependencies) + site.mkdir(parents=True) + witness = tmp_path / "child.json" + package = site / "inventory_anchor" + package.mkdir() + (package / "__init__.py").write_text( + "import json, pathlib, sys\n" + f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n", + encoding="utf-8", + ) + (package / "present.py").write_text("VALUE = 'target'\n", encoding="utf-8") + (site / "one_part.py").write_text("", encoding="utf-8") + editable = tmp_path / "editable" + editable.mkdir() + (editable / "editable_anchor.py").write_text("", encoding="utf-8") + (site / "editable.pth").write_text(str(editable) + "\n", encoding="utf-8") + launches = tmp_path / "launches.jsonl" + (site / "launches.pth").write_text( + f"import json, pathlib, sys; p = pathlib.Path({str(launches)!r}); " + "text = p.read_text(encoding='utf-8') if p.exists() else ''; " + "p.write_text(text + json.dumps(sys.executable) + '\\n', encoding='utf-8')\n", + encoding="utf-8", + ) + mapping = { + "complete": "inventory_anchor.present", + "partial": ("one_part", "absent_part"), + "host-only": "packaging", + "editable": "editable_anchor", + } + repo = tmp_path / "source" + repo.mkdir() + (repo / "pyproject.toml").write_text( + "[project.optional-dependencies]\n" + "".join(f'"{name}"=[]\n' for name in mapping), + encoding="utf-8", + ) + monkeypatch.setattr(extras, "ANCHORS", mapping) + before_path = list(sys.path) + before_env = dict(os.environ) + assert packaging.__file__ and "packaging" in sys.modules + result = features.installed_extras(repo, dependencies, python_exe=target) + assert result == ["complete", "editable"] + assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target + assert [Path(json.loads(line)) for line in launches.read_text(encoding="utf-8").splitlines()] == [target] + assert "inventory_anchor" not in sys.modules + assert sys.path == before_path + assert dict(os.environ) == before_env + + (site / "absent_part.py").write_text("", encoding="utf-8") + assert features.installed_extras(repo, dependencies, python_exe=target) == ["complete", "editable", "partial"] + with pytest.raises(features.FeatureProbeError, match="feature inventory failed"): + features.installed_extras(repo, dependencies, python_exe=tmp_path / "missing-python") diff --git a/tests/pm/test_features.py b/tests/pm/test_features.py index a347d46c05..be73001d69 100644 --- a/tests/pm/test_features.py +++ b/tests/pm/test_features.py @@ -46,6 +46,8 @@ def test_features_path_in_bundle_uses_payload_root(rooted): def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch): + import sys + repo = tmp_path / "repo" repo.mkdir() (repo / "pyproject.toml").write_text( @@ -70,7 +72,7 @@ def test_installed_extras_reports_only_anchor_resolved(tmp_path, monkeypatch): "ANCHORS", {**extras_mod.ANCHORS, "present": "somepkg", "absent": "missingmod"}, ) - got = feats.installed_extras(repo, venv) + got = feats.installed_extras(repo, venv, python_exe=Path(sys.executable)) assert "present" in got assert "absent" not in got diff --git a/tests/scripts/test_bundle_native.py b/tests/scripts/test_bundle_native.py index 854187b579..0ec8cb113c 100644 --- a/tests/scripts/test_bundle_native.py +++ b/tests/scripts/test_bundle_native.py @@ -17,9 +17,15 @@ from scripts.bundles import native def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_path, monkeypatch): + from hermes_cli.runtime_paths import site_packages + + interpreter = tmp_path / "staged-python" + subprocess.run([sys.executable, "-m", "venv", "--without-pip", str(interpreter)], + capture_output=True, check=True, timeout=60) + target_python = interpreter / ("Scripts/python.exe" if os.name == "nt" else "bin/python") repo = tmp_path / "repo" repo.mkdir() - (repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[tool.uv]\npackage=false\n', encoding="utf-8") + (repo / "pyproject.toml").write_text('[project]\nname="fixture"\nversion="1.0.0"\nrequires-python=">=3.11"\n[project.optional-dependencies]\npayloadtest=[]\n[tool.uv]\npackage=false\n', encoding="utf-8") uv = shutil.which("uv") assert uv, "native bundle test requires uv" env = {**os.environ, "UV_OFFLINE": "1", "UV_PYTHON_DOWNLOADS": "never", "UV_CACHE_DIR": str(tmp_path / "cache")} @@ -31,21 +37,37 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat monkeypatch.setattr("pm.paths.repo_root", lambda: repo) monkeypatch.setattr(native, "_bundle_package_names", lambda: []) monkeypatch.setattr(native, "_install_names", lambda names: 0) - monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: Path(sys.executable).parent)) + monkeypatch.setattr(native, "_store", lambda: SimpleNamespace(root=output / "tools", entry=lambda _: target_python.parent)) monkeypatch.setattr(native, "_facts", lambda: SimpleNamespace(get=lambda _: {"entry": "python"}, entries_in_use=lambda: [])) - monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: Path(sys.executable))) + monkeypatch.setattr(native, "get_package", lambda _: SimpleNamespace(binary=lambda *args: target_python)) monkeypatch.setattr(native, "pm_uv", lambda: (uv, dict(env))) monkeypatch.setattr(native, "_arch_guard", lambda store: []) monkeypatch.setattr("scripts.bundles.payload.relativize_links", lambda root: 0) - monkeypatch.setattr("pm.features.installed_extras", lambda *args: []) + monkeypatch.setattr("pm.extras.ANCHORS", {"payloadtest": "bundle_probe.present"}) monkeypatch.setattr("pm.packages.uv_cache_dir", lambda: tmp_path / "empty-cache") real_run = native._run_live calls = [] + witness = tmp_path / "inventory-python.json" + fail_inventory = False def child(argv, *, cwd, env): calls.append(argv[1]) assert not (output / "manifest.json").exists() - return real_run(argv, cwd=cwd, env=env) + result = real_run(argv, cwd=cwd, env=env) + if argv[1] == "sync" and result[0] == 0: + site = site_packages(output / "venv") + if fail_inventory: + shutil.rmtree(site) + else: + package = site / "bundle_probe" + package.mkdir() + (package / "__init__.py").write_text( + "import json, pathlib, sys\n" + f"pathlib.Path({str(witness)!r}).write_text(json.dumps(sys.executable), encoding='utf-8')\n", + encoding="utf-8", + ) + (package / "present.py").write_text("", encoding="utf-8") + return result monkeypatch.setattr(native, "_run_live", child) monkeypatch.setenv("HERMES_RUNTIME_DIR", str(tmp_path / "original")) @@ -53,6 +75,16 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat assert calls == ["venv", "sync"] assert (output / "hermes-agent/pyproject.toml").is_file() assert json.loads((output / "manifest.json").read_text())["repo"] == "hermes-agent" + feature_file = output / "enabled-features.json" + assert json.loads(feature_file.read_text(encoding="utf-8"))["extras"] == ["payloadtest"] + assert Path(json.loads(witness.read_text(encoding="utf-8"))) == target_python + assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") + + before = feature_file.read_bytes() + fail_inventory = True + assert native.stage_native(SimpleNamespace(out=str(output), ref="HEAD")) == 1 + assert not (output / "manifest.json").exists() + assert feature_file.read_bytes() == before assert os.environ["HERMES_RUNTIME_DIR"] == str(tmp_path / "original") monkeypatch.setattr(native, "_run_live", lambda *a, **kw: (1, "injected failure"))