diff --git a/apps/desktop/BUILDING.md b/apps/desktop/BUILDING.md index f78f0c07cd..bcfbf7c893 100644 --- a/apps/desktop/BUILDING.md +++ b/apps/desktop/BUILDING.md @@ -280,11 +280,15 @@ relaunch waiter is registered before the install request. Unknown checks, cancellation and request failures do not count as successful updates. Native acceptance requires a Store-acquired package or flight, not a sideloaded MSIX. -On Windows bundles, PM copies the verified pinned base Python to its writable -store for uv builds. The app and execution aliases retain their signed bundled -launchers and bundled Python, which load the selected dependency generation. -The new venv's generated console scripts must not replace those launchers: -out-of-package Python cannot launch the packaged tools. +On Windows bundles, every dependency generation is built on the bundled Python +and nothing runs a venv's own executables. A venv's `Scripts\python.exe` is a +redirector outside the package that starts the packaged interpreter, which +Windows refuses (WinError 5) to a process without package identity. The app and +execution aliases keep their signed bundled launchers, and children that need a +generation's packages start through `pm.environments.venv_command`: the bundled +Python with `-S` plus `pm/_venv_entry.py`, which attaches the generation's +site-packages. The new venv's generated console scripts must not replace the +launchers, and are kept off `PATH`. Sideload stable versions are `X.Y.Z.0`. Canary revisions derive from elapsed minutes after the stable baseline. The release script rejects ambiguous or diff --git a/hermes_cli/plugin_validate.py b/hermes_cli/plugin_validate.py index 3bd52fcfad..96301fa0e2 100644 --- a/hermes_cli/plugin_validate.py +++ b/hermes_cli/plugin_validate.py @@ -325,12 +325,12 @@ def _probe_options(manifest: dict) -> dict: def _run_capability_probe( - plugin_dir: Path, manifest: dict, probe: Optional[Tuple[Path, Dict[str, str]]] = None, + plugin_dir: Path, manifest: dict, probe: Optional[Tuple[List[str], Dict[str, str]]] = None, ) -> Tuple[Optional[dict], str]: """Run the recording probe in a scratch subprocess. - *probe* is ``(interpreter, env)`` of the dependency environment to import the plugin from; - None probes this interpreter. + *probe* is ``(python argv prefix, env)`` of the dependency environment to import the plugin + from (``pm.environments.venv_command``); None probes this interpreter. Returns ``(recorded, error)`` — exactly one is meaningful: *recorded* is the ``{tools, hooks, middleware, commands, providers}`` dict on @@ -342,7 +342,7 @@ def _run_capability_probe( try: result = subprocess.run( [ - str(probe[0]) if probe else sys.executable, + *(probe[0] if probe else [sys.executable]), "-c", _PROBE_SCRIPT, str(plugin_dir), @@ -479,10 +479,10 @@ def _check_builtin_collisions( def validate_plugin_dir( - plugin_dir: Path, probe: Optional[Tuple[Path, Dict[str, str]]] = None, + plugin_dir: Path, probe: Optional[Tuple[List[str], Dict[str, str]]] = None, ) -> ValidationReport: """Run every admission check against *plugin_dir* and return the report. *probe* is - ``(interpreter, env)`` for the capability probe (see ``_run_capability_probe``).""" + ``(python argv prefix, env)`` for the capability probe (see ``_run_capability_probe``).""" report = ValidationReport() plugin_dir = Path(plugin_dir) diff --git a/hermes_cli/plugins_cmd_catalog.py b/hermes_cli/plugins_cmd_catalog.py index a2bd12cffd..96170bd030 100644 --- a/hermes_cli/plugins_cmd_catalog.py +++ b/hermes_cli/plugins_cmd_catalog.py @@ -774,7 +774,7 @@ def cmd_validate(path: str, as_json: bool = False, install_deps: bool = False) - if install_deps: import pm from pm import paths - from pm.environments import activation_environment, project_python + from pm.environments import activation_environment, project_python, selected_venv, venv_command from pm.plugin_inputs import Candidates try: pm.sync_venv(plugins=Candidates([Path(path)])) @@ -782,7 +782,7 @@ def cmd_validate(path: str, as_json: bool = False, install_deps: bool = False) - # so the probe must import the plugin from that environment's interpreter. root = paths.repo_root() if project_python(root).is_file(): # a developer venv has no committed environment - probe = (project_python(root), activation_environment(root)) + probe = (venv_command(root, selected_venv(root)), activation_environment(root)) except Exception as exc: # validation still runs; the probe reports what is missing print(f"dependency preparation failed: {exc}", file=sys.stderr) report = validate_plugin_dir(Path(path), probe) diff --git a/pm/_uv.py b/pm/_uv.py index a5ec8429d0..85180f03c0 100644 --- a/pm/_uv.py +++ b/pm/_uv.py @@ -3,24 +3,20 @@ from __future__ import annotations from pathlib import Path -from pm import paths -from pm.filesystem import long_root, native -from pm.lock import Facts +from pm.filesystem import native from pm.package import InstallError from pm.registry import get_package -from pm.store import Store, current_target +from pm.store import current_target def _toolchain(*, realize: bool = True, explicit: bool = False) -> tuple[Path, Path] | None: """Resolve uv and Python without host discovery or recursive worker dispatch. - A read-only probe never installs. Windows bundle builds need a verified - writable interpreter so their venv redirectors can run outside the MSIX. + A read-only probe never installs. A sealed payload builds on its own + interpreter: a venv's redirectors then name a path inside the package, so + nothing may run them (``pm.environments.venv_command`` enters such a venv). """ - from pm.install import ( - _install, _installed_location, _lockfile, _refuse_lazy, - ensure, lazy_installs_allowed, sealed, - ) + from pm.install import _installed_location, _lockfile, ensure if realize: ensure("uv", explicit=explicit) @@ -35,21 +31,6 @@ def _toolchain(*, realize: bool = True, explicit: bool = False) -> tuple[Path, P if not realize: return None raise InstallError(name, "pinned tool is unavailable", "run `hermes pm install`") - if name == "python" and target.startswith("win32") and sealed(): - # Spelled like Store spells its root, so the comparison and the copy's file calls agree. - writable = long_root(paths.writable_store_root()) - if location[1].root != writable: - copied = _installed_location(package, lockfile, target, verify=explicit, roots=(writable,)) - if copied is None: - if not realize: - return None - if not explicit and not lazy_installs_allowed(): - raise _refuse_lazy(name, "a writable Python is required for bundled builds") - copy_store = Store(writable) - copy_facts = Facts(writable / "facts.json") - _install(package, lockfile, copy_facts, copy_store, target, copy_from=location) - copied = copy_facts, copy_store - location = copied facts, store = location binary = package.binary(store.entry(facts.get(name)["entry"]), target) if binary is None or not binary.is_file(): diff --git a/pm/_venv_entry.py b/pm/_venv_entry.py new file mode 100644 index 0000000000..121b91521a --- /dev/null +++ b/pm/_venv_entry.py @@ -0,0 +1,56 @@ +"""Run code inside a venv on another interpreter: ``python -S _venv_entry.py SITE ARGS...``. + +ARGS are what would follow the venv interpreter's own options: ``SCRIPT [ARG...]``, +``-c CODE [ARG...]`` or ``-m MODULE [ARG...]``. Built by ``pm.environments.venv_command`` +for sealed payloads, whose venv redirectors must not run (see there). Stdlib only. +""" +import os +import runpy +import site +import sys +import types + + +def _usage() -> None: + raise SystemExit("usage: _venv_entry.py SITE (SCRIPT | -c CODE | -m MODULE) [ARG...]") + + +def main() -> None: + if len(sys.argv) < 3: + _usage() + site_dir, args = sys.argv[1], sys.argv[2:] + here = os.path.dirname(os.path.abspath(__file__)) + # Python put this file's directory first; it is PM's package dir, not the caller's. + sys.path[:] = [entry for entry in sys.path + if not entry or os.path.abspath(entry) != here] + # A venv's site-packages follows the stdlib; addsitedir also runs its .pth files + # (pywin32's puts win32\lib on sys.path). + site.addsitedir(site_dir) + safe_path = sys.flags.safe_path if hasattr(sys.flags, "safe_path") else sys.flags.isolated + if args[0] == "-c": + if len(args) < 2: + _usage() + sys.argv = ["-c", *args[2:]] + if not safe_path: + sys.path.insert(0, "") + main_module = types.ModuleType("__main__") + sys.modules["__main__"] = main_module + exec(compile(args[1], "", "exec"), main_module.__dict__) + elif args[0] == "-m": + if len(args) < 2: + _usage() + sys.argv = [args[1], *args[2:]] + if not safe_path: + sys.path.insert(0, os.getcwd()) + runpy.run_module(args[1], run_name="__main__", alter_sys=True) + elif args[0].startswith("-"): + _usage() + else: + sys.argv = list(args) + if not safe_path: + sys.path.insert(0, os.path.dirname(os.path.abspath(args[0]))) + runpy.run_path(args[0], run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/pm/environments.py b/pm/environments.py index 8a685bfc4f..59ef94036c 100644 --- a/pm/environments.py +++ b/pm/environments.py @@ -338,6 +338,36 @@ def project_python(project_root: Path) -> Path: return venv_python(selected_venv(project_root)) +def _payload_store_python(root: Path) -> Path | None: + """A sealed payload's own interpreter when a venv must not be entered through its own.""" + manifest = _payload_manifest(root) + if manifest is None or str(manifest.get("target", "")).endswith("-bionic"): + return None # Termux payloads launch through their venv interpreter by design. + relative = manifest.get("runtime", {}).get("storePython") + if not isinstance(relative, str) or not relative: + return None + return _payload_path(root, relative, "interpreter") + + +def venv_command(project_root: Path, venv: Path, options: tuple[str, ...] | list[str] = ()) -> list[str]: + """The argv prefix that runs Python inside *venv*; append a script, ``-c`` or ``-m``. + + A sealed payload never enters a venv through ``Scripts\\python.exe``. That redirector is + an executable outside the package that starts the package's interpreter, which an MSIX + refuses (WinError 5) to a process without package identity. Instead the payload's own + interpreter attaches the venv's site-packages (``pm/_venv_entry.py``): the same + contract as the payload's launchers and its PM worker. Elsewhere the venv's + interpreter runs as is. *options* are interpreter flags (``-I``, ``-u``, ``-X …``). + """ + root = Path(project_root).resolve() + store_python = _payload_store_python(root) + if store_python is None: + return [str(venv_python(venv)), *options] + entry = Path(__file__).resolve().with_name("_venv_entry.py") + flags = [*options, *([] if "-S" in options else ["-S"])] + return [str(store_python), *flags, str(entry), str(site_packages(Path(venv)))] + + def venv_python_version(venv: Path) -> tuple[int, int] | None: """The interpreter version a POSIX venv actually holds, or ``None``. @@ -471,9 +501,10 @@ def activate_dependencies(project_root: Path) -> None: # `hermes` off PATH must hit the checkout's own launcher first, never the venv's copy. # A sealed payload's launchers live in /bin instead. directories = [payload_command_dir(project_root) or project_root.resolve() / ".hermes" / "bin"] - # A shipped venv's Windows redirectors read pyvenv.cfg `home`, which still names the - # build machine, so every executable in its Scripts fails on the user's machine. - if not (os.name == "nt" and environment == payload_venv(project_root)): + # A sealed payload's Windows venv redirectors never go on PATH: a shipped venv's still + # names the build machine, and a generation built since names the payload's interpreter, + # which an MSIX refuses to an outside executable (see venv_command). + if not (os.name == "nt" and _payload_manifest(project_root.resolve()) is not None): directories.append(venv_bin_dir(environment)) prefix = [str(path) for path in directories if path.is_dir()] if prefix: diff --git a/pm/packages.py b/pm/packages.py index 0f59caa1e1..315adf504a 100644 --- a/pm/packages.py +++ b/pm/packages.py @@ -444,8 +444,10 @@ class Venv(StatePackage): resolved_lock = generation / "workspace" / "uv.lock" environment.check() if repair: + from pm.environments import venv_command from pm.recovery import validate_environment - validate_environment(environment.executable, env=dict(environment.env), cwd=resolved_lock.parent) + validate_environment(venv_command(project, candidate, ("-I",)), + env=dict(environment.env), cwd=resolved_lock.parent) except BaseException: shutil.rmtree(generation, ignore_errors=True) raise diff --git a/pm/recovery.py b/pm/recovery.py index 5ecee0a6a9..d1bb8f94e5 100644 --- a/pm/recovery.py +++ b/pm/recovery.py @@ -20,8 +20,10 @@ STARTUP_IMPORTS = ( ) -def validate_environment(python: Path, *, env: dict, cwd: Path) -> None: - """Run startup import checks in the candidate, never the repairing process.""" +def validate_environment(python: Path | list[str], *, env: dict, cwd: Path) -> None: + """Run startup import checks in the candidate, never the repairing process. + + *python* is the candidate's interpreter or its argv prefix (``venv_command``).""" script = ( "import importlib, importlib.metadata, pathlib, re, tomllib\n" "project = tomllib.loads(pathlib.Path('pyproject.toml').read_text(encoding='utf-8-sig'))['project']\n" @@ -41,7 +43,8 @@ def validate_environment(python: Path, *, env: dict, cwd: Path) -> None: " bundle = pathlib.Path(loaded.where())\n" " assert bundle.is_file() and bundle.stat().st_size >= 1024, 'CA bundle is missing'\n" ) - result = subprocess.run([str(python), "-I", "-c", script], cwd=cwd, env=env, + command = [str(python), "-I"] if isinstance(python, (str, Path)) else [*python] + result = subprocess.run([*command, "-c", script], cwd=cwd, env=env, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60) if result.returncode: raise InstallError("venv", f"startup validation failed: {result.stderr.strip()[-1000:]}") diff --git a/tests/pm/test_uv_python.py b/tests/pm/test_uv_python.py index 056423ac78..9f7cfae8ea 100644 --- a/tests/pm/test_uv_python.py +++ b/tests/pm/test_uv_python.py @@ -180,74 +180,42 @@ def test_uv_refuses_discovery_when_pm_python_is_missing(installed_uv, monkeypatc assert not list(entry.iterdir()) -@pytest.mark.platforms("windows") -def test_bundled_uv_uses_a_verified_writable_python_without_changing_runtime(installed_uv, monkeypatch): +def test_bundled_uv_builds_on_the_shipped_python_and_writes_nothing(installed_uv, monkeypatch): + """A sealed payload's venvs are built on its own interpreter, never a writable copy: + the venvs are entered through ``venv_command``, so no redirector needs an outside target.""" import pm.paths as paths import pm.registry as registry from pm.store import Store, tree_digest root, uv_binary, facts, target, digest = installed_uv - ensure = importlib.import_module("pm.install") shipped = uv_binary.parent.parent writable = root / "writable-tools" monkeypatch.setattr(paths, "writable_store_root", lambda: writable) (shipped.parent / "manifest.json").write_text("{}", encoding="utf-8") - class FixturePython(Python): - def verify(self, entry, target): - return "" if self.binary(entry, target).read_bytes() == b"pinned interpreter" else "damaged Python" - class FixtureUv(Uv): emulated_arch_targets = {target} monkeypatch.setitem(registry._packages, "uv", FixtureUv()) facts.record("uv", "test", uv_binary.parent.name, {}, shipped, target=target, artifacts=[digest], digest=tree_digest(uv_binary.parent)) - python = FixturePython() + python = Python() monkeypatch.setitem(registry._packages, "python", python) entry = shipped / python.store_entry("test", target) - entry.mkdir() - binary = entry / "python.exe" + binary = python.binary(entry, target) + binary.parent.mkdir(parents=True) binary.write_bytes(b"pinned interpreter") - (entry / "python.dll").write_bytes(b"pinned runtime") + binary.chmod(0o755) facts.record("python", "test", entry.name, python.env(entry, target), shipped, target=target, artifacts=[digest], digest=tree_digest(entry)) before = facts.path.read_bytes() - shipped_digest = tree_digest(entry) - monkeypatch.setattr(ensure, "lazy_installs_allowed", lambda: False) + monkeypatch.setattr(Store, "fetch_many", lambda *a, **k: pytest.fail("the shipped Python must not be fetched")) + monkeypatch.setattr(shutil, "copytree", lambda *a, **k: pytest.fail("the shipped Python must not be copied")) - def no_download(*args, **kwargs): - raise AssertionError("the verified Python must be copied without downloading") - - monkeypatch.setattr(Store, "fetch_many", no_download) - assert _toolchain(realize=False) is None + assert _toolchain(realize=False) == (uv_binary, binary) + assert _toolchain(explicit=True) == (uv_binary, binary) assert not writable.exists() - with pytest.raises(InstallError, match="lazy installs are disabled"): - _toolchain() - assert not writable.exists() - - resolved_uv, python = _toolchain(explicit=True) - copied = writable / entry.name - assert resolved_uv == uv_binary - assert python == copied / "python.exe" - assert tree_digest(copied) == shipped_digest - copied_fact = Facts(writable / "facts.json").get("python") - assert copied_fact["digest"] == shipped_digest - assert copied_fact["artifacts"] == [digest] - assert copied_fact["target"] == target - assert ensure.installed_package("python").binary == binary - assert str(copied) not in ensure.env_for("python")["PATH"] - - def no_copy(*args, **kwargs): - raise AssertionError("a matching copy must be reused") - - with monkeypatch.context() as reuse: - reuse.setattr(shutil, "copytree", no_copy) - assert _toolchain(realize=False)[1] == python - assert _toolchain()[1] == python - assert facts.path.read_bytes() == before - assert tree_digest(entry) == shipped_digest @pytest.mark.parametrize("damage", [None, "source", "copy", "publication"]) diff --git a/tests/pm/test_venv_command.py b/tests/pm/test_venv_command.py new file mode 100644 index 0000000000..af86424b28 --- /dev/null +++ b/tests/pm/test_venv_command.py @@ -0,0 +1,112 @@ +"""A sealed payload enters its venvs on its own interpreter, never through their redirectors. + +A venv's ``Scripts\\python.exe`` sits outside the MSIX and starts the package's interpreter, +which Windows refuses (WinError 5) to a process without package identity. ``venv_command`` +must therefore run the payload's interpreter with the venv attached, and the result must +behave like the venv's own interpreter for scripts, ``-c`` and ``-m``. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import stat +import subprocess +import sys + +import pytest + +from pm.environments import site_packages, venv_command, venv_python + + +def _venv(root: Path) -> Path: + """A venv whose interpreter must never run, with a module and a .pth-added directory.""" + venv = root / "venv" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + venv.mkdir(parents=True) + (venv / "pyvenv.cfg").write_text(f"home = /nonexistent\nversion = {version}.0\n", encoding="utf-8") + trap = venv_python(venv) + trap.parent.mkdir(parents=True) + trap.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") + trap.chmod(trap.stat().st_mode | stat.S_IEXEC) + site = site_packages(venv) + site.mkdir(parents=True) + (site / "venv_only_dep.py").write_text("VALUE = 'from the venv'\n", encoding="utf-8") + (site / "venv_only_cli.py").write_text( + "import json, sys, venv_only_dep\n" + "print(json.dumps([venv_only_dep.VALUE, sys.argv]))\n", encoding="utf-8") + extra = root / "pth-added" + extra.mkdir() + (extra / "pth_only_dep.py").write_text("VALUE = 'from a .pth'\n", encoding="utf-8") + (site / "extra.pth").write_text(str(extra) + "\n", encoding="utf-8") + return venv + + +def _payload(root: Path, python: str) -> Path: + """A sealed payload layout whose recorded interpreter is *python*.""" + payload = root / "payload" + repo = payload / "hermes-agent" + repo.mkdir(parents=True) + tools = payload / "tools" / "python" + tools.mkdir(parents=True) + store_python = tools / "python" + store_python.write_text(f"#!/bin/sh\nexec {python!s} \"$@\"\n", encoding="utf-8") + store_python.chmod(store_python.stat().st_mode | stat.S_IEXEC) + (payload / "manifest.json").write_text(json.dumps({ + "schema": 1, "target": "test", "repo": "hermes-agent", "venv": "venv", "store": "tools", + "runtime": {"repoDir": "hermes-agent", "toolsDir": "tools", "storePython": "tools/python/python", + "sitePackages": "venv/site", "commands": {}}, + }), encoding="utf-8") + return repo + + +def _run(command: list[str], cwd: Path) -> list: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True, timeout=60, + env={k: v for k, v in os.environ.items() if k not in ("PYTHONPATH", "PYTHONHOME")}) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +def test_unsealed_trees_run_the_venv_interpreter(tmp_path): + venv = _venv(tmp_path) + checkout = tmp_path / "checkout" + checkout.mkdir() + assert venv_command(checkout, venv, ("-I", "-u")) == [str(venv_python(venv)), "-I", "-u"] + + +@pytest.mark.platforms("linux", "macos") +def test_sealed_payload_runs_scripts_code_and_modules_without_the_redirector(tmp_path): + venv = _venv(tmp_path) + repo = _payload(tmp_path, sys.executable) + prefix = venv_command(repo, venv, ("-I", "-u")) + assert str(venv_python(venv)) not in prefix + assert prefix[0] == str((repo.parent / "tools/python/python").resolve()) + assert "-S" in prefix # the payload interpreter's own site must not load + + script = tmp_path / "work" / "script.py" + script.parent.mkdir() + script.write_text("import json, sys, venv_only_dep, pth_only_dep\n" + "print(json.dumps([venv_only_dep.VALUE, pth_only_dep.VALUE, sys.argv]))\n", + encoding="utf-8") + assert _run([*prefix, str(script), "a", "--flag"], tmp_path) == [ + "from the venv", "from a .pth", [str(script), "a", "--flag"]] + + code = ("import json, sys, venv_only_dep, pth_only_dep; " + "print(json.dumps([venv_only_dep.VALUE, pth_only_dep.VALUE, sys.argv, __name__]))") + assert _run([*prefix, "-c", code, "x"], tmp_path) == [ + "from the venv", "from a .pth", ["-c", "x"], "__main__"] + + value, argv = _run([*prefix, "-m", "venv_only_cli", "y"], tmp_path) + assert value == "from the venv" + assert argv[1:] == ["y"] and argv[0].endswith("venv_only_cli.py") + + +def test_entry_keeps_isolation_and_pm_off_the_import_path(tmp_path): + """Under -I neither the caller's cwd nor PM's own directory shadows the venv's modules.""" + venv = _venv(tmp_path) + (tmp_path / "venv_only_dep.py").write_text("VALUE = 'cwd shadow'\n", encoding="utf-8") + entry = Path(__import__("pm").__file__).with_name("_venv_entry.py") + code = ("import json, sys, venv_only_dep; " + f"print(json.dumps([venv_only_dep.VALUE, {str(entry.parent)!r} in sys.path]))") + assert _run([sys.executable, "-I", "-S", str(entry), str(site_packages(venv)), "-c", code], + tmp_path) == ["from the venv", False]