diff --git a/pm/environment.py b/pm/environment.py index e48a533623..80877a6421 100644 --- a/pm/environment.py +++ b/pm/environment.py @@ -19,6 +19,33 @@ from typing import TextIO from pm.package import InstallError +def prune_site_pth(venv_dir: Path) -> None: + """Drop .pth files that must never execute inside a shipped payload. + + ``uv sync`` leaves two behind: ``_virtualenv.pth`` (repoints + ``sys.prefix`` at the venv) and the project's ``__editable__`` pointer + (names the BUILD machine — the payload wires the repo snapshot itself, + see scripts/build/launcher_wrapper.py). Bundled launchers process every + other .pth with ``site.addsitedir()``; pywin32.pth is load-bearing on + Windows (win32\\lib on sys.path is what makes ``import pywintypes`` + resolve, which portalocker/concurrent-log-handler need to write logs). + """ + if (venv_dir / "Scripts").is_dir(): + sites = [venv_dir / "Lib" / "site-packages"] + else: + lib = venv_dir / "lib" + sites = sorted(lib.glob("python*/site-packages")) if lib.is_dir() else [] + for site_dir in sites: + if not site_dir.is_dir(): + continue + for pth in site_dir.glob("*.pth"): + if pth.name == "_virtualenv.pth" or pth.name.startswith("__editable__"): + try: + pth.unlink() + except OSError: + pass + + def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str], timeout: int, output: TextIO) -> subprocess.CompletedProcess: """Keep CI progress live, a bounded diagnostic tail, and a wall-clock timeout.""" diff --git a/scripts/build/launcher_wrapper.py b/scripts/build/launcher_wrapper.py index 9b5c3399bf..d7197d3e2e 100644 --- a/scripts/build/launcher_wrapper.py +++ b/scripts/build/launcher_wrapper.py @@ -17,7 +17,11 @@ The wrapper replaces everything the old rust shim + its sidecar did on sys.path (same order, same reason as the old shim: the repo's hermes_cli wins over anything stale in site-packages — the sealed payload's venv has no working editable install, its pointer names the - BUILD machine), + BUILD machine). The site entry goes through ``site.addsitedir()`` — + the only mechanism that runs ``.pth`` files — because pywin32.pth is + what puts win32\\lib on sys.path and therefore what makes + ``import pywintypes`` resolve on Windows bundles (portalocker's + Win32Locker → concurrent-log-handler → hermes_logging.py's files), * drop inherited PYTHONPATH / PYTHONHOME so foreign installs can never shadow the bundle, * default sys.pycache_prefix to the user-level cache (%LOCALAPPDATA% @@ -31,6 +35,7 @@ directly (tests/scripts/test_desktop_cli_wrapper.py). import importlib import os +import site import sys HERMES_ENTRY_MODULE = "__HERMES_ENTRY_MODULE__" @@ -78,8 +83,21 @@ def configure(here, environ=None): # the stdlib entirely. environ.pop("PYTHONPATH", None) environ.pop("PYTHONHOME", None) - entries = payload_sys_paths(here) - sys.path[0:0] = entries + repo_entry, site_entry = payload_sys_paths(here) + # Repo snapshot first — its hermes_cli wins over anything stale in + # site-packages (the sealed payload has no working editable install). + sys.path.insert(0, repo_entry) + # addsitedir(), not a raw append: only it processes the venv's .pth + # files. pywin32.pth is load-bearing on Windows — it puts win32\lib + # on sys.path, which is what makes `import pywintypes` resolve, and + # without that portalocker's Win32Locker dies and + # concurrent-log-handler silently drops every file-log record (the + # same trap gateway/run.py's MCP venv bootstrap works around). Keep + # site-packages directly after the repo, ahead of .pth-added dirs. + site.addsitedir(site_entry) + if site_entry in sys.path: + sys.path.remove(site_entry) + sys.path.insert(1 if sys.path and sys.path[0] == repo_entry else 0, site_entry) if not environ.get("PYTHONPYCACHEPREFIX"): default = default_pycache_dir(environ) if default: @@ -88,7 +106,7 @@ def configure(here, environ=None): # startup, but setting it in os.environ cannot retro-activate # it — sys.pycache_prefix is the live switch. sys.pycache_prefix = default - return entries + return [repo_entry, site_entry] def main(argv=None): diff --git a/scripts/build/launchers.py b/scripts/build/launchers.py index 99135bc85c..c7afcfc34d 100644 --- a/scripts/build/launchers.py +++ b/scripts/build/launchers.py @@ -42,7 +42,11 @@ export HERMES_NODE="$root/tools/node/data/data/com.termux/files/usr/bin/node" export HERMES_RUNTIME_DIR="$root/tools" export PATH="$root/tools/npm/bin:$root/tools/node/data/data/com.termux/files/usr/bin:$root/tools/ffmpeg/data/data/com.termux/files/usr/bin:$root/tools/ripgrep:$PATH" ''' - code = f"import sys; sys.argv[0]={name!r}; from {module} import {func}; sys.exit({func}())" + code = ( + f"import os, site, sys; sys.argv[0]={name!r}; " + "site.addsitedir(os.environ['HERMES_SITE']); " + f"from {module} import {func}; sys.exit({func}())" + ) return f'''{header} set -eu self="$0" @@ -60,6 +64,11 @@ SITE={shell_path(site)} [ -x "$PYTHON" ] || {{ printf '%s\\n' 'Bundled interpreter missing; reinstall Hermes.' >&2; exit 2; }} unset PYTHONPATH PYTHONHOME export PYTHONPATH="$REPO:$SITE" +# PYTHONPATH cannot process .pth files (only site.addsitedir() can), and +# the venv's .pth files are load-bearing (pywin32.pth -> win32\\lib -> +# `import pywintypes` on Windows bundles; the win32 wrapper mirrors this +# in launcher_wrapper.py). The -c bootstrap below runs addsitedir() on it. +export HERMES_SITE="$SITE" export PYTHONPYCACHEPREFIX="${{PYTHONPYCACHEPREFIX:-${{XDG_CACHE_HOME:-$HOME/.cache}}/hermes-pycache}}" {extra}exec "$PYTHON" -P -c {shlex.quote(code)} "$@" ''' diff --git a/scripts/build/python_env.py b/scripts/build/python_env.py index b45cdb8972..5a13d87843 100644 --- a/scripts/build/python_env.py +++ b/scripts/build/python_env.py @@ -14,7 +14,7 @@ import shutil import subprocess import sys -from pm.environment import PythonEnvironment +from pm.environment import PythonEnvironment, prune_site_pth from pm.package import InstallError @@ -44,6 +44,11 @@ def build_python_environment( environment.sync(source, extras=extras, all_extras=all_extras, no_install_project=no_install_project) environment.check() + # The sealed payload must never process uv's venv-marker or + # editable-install .pth files (see pm.environment.prune_site_pth): + # the launcher addsitedirs the venv, and those two would repoint + # sys.prefix / shadow the repo snapshot with build-machine paths. + prune_site_pth(out) except BaseException: shutil.rmtree(out, ignore_errors=True) raise diff --git a/tests/pm/test_environment_build.py b/tests/pm/test_environment_build.py index 809aecdf4d..a65640394a 100644 --- a/tests/pm/test_environment_build.py +++ b/tests/pm/test_environment_build.py @@ -34,6 +34,39 @@ def _run(command, *, cwd: Path, env: dict) -> str: return result.stdout.strip() +def test_prune_site_pth_keeps_only_load_bearing_pth(tmp_path): + """The payload venv must not carry uv's venv-marker or editable-install + .pth files: the launcher addsitedirs the venv, and those two would repoint + sys.prefix / shadow the repo snapshot with build-machine paths. Everything + else (pywin32.pth!) must survive — it is what makes `import pywintypes` + resolve on Windows bundles.""" + from pm.environment import prune_site_pth + + # Windows layout (Scripts/ present). + win_venv = tmp_path / "win-venv" + (win_venv / "Scripts").mkdir(parents=True) + win_site = win_venv / "Lib" / "site-packages" + win_site.mkdir(parents=True) + # POSIX layout (bin/ present, versioned site-packages). + posix_venv = tmp_path / "posix-venv" + (posix_venv / "bin").mkdir(parents=True) + posix_site = posix_venv / "lib" / "python3.14" / "site-packages" + posix_site.mkdir(parents=True) + + for site in (win_site, posix_site): + (site / "pywin32.pth").write_text("win32\nwin32\\lib\nimport pywin32_bootstrap\n", encoding="utf-8") + (site / "_virtualenv.pth").write_text("import _virtualenv\n", encoding="utf-8") + (site / "__editable__.hermes_agent-0.21.1.pth").write_text( + "import __editable___hermes_agent_0_21_1_finder\n", encoding="utf-8" + ) + + prune_site_pth(win_venv) + prune_site_pth(posix_venv) + + assert sorted(p.name for p in win_site.glob("*.pth")) == ["pywin32.pth"] + assert sorted(p.name for p in posix_site.glob("*.pth")) == ["pywin32.pth"] + + @pytest.fixture def locked_project(tmp_path): uv = shutil.which("uv") diff --git a/tests/scripts/test_desktop_cli_wrapper.py b/tests/scripts/test_desktop_cli_wrapper.py index aa60bc693f..88c8ee7c49 100644 --- a/tests/scripts/test_desktop_cli_wrapper.py +++ b/tests/scripts/test_desktop_cli_wrapper.py @@ -91,6 +91,36 @@ def test_configure_prepends_repo_then_site_packages(tmp_path): sys.path[:] = original +def test_configure_processes_site_pth_files(tmp_path): + """site-packages goes through site.addsitedir(), not a raw append. + + Only addsitedir() runs .pth files, and pywin32.pth is load-bearing on + Windows bundles: it puts win32\\lib on sys.path, which is what makes + `import pywintypes` resolve (portalocker's Win32Locker → + concurrent-log-handler → hermes_logging.py's file handlers). The old + raw sys.path[0:0] = entries skipped .pth processing entirely, silently + killing file logging on Windows bundles (and any future .pth-based + dependency on every platform). + """ + ns = _load() + here = str(tmp_path / "bin") + site = os.path.join(here, "..", "venv", "Lib", "site-packages") + os.makedirs(site) + added = tmp_path / "pth-added" + added.mkdir() + with open(os.path.join(site, "load-bearing.pth"), "w", encoding="utf-8") as f: + f.write(f"{added}\n") + original = list(sys.path) + try: + ns["configure"](here, environ={}) + repo = os.path.join(here, "..", "repo") + # Repo first, site second, .pth-added dirs after both. + assert sys.path[:2] == [repo, site] + assert sys.path.index(str(added)) > 1 + finally: + sys.path[:] = original + + def test_configure_drops_inherited_pythonpath_and_pythonhome(tmp_path): ns = _load() original = list(sys.path)