fix(bundle): process the venv's .pth files in payload launchers
The minted launchers wired venv site-packages onto sys.path with a raw insert (win32 wrapper) / PYTHONPATH (posix), neither of which runs .pth files. pywin32.pth is load-bearing on Windows: it puts win32\lib on sys.path, which is what makes 'import pywintypes' resolve — without it portalocker's Win32Locker dies and concurrent-log-handler silently drops every file-log record on Windows bundles. * launcher_wrapper.py: site.addsitedir() for the site entry (repo first, site directly after, .pth dirs last) * launchers.py posix: same via HERMES_SITE env in the -c bootstrap * pm/environment.py: prune_site_pth() drops _virtualenv.pth and the __editable__ pointer (build-machine path) that must never run in a sealed payload * python_env.py: run the prune after every environment build
This commit is contained in:
@@ -19,6 +19,33 @@ from typing import TextIO
|
||||
from pm.package import InstallError
|
||||
|
||||
|
||||
def prune_site_pth(venv_dir: Path) -> None:
|
||||
"""Drop .pth files that must never execute inside a shipped payload.
|
||||
|
||||
``uv sync`` leaves two behind: ``_virtualenv.pth`` (repoints
|
||||
``sys.prefix`` at the venv) and the project's ``__editable__`` pointer
|
||||
(names the BUILD machine — the payload wires the repo snapshot itself,
|
||||
see scripts/build/launcher_wrapper.py). Bundled launchers process every
|
||||
other .pth with ``site.addsitedir()``; pywin32.pth is load-bearing on
|
||||
Windows (win32\\lib on sys.path is what makes ``import pywintypes``
|
||||
resolve, which portalocker/concurrent-log-handler need to write logs).
|
||||
"""
|
||||
if (venv_dir / "Scripts").is_dir():
|
||||
sites = [venv_dir / "Lib" / "site-packages"]
|
||||
else:
|
||||
lib = venv_dir / "lib"
|
||||
sites = sorted(lib.glob("python*/site-packages")) if lib.is_dir() else []
|
||||
for site_dir in sites:
|
||||
if not site_dir.is_dir():
|
||||
continue
|
||||
for pth in site_dir.glob("*.pth"):
|
||||
if pth.name == "_virtualenv.pth" or pth.name.startswith("__editable__"):
|
||||
try:
|
||||
pth.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str],
|
||||
timeout: int, output: TextIO) -> subprocess.CompletedProcess:
|
||||
"""Keep CI progress live, a bounded diagnostic tail, and a wall-clock timeout."""
|
||||
|
||||
@@ -17,7 +17,11 @@ The wrapper replaces everything the old rust shim + its sidecar did
|
||||
on sys.path (same order, same reason as the old shim: the repo's
|
||||
hermes_cli wins over anything stale in site-packages — the sealed
|
||||
payload's venv has no working editable install, its pointer names the
|
||||
BUILD machine),
|
||||
BUILD machine). The site entry goes through ``site.addsitedir()`` —
|
||||
the only mechanism that runs ``.pth`` files — because pywin32.pth is
|
||||
what puts win32\\lib on sys.path and therefore what makes
|
||||
``import pywintypes`` resolve on Windows bundles (portalocker's
|
||||
Win32Locker → concurrent-log-handler → hermes_logging.py's files),
|
||||
* drop inherited PYTHONPATH / PYTHONHOME so foreign installs can never
|
||||
shadow the bundle,
|
||||
* default sys.pycache_prefix to the user-level cache (%LOCALAPPDATA%
|
||||
@@ -31,6 +35,7 @@ directly (tests/scripts/test_desktop_cli_wrapper.py).
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import site
|
||||
import sys
|
||||
|
||||
HERMES_ENTRY_MODULE = "__HERMES_ENTRY_MODULE__"
|
||||
@@ -78,8 +83,21 @@ def configure(here, environ=None):
|
||||
# the stdlib entirely.
|
||||
environ.pop("PYTHONPATH", None)
|
||||
environ.pop("PYTHONHOME", None)
|
||||
entries = payload_sys_paths(here)
|
||||
sys.path[0:0] = entries
|
||||
repo_entry, site_entry = payload_sys_paths(here)
|
||||
# Repo snapshot first — its hermes_cli wins over anything stale in
|
||||
# site-packages (the sealed payload has no working editable install).
|
||||
sys.path.insert(0, repo_entry)
|
||||
# addsitedir(), not a raw append: only it processes the venv's .pth
|
||||
# files. pywin32.pth is load-bearing on Windows — it puts win32\lib
|
||||
# on sys.path, which is what makes `import pywintypes` resolve, and
|
||||
# without that portalocker's Win32Locker dies and
|
||||
# concurrent-log-handler silently drops every file-log record (the
|
||||
# same trap gateway/run.py's MCP venv bootstrap works around). Keep
|
||||
# site-packages directly after the repo, ahead of .pth-added dirs.
|
||||
site.addsitedir(site_entry)
|
||||
if site_entry in sys.path:
|
||||
sys.path.remove(site_entry)
|
||||
sys.path.insert(1 if sys.path and sys.path[0] == repo_entry else 0, site_entry)
|
||||
if not environ.get("PYTHONPYCACHEPREFIX"):
|
||||
default = default_pycache_dir(environ)
|
||||
if default:
|
||||
@@ -88,7 +106,7 @@ def configure(here, environ=None):
|
||||
# startup, but setting it in os.environ cannot retro-activate
|
||||
# it — sys.pycache_prefix is the live switch.
|
||||
sys.pycache_prefix = default
|
||||
return entries
|
||||
return [repo_entry, site_entry]
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
||||
@@ -42,7 +42,11 @@ export HERMES_NODE="$root/tools/node/data/data/com.termux/files/usr/bin/node"
|
||||
export HERMES_RUNTIME_DIR="$root/tools"
|
||||
export PATH="$root/tools/npm/bin:$root/tools/node/data/data/com.termux/files/usr/bin:$root/tools/ffmpeg/data/data/com.termux/files/usr/bin:$root/tools/ripgrep:$PATH"
|
||||
'''
|
||||
code = f"import sys; sys.argv[0]={name!r}; from {module} import {func}; sys.exit({func}())"
|
||||
code = (
|
||||
f"import os, site, sys; sys.argv[0]={name!r}; "
|
||||
"site.addsitedir(os.environ['HERMES_SITE']); "
|
||||
f"from {module} import {func}; sys.exit({func}())"
|
||||
)
|
||||
return f'''{header}
|
||||
set -eu
|
||||
self="$0"
|
||||
@@ -60,6 +64,11 @@ SITE={shell_path(site)}
|
||||
[ -x "$PYTHON" ] || {{ printf '%s\\n' 'Bundled interpreter missing; reinstall Hermes.' >&2; exit 2; }}
|
||||
unset PYTHONPATH PYTHONHOME
|
||||
export PYTHONPATH="$REPO:$SITE"
|
||||
# PYTHONPATH cannot process .pth files (only site.addsitedir() can), and
|
||||
# the venv's .pth files are load-bearing (pywin32.pth -> win32\\lib ->
|
||||
# `import pywintypes` on Windows bundles; the win32 wrapper mirrors this
|
||||
# in launcher_wrapper.py). The -c bootstrap below runs addsitedir() on it.
|
||||
export HERMES_SITE="$SITE"
|
||||
export PYTHONPYCACHEPREFIX="${{PYTHONPYCACHEPREFIX:-${{XDG_CACHE_HOME:-$HOME/.cache}}/hermes-pycache}}"
|
||||
{extra}exec "$PYTHON" -P -c {shlex.quote(code)} "$@"
|
||||
'''
|
||||
|
||||
@@ -14,7 +14,7 @@ import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from pm.environment import PythonEnvironment
|
||||
from pm.environment import PythonEnvironment, prune_site_pth
|
||||
from pm.package import InstallError
|
||||
|
||||
|
||||
@@ -44,6 +44,11 @@ def build_python_environment(
|
||||
environment.sync(source, extras=extras, all_extras=all_extras,
|
||||
no_install_project=no_install_project)
|
||||
environment.check()
|
||||
# The sealed payload must never process uv's venv-marker or
|
||||
# editable-install .pth files (see pm.environment.prune_site_pth):
|
||||
# the launcher addsitedirs the venv, and those two would repoint
|
||||
# sys.prefix / shadow the repo snapshot with build-machine paths.
|
||||
prune_site_pth(out)
|
||||
except BaseException:
|
||||
shutil.rmtree(out, ignore_errors=True)
|
||||
raise
|
||||
|
||||
@@ -34,6 +34,39 @@ def _run(command, *, cwd: Path, env: dict) -> str:
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def test_prune_site_pth_keeps_only_load_bearing_pth(tmp_path):
|
||||
"""The payload venv must not carry uv's venv-marker or editable-install
|
||||
.pth files: the launcher addsitedirs the venv, and those two would repoint
|
||||
sys.prefix / shadow the repo snapshot with build-machine paths. Everything
|
||||
else (pywin32.pth!) must survive — it is what makes `import pywintypes`
|
||||
resolve on Windows bundles."""
|
||||
from pm.environment import prune_site_pth
|
||||
|
||||
# Windows layout (Scripts/ present).
|
||||
win_venv = tmp_path / "win-venv"
|
||||
(win_venv / "Scripts").mkdir(parents=True)
|
||||
win_site = win_venv / "Lib" / "site-packages"
|
||||
win_site.mkdir(parents=True)
|
||||
# POSIX layout (bin/ present, versioned site-packages).
|
||||
posix_venv = tmp_path / "posix-venv"
|
||||
(posix_venv / "bin").mkdir(parents=True)
|
||||
posix_site = posix_venv / "lib" / "python3.14" / "site-packages"
|
||||
posix_site.mkdir(parents=True)
|
||||
|
||||
for site in (win_site, posix_site):
|
||||
(site / "pywin32.pth").write_text("win32\nwin32\\lib\nimport pywin32_bootstrap\n", encoding="utf-8")
|
||||
(site / "_virtualenv.pth").write_text("import _virtualenv\n", encoding="utf-8")
|
||||
(site / "__editable__.hermes_agent-0.21.1.pth").write_text(
|
||||
"import __editable___hermes_agent_0_21_1_finder\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
prune_site_pth(win_venv)
|
||||
prune_site_pth(posix_venv)
|
||||
|
||||
assert sorted(p.name for p in win_site.glob("*.pth")) == ["pywin32.pth"]
|
||||
assert sorted(p.name for p in posix_site.glob("*.pth")) == ["pywin32.pth"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def locked_project(tmp_path):
|
||||
uv = shutil.which("uv")
|
||||
|
||||
@@ -91,6 +91,36 @@ def test_configure_prepends_repo_then_site_packages(tmp_path):
|
||||
sys.path[:] = original
|
||||
|
||||
|
||||
def test_configure_processes_site_pth_files(tmp_path):
|
||||
"""site-packages goes through site.addsitedir(), not a raw append.
|
||||
|
||||
Only addsitedir() runs .pth files, and pywin32.pth is load-bearing on
|
||||
Windows bundles: it puts win32\\lib on sys.path, which is what makes
|
||||
`import pywintypes` resolve (portalocker's Win32Locker →
|
||||
concurrent-log-handler → hermes_logging.py's file handlers). The old
|
||||
raw sys.path[0:0] = entries skipped .pth processing entirely, silently
|
||||
killing file logging on Windows bundles (and any future .pth-based
|
||||
dependency on every platform).
|
||||
"""
|
||||
ns = _load()
|
||||
here = str(tmp_path / "bin")
|
||||
site = os.path.join(here, "..", "venv", "Lib", "site-packages")
|
||||
os.makedirs(site)
|
||||
added = tmp_path / "pth-added"
|
||||
added.mkdir()
|
||||
with open(os.path.join(site, "load-bearing.pth"), "w", encoding="utf-8") as f:
|
||||
f.write(f"{added}\n")
|
||||
original = list(sys.path)
|
||||
try:
|
||||
ns["configure"](here, environ={})
|
||||
repo = os.path.join(here, "..", "repo")
|
||||
# Repo first, site second, .pth-added dirs after both.
|
||||
assert sys.path[:2] == [repo, site]
|
||||
assert sys.path.index(str(added)) > 1
|
||||
finally:
|
||||
sys.path[:] = original
|
||||
|
||||
|
||||
def test_configure_drops_inherited_pythonpath_and_pythonhome(tmp_path):
|
||||
ns = _load()
|
||||
original = list(sys.path)
|
||||
|
||||
Reference in New Issue
Block a user