fix(bundle): process the venv's .pth files in payload launchers

The minted launchers wired venv site-packages onto sys.path with a raw
insert (win32 wrapper) / PYTHONPATH (posix), neither of which runs .pth
files. pywin32.pth is load-bearing on Windows: it puts win32\lib on
sys.path, which is what makes 'import pywintypes' resolve — without it
portalocker's Win32Locker dies and concurrent-log-handler silently drops
every file-log record on Windows bundles.

* launcher_wrapper.py: site.addsitedir() for the site entry (repo first,
  site directly after, .pth dirs last)
* launchers.py posix: same via HERMES_SITE env in the -c bootstrap
* pm/environment.py: prune_site_pth() drops _virtualenv.pth and the
  __editable__ pointer (build-machine path) that must never run in a
  sealed payload
* python_env.py: run the prune after every environment build
This commit is contained in:
ethernet
2026-09-11 16:45:28 -04:00
parent ad231c164b
commit f67a3b59db
6 changed files with 128 additions and 6 deletions

View File

@@ -19,6 +19,33 @@ from typing import TextIO
from pm.package import InstallError
def prune_site_pth(venv_dir: Path) -> None:
"""Drop .pth files that must never execute inside a shipped payload.
``uv sync`` leaves two behind: ``_virtualenv.pth`` (repoints
``sys.prefix`` at the venv) and the project's ``__editable__`` pointer
(names the BUILD machine — the payload wires the repo snapshot itself,
see scripts/build/launcher_wrapper.py). Bundled launchers process every
other .pth with ``site.addsitedir()``; pywin32.pth is load-bearing on
Windows (win32\\lib on sys.path is what makes ``import pywintypes``
resolve, which portalocker/concurrent-log-handler need to write logs).
"""
if (venv_dir / "Scripts").is_dir():
sites = [venv_dir / "Lib" / "site-packages"]
else:
lib = venv_dir / "lib"
sites = sorted(lib.glob("python*/site-packages")) if lib.is_dir() else []
for site_dir in sites:
if not site_dir.is_dir():
continue
for pth in site_dir.glob("*.pth"):
if pth.name == "_virtualenv.pth" or pth.name.startswith("__editable__"):
try:
pth.unlink()
except OSError:
pass
def _run_streaming(command: list[str], *, cwd: Path, env: dict[str, str],
timeout: int, output: TextIO) -> subprocess.CompletedProcess:
"""Keep CI progress live, a bounded diagnostic tail, and a wall-clock timeout."""

View File

@@ -17,7 +17,11 @@ The wrapper replaces everything the old rust shim + its sidecar did
on sys.path (same order, same reason as the old shim: the repo's
hermes_cli wins over anything stale in site-packages — the sealed
payload's venv has no working editable install, its pointer names the
BUILD machine),
BUILD machine). The site entry goes through ``site.addsitedir()`` —
the only mechanism that runs ``.pth`` files — because pywin32.pth is
what puts win32\\lib on sys.path and therefore what makes
``import pywintypes`` resolve on Windows bundles (portalocker's
Win32Locker → concurrent-log-handler → hermes_logging.py's files),
* drop inherited PYTHONPATH / PYTHONHOME so foreign installs can never
shadow the bundle,
* default sys.pycache_prefix to the user-level cache (%LOCALAPPDATA%
@@ -31,6 +35,7 @@ directly (tests/scripts/test_desktop_cli_wrapper.py).
import importlib
import os
import site
import sys
HERMES_ENTRY_MODULE = "__HERMES_ENTRY_MODULE__"
@@ -78,8 +83,21 @@ def configure(here, environ=None):
# the stdlib entirely.
environ.pop("PYTHONPATH", None)
environ.pop("PYTHONHOME", None)
entries = payload_sys_paths(here)
sys.path[0:0] = entries
repo_entry, site_entry = payload_sys_paths(here)
# Repo snapshot first — its hermes_cli wins over anything stale in
# site-packages (the sealed payload has no working editable install).
sys.path.insert(0, repo_entry)
# addsitedir(), not a raw append: only it processes the venv's .pth
# files. pywin32.pth is load-bearing on Windows — it puts win32\lib
# on sys.path, which is what makes `import pywintypes` resolve, and
# without that portalocker's Win32Locker dies and
# concurrent-log-handler silently drops every file-log record (the
# same trap gateway/run.py's MCP venv bootstrap works around). Keep
# site-packages directly after the repo, ahead of .pth-added dirs.
site.addsitedir(site_entry)
if site_entry in sys.path:
sys.path.remove(site_entry)
sys.path.insert(1 if sys.path and sys.path[0] == repo_entry else 0, site_entry)
if not environ.get("PYTHONPYCACHEPREFIX"):
default = default_pycache_dir(environ)
if default:
@@ -88,7 +106,7 @@ def configure(here, environ=None):
# startup, but setting it in os.environ cannot retro-activate
# it — sys.pycache_prefix is the live switch.
sys.pycache_prefix = default
return entries
return [repo_entry, site_entry]
def main(argv=None):

View File

@@ -42,7 +42,11 @@ export HERMES_NODE="$root/tools/node/data/data/com.termux/files/usr/bin/node"
export HERMES_RUNTIME_DIR="$root/tools"
export PATH="$root/tools/npm/bin:$root/tools/node/data/data/com.termux/files/usr/bin:$root/tools/ffmpeg/data/data/com.termux/files/usr/bin:$root/tools/ripgrep:$PATH"
'''
code = f"import sys; sys.argv[0]={name!r}; from {module} import {func}; sys.exit({func}())"
code = (
f"import os, site, sys; sys.argv[0]={name!r}; "
"site.addsitedir(os.environ['HERMES_SITE']); "
f"from {module} import {func}; sys.exit({func}())"
)
return f'''{header}
set -eu
self="$0"
@@ -60,6 +64,11 @@ SITE={shell_path(site)}
[ -x "$PYTHON" ] || {{ printf '%s\\n' 'Bundled interpreter missing; reinstall Hermes.' >&2; exit 2; }}
unset PYTHONPATH PYTHONHOME
export PYTHONPATH="$REPO:$SITE"
# PYTHONPATH cannot process .pth files (only site.addsitedir() can), and
# the venv's .pth files are load-bearing (pywin32.pth -> win32\\lib ->
# `import pywintypes` on Windows bundles; the win32 wrapper mirrors this
# in launcher_wrapper.py). The -c bootstrap below runs addsitedir() on it.
export HERMES_SITE="$SITE"
export PYTHONPYCACHEPREFIX="${{PYTHONPYCACHEPREFIX:-${{XDG_CACHE_HOME:-$HOME/.cache}}/hermes-pycache}}"
{extra}exec "$PYTHON" -P -c {shlex.quote(code)} "$@"
'''

View File

@@ -14,7 +14,7 @@ import shutil
import subprocess
import sys
from pm.environment import PythonEnvironment
from pm.environment import PythonEnvironment, prune_site_pth
from pm.package import InstallError
@@ -44,6 +44,11 @@ def build_python_environment(
environment.sync(source, extras=extras, all_extras=all_extras,
no_install_project=no_install_project)
environment.check()
# The sealed payload must never process uv's venv-marker or
# editable-install .pth files (see pm.environment.prune_site_pth):
# the launcher addsitedirs the venv, and those two would repoint
# sys.prefix / shadow the repo snapshot with build-machine paths.
prune_site_pth(out)
except BaseException:
shutil.rmtree(out, ignore_errors=True)
raise

View File

@@ -34,6 +34,39 @@ def _run(command, *, cwd: Path, env: dict) -> str:
return result.stdout.strip()
def test_prune_site_pth_keeps_only_load_bearing_pth(tmp_path):
"""The payload venv must not carry uv's venv-marker or editable-install
.pth files: the launcher addsitedirs the venv, and those two would repoint
sys.prefix / shadow the repo snapshot with build-machine paths. Everything
else (pywin32.pth!) must survive — it is what makes `import pywintypes`
resolve on Windows bundles."""
from pm.environment import prune_site_pth
# Windows layout (Scripts/ present).
win_venv = tmp_path / "win-venv"
(win_venv / "Scripts").mkdir(parents=True)
win_site = win_venv / "Lib" / "site-packages"
win_site.mkdir(parents=True)
# POSIX layout (bin/ present, versioned site-packages).
posix_venv = tmp_path / "posix-venv"
(posix_venv / "bin").mkdir(parents=True)
posix_site = posix_venv / "lib" / "python3.14" / "site-packages"
posix_site.mkdir(parents=True)
for site in (win_site, posix_site):
(site / "pywin32.pth").write_text("win32\nwin32\\lib\nimport pywin32_bootstrap\n", encoding="utf-8")
(site / "_virtualenv.pth").write_text("import _virtualenv\n", encoding="utf-8")
(site / "__editable__.hermes_agent-0.21.1.pth").write_text(
"import __editable___hermes_agent_0_21_1_finder\n", encoding="utf-8"
)
prune_site_pth(win_venv)
prune_site_pth(posix_venv)
assert sorted(p.name for p in win_site.glob("*.pth")) == ["pywin32.pth"]
assert sorted(p.name for p in posix_site.glob("*.pth")) == ["pywin32.pth"]
@pytest.fixture
def locked_project(tmp_path):
uv = shutil.which("uv")

View File

@@ -91,6 +91,36 @@ def test_configure_prepends_repo_then_site_packages(tmp_path):
sys.path[:] = original
def test_configure_processes_site_pth_files(tmp_path):
"""site-packages goes through site.addsitedir(), not a raw append.
Only addsitedir() runs .pth files, and pywin32.pth is load-bearing on
Windows bundles: it puts win32\\lib on sys.path, which is what makes
`import pywintypes` resolve (portalocker's Win32Locker →
concurrent-log-handler → hermes_logging.py's file handlers). The old
raw sys.path[0:0] = entries skipped .pth processing entirely, silently
killing file logging on Windows bundles (and any future .pth-based
dependency on every platform).
"""
ns = _load()
here = str(tmp_path / "bin")
site = os.path.join(here, "..", "venv", "Lib", "site-packages")
os.makedirs(site)
added = tmp_path / "pth-added"
added.mkdir()
with open(os.path.join(site, "load-bearing.pth"), "w", encoding="utf-8") as f:
f.write(f"{added}\n")
original = list(sys.path)
try:
ns["configure"](here, environ={})
repo = os.path.join(here, "..", "repo")
# Repo first, site second, .pth-added dirs after both.
assert sys.path[:2] == [repo, site]
assert sys.path.index(str(added)) > 1
finally:
sys.path[:] = original
def test_configure_drops_inherited_pythonpath_and_pythonhome(tmp_path):
ns = _load()
original = list(sys.path)