fix(cron): process .pth files for Windows uv-venv script jobs (#86567)
_windows_cron_python_invocation bypasses the uv venv launcher (to avoid flashing a console window) and re-attaches the venv via PYTHONPATH — but PYTHONPATH entries are plain sys.path additions and never get .pth processing, so editable installs (pip install -e) were invisible to cron script jobs (ModuleNotFoundError). Bootstrap the script with site.addsitedir() on the venv site-packages, then exec it as __main__ via runpy.run_path, preserving the script directory on sys.path (python script.py semantics). Falls back to a plain invocation when the venv layout is unresolvable.
This commit is contained in:
SHT
Teknium
@@ -3106,6 +3106,42 @@ def _drain_script_pipes(proc: subprocess.Popen) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _windows_cron_bootstrap_argv(
|
||||
python_exe: str,
|
||||
env_overlay: dict[str, str],
|
||||
script_path: str,
|
||||
) -> list[str]:
|
||||
"""Bootstrap a cron script under the base interpreter with ``.pth`` support.
|
||||
|
||||
The uv-venv overlay mode runs the base ``python.exe`` (to avoid the
|
||||
launcher re-execing a console interpreter and flashing a window) and
|
||||
re-attaches the venv via ``PYTHONPATH``. But ``PYTHONPATH`` entries are
|
||||
plain ``sys.path`` additions — Python's site initialization never
|
||||
processes ``.pth`` files for them (only ``site.addsitedir()`` does) — so
|
||||
editable installs (``pip install -e``, ``__editable__*.pth`` links) are
|
||||
invisible to cron script jobs.
|
||||
|
||||
Bootstrap with ``site.addsitedir()`` on the venv ``site-packages``, then
|
||||
exec the script as ``__main__``. ``runpy.run_path`` keeps ``__file__``
|
||||
correct; ``sys.path[0]`` is set to the script's directory to preserve the
|
||||
``python script.py`` import semantics. Falls back to a plain invocation
|
||||
if the venv layout is unresolvable — the pre-existing PYTHONPATH
|
||||
behaviour is strictly better than failing to run at all.
|
||||
"""
|
||||
site_packages = Path(env_overlay.get("VIRTUAL_ENV", "")) / "Lib" / "site-packages"
|
||||
if not site_packages.is_dir():
|
||||
return [python_exe, script_path]
|
||||
bootstrap = (
|
||||
"import os, runpy, site, sys;"
|
||||
f"site.addsitedir({str(site_packages)!r});"
|
||||
"script = sys.argv[1];"
|
||||
"sys.argv = [script] + sys.argv[2:];"
|
||||
"sys.path.insert(0, os.path.dirname(os.path.abspath(script)));"
|
||||
"runpy.run_path(script, run_name='__main__')"
|
||||
)
|
||||
return [python_exe, "-c", bootstrap, script_path]
|
||||
|
||||
|
||||
def _run_job_script(
|
||||
script_path: str,
|
||||
workdir: Optional[str] = None,
|
||||
@@ -3207,7 +3243,13 @@ def _run_job_script(
|
||||
env_overlay: dict[str, str] = {}
|
||||
else:
|
||||
python_exe, env_overlay = _windows_cron_python_invocation(sys.executable)
|
||||
argv = [python_exe, str(path)]
|
||||
if env_overlay:
|
||||
# Overlay mode (Windows uv venv): PYTHONPATH alone cannot make
|
||||
# editable installs importable — .pth processing needs
|
||||
# site.addsitedir() (see _windows_cron_bootstrap_argv).
|
||||
argv = _windows_cron_bootstrap_argv(python_exe, env_overlay, str(path))
|
||||
else:
|
||||
argv = [python_exe, str(path)]
|
||||
|
||||
try:
|
||||
from tools.environments.local import build_subprocess_env
|
||||
|
||||
@@ -9,6 +9,7 @@ Tests cover:
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import textwrap
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -181,7 +182,15 @@ class TestRunJobScript:
|
||||
|
||||
assert success is True
|
||||
assert output == "ok"
|
||||
assert captured["argv"] == [str(base_python), str(script.resolve())]
|
||||
# Overlay mode bootstraps with site.addsitedir() so .pth files
|
||||
# (editable installs) are processed — plain PYTHONPATH cannot do that.
|
||||
assert captured["argv"][0] == str(base_python)
|
||||
assert captured["argv"][1] == "-c"
|
||||
assert "site.addsitedir" in captured["argv"][2]
|
||||
m = re.search(r"site\.addsitedir\('([^']*)'\)", captured["argv"][2])
|
||||
assert m is not None
|
||||
assert Path(m.group(1)) == site_packages
|
||||
assert captured["argv"][3] == str(script.resolve())
|
||||
# The script runner always adds CREATE_NEW_PROCESS_GROUP on win32 so a
|
||||
# cancel can taskkill the whole tree; on POSIX the getattr default is
|
||||
# 0 and the flag set is exactly windows_hide_flags().
|
||||
@@ -193,7 +202,83 @@ class TestRunJobScript:
|
||||
assert env["VIRTUAL_ENV"] == str(venv)
|
||||
assert str(site_packages) in env["PYTHONPATH"]
|
||||
|
||||
def test_bootstrap_argv_makes_pth_editable_installs_importable(self, cron_env, tmp_path):
|
||||
"""The bootstrap must process .pth files — the whole reason the
|
||||
overlay mode exists is that PYTHONPATH alone cannot (editable
|
||||
installs would raise ModuleNotFoundError in cron scripts)."""
|
||||
import subprocess
|
||||
|
||||
from cron.scheduler import _windows_cron_bootstrap_argv
|
||||
|
||||
venv = tmp_path / "venv"
|
||||
site_packages = venv / "Lib" / "site-packages"
|
||||
site_packages.mkdir(parents=True)
|
||||
# Simulate `pip install -e`: a .pth file pointing at a source dir.
|
||||
editable_src = tmp_path / "editable_pkg"
|
||||
editable_src.mkdir()
|
||||
(editable_src / "mypkg.py").write_text("VALUE = 42\n", encoding="utf-8")
|
||||
(site_packages / "editable.pth").write_text(
|
||||
str(editable_src) + "\n", encoding="utf-8"
|
||||
)
|
||||
|
||||
script = cron_env / "scripts" / "probe.py"
|
||||
script.write_text("import mypkg; print(mypkg.VALUE)\n", encoding="utf-8")
|
||||
|
||||
argv = _windows_cron_bootstrap_argv(
|
||||
sys.executable, {"VIRTUAL_ENV": str(venv)}, str(script)
|
||||
)
|
||||
# Run the bootstrap with the current interpreter (stands in for the
|
||||
# base python.exe on Windows; the semantics are interpreter-agnostic).
|
||||
result = subprocess.run(argv, capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stdout.strip() == "42"
|
||||
|
||||
def test_bootstrap_keeps_script_directory_on_sys_path(self, cron_env, tmp_path):
|
||||
"""`python script.py` puts the script's directory on sys.path, so a
|
||||
script may import a sibling module. The bootstrap must preserve that
|
||||
(runpy.run_path alone does not add it)."""
|
||||
import subprocess
|
||||
|
||||
from cron.scheduler import _windows_cron_bootstrap_argv
|
||||
|
||||
venv = tmp_path / "venv"
|
||||
site_packages = venv / "Lib" / "site-packages"
|
||||
site_packages.mkdir(parents=True)
|
||||
|
||||
(cron_env / "scripts" / "sibling_helper.py").write_text(
|
||||
"GREETING = 'sibling ok'\n", encoding="utf-8"
|
||||
)
|
||||
script = cron_env / "scripts" / "probe.py"
|
||||
script.write_text(
|
||||
"import sibling_helper; print(sibling_helper.GREETING)\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
argv = _windows_cron_bootstrap_argv(
|
||||
sys.executable, {"VIRTUAL_ENV": str(venv)}, str(script)
|
||||
)
|
||||
result = subprocess.run(argv, capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stdout.strip() == "sibling ok"
|
||||
|
||||
def test_bootstrap_argv_falls_back_without_site_packages(self, cron_env, tmp_path):
|
||||
"""Unresolvable venv layout must not break the run — fall back to a
|
||||
plain invocation (pre-existing PYTHONPATH behaviour)."""
|
||||
from cron.scheduler import _windows_cron_bootstrap_argv
|
||||
|
||||
script = cron_env / "scripts" / "probe.py"
|
||||
script.write_text('print("ok")\n', encoding="utf-8")
|
||||
|
||||
argv = _windows_cron_bootstrap_argv(
|
||||
sys.executable, {"VIRTUAL_ENV": str(tmp_path / "missing")}, str(script)
|
||||
)
|
||||
assert argv == [sys.executable, str(script)]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32",
|
||||
reason="Windows always takes the overlay/creationflags branch",
|
||||
)
|
||||
def test_non_windows_script_preserves_default_text_decoding(self, cron_env, monkeypatch):
|
||||
# No platform patching: the Linux CI host already takes this branch.
|
||||
from cron import scheduler as sched_mod
|
||||
@@ -234,6 +319,42 @@ class TestRunJobScript:
|
||||
assert "encoding" not in captured["kwargs"]
|
||||
assert "errors" not in captured["kwargs"]
|
||||
|
||||
def test_non_overlay_branch_keeps_plain_argv(self, cron_env, monkeypatch):
|
||||
"""When the Windows uv-venv overlay is NOT active, the invocation must
|
||||
stay a plain `python script.py` — the bootstrap is overlay-only.
|
||||
Cross-platform: forces the non-overlay branch explicitly."""
|
||||
from cron import scheduler as sched_mod
|
||||
from cron.scheduler import _run_job_script
|
||||
|
||||
script = cron_env / "scripts" / "probe.py"
|
||||
script.write_text('print("ok")\n', encoding="utf-8")
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakeProc:
|
||||
def __init__(self, argv, **kwargs):
|
||||
captured["argv"] = argv
|
||||
self.returncode = 0
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def communicate(self, timeout=None):
|
||||
return ("ok\n", "")
|
||||
|
||||
monkeypatch.setattr(
|
||||
sched_mod,
|
||||
"_windows_cron_python_invocation",
|
||||
lambda python_exe: (python_exe, {}),
|
||||
)
|
||||
monkeypatch.setattr(sched_mod.subprocess, "Popen", FakeProc)
|
||||
|
||||
success, output = _run_job_script("probe.py")
|
||||
|
||||
assert success is True
|
||||
assert output == "ok"
|
||||
assert captured["argv"] == [sys.executable, str(script.resolve())]
|
||||
|
||||
def test_emoji_stdout_round_trips_through_script_capture(self, cron_env):
|
||||
"""Emoji in script stdout must reach the caller intact (#42384).
|
||||
|
||||
|
||||
Reference in New Issue
Block a user