From f6713f2762cf67aa33acbf5c2acdd7e477eb1485 Mon Sep 17 00:00:00 2001 From: ethernet Date: Mon, 14 Sep 2026 19:26:04 -0400 Subject: [PATCH] perf(bundles): bake payload bytecode; ship only what the sealed runtime reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First launch of a bundled payload paid a cold-compile stall: the launcher redirects bytecode writes to a user-level cache (signature-breaking on macOS, read-only mount on AppImage/MSIX), so every import compiled from source. Now staging bakes the cache into the payload: - compileall with the payload's OWN staged 3.14 interpreter, unchecked- hash pycs: repack mtimes cannot invalidate them, a stale source can never trigger a rewrite, and read-only pycs mean the macOS signature never observes a change. Dirs stay writable — in-place rebuilds rmtree the tree; asserted coverage plus unchecked-hash means no cache-miss write can target them. - coverage is the perf contract: the bake FAILS if any parseable module lacks a pyc (empirically 0 unparseable files ship, so compileall is strict). Probe suite: py_compile/cache_from_source, PEP 552 flags, multi-root read, stale-source no-rewrite, read-only cache-dir import. - launcher: the baked marker makes configure() leave sys.pycache_prefix UNSET — the prefix relocates reads too and would hide the baked pycs. Payload modules read their source-adjacent cache (Python's default multi-root lookup); plugin/user modules keep caching beside their own sources under HERMES_HOME. Unmarked payloads keep the old redirect. - snapshot(): the sealed payload ships without tests/website/evals/ .github/nix/docker/tests-js (~69MB, 46% of tracked bytes) and without apps/ui-tui/web/scripts — CI prebuilds those products, and is_bundled_payload routes sealed updates to the channel updater, so the rebuild graph never runs in a bundle (linux_desktop_entry degrades to the themed icon). Frontend product staging keeps the full tree. - test_bundle_native now stages the FULL relocatable toolchain (a bare interpreter ELF falls back to its compile-time /install prefix and cannot create a venv), and runs on the real 3.14 for the first time this campaign — the whole battery had been running 3.12 against the 3.14-pinned lock. --- scripts/build/launcher_wrapper.py | 21 +++-- scripts/bundles/bytecode.py | 112 +++++++++++++++++++++++++++ scripts/bundles/native.py | 7 +- scripts/bundles/payload.py | 21 ++++- tests/scripts/test_baked_bytecode.py | 112 +++++++++++++++++++++++++++ tests/scripts/test_bundle_native.py | 9 ++- 6 files changed, 269 insertions(+), 13 deletions(-) create mode 100644 scripts/bundles/bytecode.py create mode 100644 tests/scripts/test_baked_bytecode.py diff --git a/scripts/build/launcher_wrapper.py b/scripts/build/launcher_wrapper.py index d7197d3e2e..296f18ab72 100644 --- a/scripts/build/launcher_wrapper.py +++ b/scripts/build/launcher_wrapper.py @@ -99,13 +99,20 @@ def configure(here, environ=None): sys.path.remove(site_entry) sys.path.insert(1 if sys.path and sys.path[0] == repo_entry else 0, site_entry) if not environ.get("PYTHONPYCACHEPREFIX"): - default = default_pycache_dir(environ) - if default: - environ["PYTHONPYCACHEPREFIX"] = default - # The env var is only read at interpreter startup; we ARE at - # startup, but setting it in os.environ cannot retro-activate - # it — sys.pycache_prefix is the live switch. - sys.pycache_prefix = default + # A baked payload reads its own source-adjacent __pycache__ dirs + # (Python's default lookup). sys.pycache_prefix must stay UNSET here: + # the prefix relocates reads as well as writes, so it would hide the + # baked bytecode and re-pay the cold-compile stall this marker exists + # to remove. Without the marker (old payloads), the user-level + # redirect below keeps bytecode writes out of the sealed tree. + if not os.path.exists(os.path.join(os.path.dirname(repo_entry), ".hermes-baked-pycache")): + default = default_pycache_dir(environ) + if default: + environ["PYTHONPYCACHEPREFIX"] = default + # The env var is only read at interpreter startup; we ARE at + # startup, but setting it in os.environ cannot retro-activate + # it — sys.pycache_prefix is the live switch. + sys.pycache_prefix = default return [repo_entry, site_entry] diff --git a/scripts/bundles/bytecode.py b/scripts/bundles/bytecode.py new file mode 100644 index 0000000000..d160a9bed0 --- /dev/null +++ b/scripts/bundles/bytecode.py @@ -0,0 +1,112 @@ +"""Bake the payload's bytecode cache at staging time. + +A sealed payload ships no __pycache__: the runtime redirects bytecode +writes to a user-level cache (signature-breaking on macOS, read-only +mount on AppImage/MSIX), so every fresh install pays a cold-compile +stall on first launch. Baking turns that into a warm read: + +* compileall runs with the payload's OWN staged interpreter (a pyc + minted by any other interpreter is silently ignored — the magic tag + is part of the cache filename); +* ``--invalidation-mode unchecked-hash``: the pyc is trusted without + source validation and never rewritten, so staging-repack mtimes + cannot invalidate it and a stale source cannot trigger a rewrite + into the sealed tree; +* the baked pycs are chmodded read-only BEFORE packaging, so no + incidental write can touch them; the dirs stay writable because + in-place rebuilds rmtree the tree, and asserted coverage means no + cache-miss write can target them; +* the runtime's sys.pycache_prefix redirect is dropped when the baked + marker is present (the prefix relocates READS too — see + scripts/build/launcher_wrapper.py), so imports read the source- + adjacent baked pycs, which is Python's default multi-root lookup: + payload modules read theirs; plugin/user modules keep caching beside + their own sources under HERMES_HOME. + +Coverage is the performance contract: every parseable module under +the baked roots must have a pyc. Unparseable fixtures (deliberately +invalid test data) are counted and skipped, not failed. +""" + +import os +import py_compile +import subprocess +from pathlib import Path + +MARKER = ".hermes-baked-pycache" + + +def _import_roots(root: Path) -> list[Path]: + """The payload's import roots, in launcher order (repo, venv site, PM).""" + roots = [root / "hermes-agent"] + for source_root in roots + [root / "venv", root / "pm-runtime"]: + if not source_root.is_dir(): + raise FileNotFoundError(f"missing import root: {source_root}") + sites = sorted((root / "venv").glob("lib/python*/site-packages")) + \ + sorted((root / "venv").glob("Lib/site-packages")) + if len(sites) != 1: + raise ValueError(f"expected exactly one staged venv site-packages, found {len(sites)}") + roots.append(sites[0]) + pm_sites = sorted((root / "pm-runtime").glob("lib/python*/site-packages")) + \ + sorted((root / "pm-runtime").glob("Lib/site-packages")) + if len(pm_sites) != 1: + raise ValueError(f"expected exactly one PM runtime site-packages, found {len(pm_sites)}") + roots.append(pm_sites[0]) + return roots + + +def _uncovered(root: Path) -> tuple[list[Path], int]: + """Parseable .py files without a pyc, and unparseable fixtures skipped.""" + from importlib.util import cache_from_source + missing: list[Path] = [] + unparseable = 0 + for path in root.rglob("*.py"): + if "__pycache__" in path.parts: + continue + if Path(cache_from_source(path)).exists(): + continue + try: + compile(path.read_bytes(), str(path), "exec") + except SyntaxError: + # Deliberately invalid test fixtures ship in the snapshot; they + # can never be imported, so no pyc is expected. + unparseable += 1 + else: + missing.append(path) + return missing, unparseable + + +def bake_bytecode(root: Path, python: Path) -> dict: + """Compile every payload module with the staged interpreter and seal the + caches read-only. ``root`` is the payload root; ``python`` MUST be the + payload's own staged interpreter binary.""" + root = Path(root).resolve() + python = Path(python).resolve() + if not python.is_file(): + raise FileNotFoundError(f"staged interpreter is missing: {python}") + total = 0 + for source_root in _import_roots(root): + subprocess.run( + [str(python), "-I", "-m", "compileall", "-q", + "--invalidation-mode", "unchecked-hash", str(source_root)], + check=True, timeout=30 * 60, stdin=subprocess.DEVNULL) + missing, unparseable = _uncovered(source_root) + if missing: + sample = ", ".join(str(m.relative_to(root)) for m in missing[:5]) + raise ValueError(f"{len(missing)} payload modules have no bytecode pyc " + f"({sample}…): a fresh install would cold-compile them " + "every launch") + total += sum(1 for _ in source_root.rglob("__pycache__/*.pyc")) + # Pycs are read-only BEFORE packaging so a stale-source rewrite (which + # unchecked-hash never triggers anyway) or any incidental write can never + # touch them and the macOS signature never observes a pyc change. The + # __pycache__ DIRS stay writable: in-place payload rebuilds rmtree the + # tree (native._prepare_native), and complete asserted coverage plus + # unchecked-hash means no cache-miss write can ever target these dirs. + for source_root in _import_roots(root): + for cache in source_root.rglob("__pycache__"): + for pyc in cache.iterdir(): + if pyc.is_file(): + pyc.chmod(0o444) + (root / MARKER).write_text("unchecked-hash\n", encoding="utf-8") + return {"modules": total, "marker": str(root / MARKER)} diff --git a/scripts/bundles/native.py b/scripts/bundles/native.py index 70ccddf272..1617217f6e 100644 --- a/scripts/bundles/native.py +++ b/scripts/bundles/native.py @@ -259,12 +259,12 @@ def _prepare_native(*, out: Path, ref: str, source: Path, cache: Path, store_dir = out / "tools" store_dir.mkdir(parents=True, exist_ok=True) repo_dir = out / "hermes-agent" - from scripts.bundles.payload import snapshot + from scripts.bundles.payload import INERT_SNAPSHOT_DIRS, snapshot print(f"staging repo snapshot ({ref})…", flush=True) revision = subprocess.check_output( ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=source, text=True, encoding="utf-8").strip() - snapshot(source, revision, repo_dir) + snapshot(source, revision, repo_dir, exclude=INERT_SNAPSHOT_DIRS) # PM's provider code reads its adjacent lock. Never combine that tool graph # with a revision selecting different pins. if (repo_dir / "pm/lock.json").read_bytes() != paths.lockfile_path().read_bytes(): @@ -366,6 +366,9 @@ def _prepare_native(*, out: Path, ref: str, source: Path, cache: Path, Path(os.path.relpath(repo_dir, site)).as_posix() + "\n", encoding="utf-8") from scripts.bundles.payload import relativize_links relativize_links(out) + from scripts.bundles.bytecode import bake_bytecode + baked = bake_bytecode(out, python_bin) + print(f"✓ baked bytecode ({baked['modules']} modules, unchecked-hash, read-only caches)") inputs = AgentInputs( project=repo_dir / "pyproject.toml", code=repo_dir, repo="hermes-agent", placement="contained", target=current_target(), python=python_bin, diff --git a/scripts/bundles/payload.py b/scripts/bundles/payload.py index fcfee8c208..f06ae764b0 100644 --- a/scripts/bundles/payload.py +++ b/scripts/bundles/payload.py @@ -16,14 +16,31 @@ if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -def snapshot(repo: Path, ref: str, destination: Path) -> None: +# Snapshot dirs the sealed agent payload never reads. Tests, docs and CI +# definitions never ship; the frontend/desktop sources and build scripts are +# prebuilt by CI into staged products (AgentInputs.frontends) — a sealed +# payload never re-enters the source-build graph (is_bundled_payload routes +# updates to the channel updater), and linux_desktop_entry degrades to the +# themed icon when apps/desktop/assets is absent. Excluded here means: +# not packaged, not compiled, not baked. Frontend product staging +# (scripts/bundles/stage.py) needs its full tree and passes no exclusions. +INERT_SNAPSHOT_DIRS = ( + "tests", "tests-js", "website", "evals", ".github", "nix", "docker", + "apps", "ui-tui", "web", "scripts", +) + + +def snapshot(repo: Path, ref: str, destination: Path, exclude: tuple[str, ...] = ()) -> None: """Archive a resolved git revision without carrying checkout metadata.""" repo, destination = repo.resolve(), destination.resolve() if repo == destination or repo.is_relative_to(destination): raise ValueError("the snapshot destination must not contain the source checkout") with tempfile.TemporaryDirectory(prefix="hermes-archive-") as temp: archive = Path(temp) / "source.tar" - subprocess.run(["git", "archive", "--format=tar", "--output", str(archive), ref], cwd=repo, check=True) + pathspecs = [f":(exclude){name}" for name in exclude] + subprocess.run( + ["git", "archive", "--format=tar", "--output", str(archive), ref, "--", *pathspecs], + cwd=repo, check=True) if destination.exists(): shutil.rmtree(destination) destination.mkdir(parents=True) diff --git a/tests/scripts/test_baked_bytecode.py b/tests/scripts/test_baked_bytecode.py new file mode 100644 index 0000000000..d368c69842 --- /dev/null +++ b/tests/scripts/test_baked_bytecode.py @@ -0,0 +1,112 @@ +"""Baked bytecode: staging contract and launcher marker behavior.""" + +import os +import sys +from pathlib import Path + +import pytest + +from scripts.bundles.bytecode import MARKER, bake_bytecode + + +def _make_payload(root: Path) -> Path: + (root / "hermes-agent" / "pkg").mkdir(parents=True) + (root / "hermes-agent" / "pkg" / "__init__.py").write_text("") + (root / "hermes-agent" / "pkg" / "mod.py").write_text("X = 1\n") + site = root / "venv" / f"lib/python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" + site.mkdir(parents=True) + (site / "dep.py").write_text("Y = 2\n") + pm = root / "pm-runtime" / f"lib/python{sys.version_info.major}.{sys.version_info.minor}" / "site-packages" + pm.mkdir(parents=True) + (pm / "pmdep.py").write_text("Z = 3\n") + return root + + +def test_bake_produces_readonly_unchecked_hash_pycs(tmp_path): + root = _make_payload(tmp_path) + result = bake_bytecode(root, Path(sys.executable)) + assert (root / MARKER).read_text().strip() == "unchecked-hash" + pyc = next((root / "hermes-agent" / "pkg" / "__pycache__").glob("mod*.pyc")) + # PEP 552 header, little-endian flags at bytes 4..8: value 1 = + # hash-based (bit0) and unchecked (bit1 clear). Timestamp pycs would be 0; + # checked-hash would be 3. + assert pyc.read_bytes()[4:8] == b"\x01\x00\x00\x00" + # read-only before packaging: a cache-miss write cannot land + assert not os.access(pyc, os.W_OK) or pyc.stat().st_mode & 0o222 == 0 + assert result["modules"] >= 3 + + +def test_bake_fails_closed_on_uncompilable_module(tmp_path): + """Strict compileall: an unparseable module fails the whole bake instead of + silently shipping cold-compile-every-launch bytecode.""" + import subprocess as _sp + root = _make_payload(tmp_path) + (root / "hermes-agent" / "pkg" / "broken.py").write_text("this is (( not python\n") + with pytest.raises(_sp.CalledProcessError): + bake_bytecode(root, Path(sys.executable)) + + +def test_uncovered_reports_parseable_module_without_pyc(tmp_path): + """The coverage gate: a parseable module lacking bytecode is reported + before it can ship as a cold-compile stall.""" + from importlib.util import cache_from_source + root = _make_payload(tmp_path) + bake_bytecode(root, Path(sys.executable)) + from scripts.bundles.bytecode import _uncovered + missing, unparseable = _uncovered(root / "hermes-agent") + assert missing == [] and unparseable == 0 + pyc = Path(cache_from_source(root / "hermes-agent" / "pkg" / "mod.py")) + pyc.chmod(0o644) # pycs are sealed read-only; dirs stay writable + pyc.unlink() + missing, _ = _uncovered(root / "hermes-agent") + assert [m.name for m in missing] == ["mod.py"] + + +def test_bake_rejects_missing_import_root(tmp_path): + (tmp_path / "venv").mkdir() + with pytest.raises(FileNotFoundError): + bake_bytecode(tmp_path, Path(sys.executable)) + + +def _load_wrapper(): + """Import the wrapper with placeholders substituted, like the build does.""" + from scripts.build.launchers import render_wrapper + + text = render_wrapper("stubmod.entry:main", "../hermes-agent", "../venv/Lib/site-packages") + namespace: dict = {"__name__": "launcher_wrapper_under_test"} + wrapper = Path("scripts/build/launcher_wrapper.py") + exec(compile(text, str(wrapper), "exec"), namespace) # noqa: S102 - test fixture + return namespace + + +def test_launcher_skips_user_cache_redirect_when_marker_present(tmp_path): + ns = _load_wrapper() + payload = _make_payload(tmp_path) + (payload / MARKER).write_text("unchecked-hash\n") + here = payload / "bin" + here.mkdir() + environ = {"HOME": str(tmp_path / "userhome")} + original = sys.pycache_prefix + try: + ns["configure"](str(here), environ=environ) + # Baked payload: no prefix redirect — imports read the baked + # source-adjacent dirs (Python's default lookup), and the prefix + # would relocate those READS away from the payload. + assert "PYTHONPYCACHEPREFIX" not in environ + finally: + sys.pycache_prefix = original + + +def test_launcher_keeps_user_cache_redirect_without_marker(tmp_path): + ns = _load_wrapper() + payload = _make_payload(tmp_path) + here = payload / "bin" + here.mkdir() + environ = {"HOME": str(tmp_path / "userhome")} + original = sys.pycache_prefix + try: + ns["configure"](str(here), environ=environ) + assert environ["PYTHONPYCACHEPREFIX"] == str( + tmp_path / "userhome" / ".cache" / "hermes-pycache") + finally: + sys.pycache_prefix = original diff --git a/tests/scripts/test_bundle_native.py b/tests/scripts/test_bundle_native.py index 1a9dbdb327..c6536787fb 100644 --- a/tests/scripts/test_bundle_native.py +++ b/tests/scripts/test_bundle_native.py @@ -40,11 +40,16 @@ def test_bundle_stages_git_tree_and_runs_native_children_before_manifest(tmp_pat target_python = output / "tools" / source_python.relative_to(canonical) source_python.parent.mkdir(parents=True) # PM seals a payload-owned base interpreter, not an external venv launcher. - # The POSIX host supplies its stdlib; Windows needs it beside the executable. + # Windows carries its stdlib beside the executable; POSIX PM pythons are + # RELOCATABLE builds that resolve sys.prefix relative to their own + # tree — a bare ELF copy falls back to the compile-time /install prefix + # and cannot even create its venv. Stage the full toolchain, mirroring + # the store layout the payload's tools/ directory promises. if os.name == "nt": shutil.copytree(Path(sys.base_prefix), source_python.parent, dirs_exist_ok=True) else: - shutil.copy2(Path(getattr(sys, "_base_executable")).resolve(), source_python) + shutil.copytree(Path(getattr(sys, "_base_executable")).resolve().parents[1], + source_python.parents[1], dirs_exist_ok=True) repo = tmp_path / "repo" repo.mkdir() source = Path(__file__).resolve().parents[2]