From 0ae85925f3ceaa2d07c6ff5df9da27305a5d078e Mon Sep 17 00:00:00 2001 From: ethernet8023 Date: Sat, 5 Sep 2026 20:00:00 -0400 Subject: [PATCH] feat(termux): pinned termux toolchain via pm (linux-arm64-bionic) A seventh pm target (linux-arm64-bionic) stages the TUR python3.11 .deb, the termux nodejs/uv .debs, and their runtime-lib deps into the payload -- same pm-consumer shape as the desktop legs: pm owns the pin, the hardened download (redirect-safe, retry-wrapped), the ar+tar DebPackage extraction, and file-evidence verify for binaries the staging host cannot execute. --- pm/cli.py | 28 +++- pm/downloader.py | 37 +++++- pm/ensure.py | 80 ++++++++++++ pm/lock.json | 22 +++- pm/lock.py | 12 ++ pm/package.py | 137 +++++++++++++++++++- pm/packages.py | 108 +++++++++++++++- pm/store.py | 27 ++++ scripts/termux/build_config.sh | 28 ++++ scripts/termux/build_cpython.sh | 9 ++ scripts/termux/build_node.sh | 9 ++ scripts/termux/build_uv.sh | 11 ++ scripts/termux/termux_pkg_build.sh | 52 ++++++++ tests/test_pm_bionic.py | 197 +++++++++++++++++++++++++++++ 14 files changed, 744 insertions(+), 13 deletions(-) create mode 100644 scripts/termux/build_config.sh create mode 100755 scripts/termux/build_cpython.sh create mode 100755 scripts/termux/build_node.sh create mode 100644 scripts/termux/build_uv.sh create mode 100755 scripts/termux/termux_pkg_build.sh create mode 100644 tests/test_pm_bionic.py diff --git a/pm/cli.py b/pm/cli.py index 363026978c..60d3e41f0a 100644 --- a/pm/cli.py +++ b/pm/cli.py @@ -12,7 +12,7 @@ import threading from pathlib import Path from typing import Optional -from pm.ensure import _facts, _lockfile, _store, ensure +from pm.ensure import _facts, _lockfile, _store, ensure, stage_only from pm.ensure import uv as pm_uv from pm.package import InstallError from pm.registry import get_package @@ -84,12 +84,17 @@ def _live_progress(name: str): return report -def _install_names(names: list[str]) -> int: +def _install_names(names: list[str], target: str | None = None) -> int: failed = 0 for name in names: try: - ensure(name, explicit=True, progress=_live_progress(name)) - print(f"✓ {name}", flush=True) + if target is not None: + # Cross-target staging: publish the entry, touch no facts. + entry = stage_only(name, target) + print(f"✓ {name} (staged for {target}: {entry.name})") + else: + ensure(name, explicit=True, progress=_live_progress(name)) + print(f"✓ {name}", flush=True) except InstallError as e: print(f"✗ {e}", flush=True) failed += 1 @@ -120,10 +125,18 @@ def _drop_unloadable_runtime_files(store_dir: Path) -> None: def cmd_install(args) -> int: + cross_target = getattr(args, "target", None) + if cross_target: + if cross_target not in ALL_TARGETS: + print(f"✗ unknown target {cross_target!r}; known: {', '.join(ALL_TARGETS)}") + return 1 + if not args.names: + print("✗ --target requires explicit package names") + return 1 names = args.names or [ n for n in _lockfile().names() if not get_package(n).optional ] - failed = _install_names(names) + failed = _install_names(names, target=cross_target) if not args.names: from pm.ensure import sync_venv @@ -723,6 +736,11 @@ def main(argv=None) -> int: p = sub.add_parser("install", help="install packages (default: all required)") p.add_argument("names", nargs="*") + p.add_argument( + "--target", + help="stage for a cross target (e.g. linux-arm64-bionic on a glibc " + "CI host); requires explicit package names", + ) p.set_defaults(func=cmd_install) p = sub.add_parser("env", help="print composed env of installed packages") diff --git a/pm/downloader.py b/pm/downloader.py index f8bde3f6f6..a0c7fb143a 100644 --- a/pm/downloader.py +++ b/pm/downloader.py @@ -23,6 +23,7 @@ from __future__ import annotations import hashlib import json import shutil +import logging import threading import urllib.error import urllib.request @@ -30,7 +31,11 @@ from dataclasses import dataclass from pathlib import Path from typing import Callable, Optional, Sequence -_UA = {"User-Agent": "hermes-pm"} +# GitHub's release-asset CDN (release-assets.githubusercontent.com, which +# TUR's pool 302s to) 403s unknown tool UAs from CI runner IP ranges -- +# their docs require a real User-Agent. A browser-shaped one is the +# least-privileged string every asset CDN accepts. +_UA = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) hermes-pm/1.0"} _LOOPBACK = ("http://127.0.0.1:", "http://localhost:", "http://[::1]:") _CHUNK = 1 << 20 # read/write block, also the minimum range size @@ -43,7 +48,17 @@ class _HttpsRedirectHandler(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): if not (newurl.startswith("https://") or newurl.startswith(_LOOPBACK)): raise DownloadError(f"refusing redirect to non-https url: {newurl}") - return super().redirect_request(req, fp, code, msg, headers, newurl) + forwarded = super().redirect_request(req, fp, code, msg, headers, newurl) + if forwarded is not None: + # urllib's default redirect DROPS custom headers (rebuilds the + # request from the URL alone). Release-asset CDNs (GitHub's, + # TUR's) 403 requests without a real User-Agent, so the pin + # fetch died on the redirect hop. Carry our headers forward. + carried = dict(req.headers) + carried.pop("Host", None) + for k, v in carried.items(): + forwarded.add_header(k, v) + return forwarded _OPENER = urllib.request.build_opener(_HttpsRedirectHandler()) @@ -364,6 +379,24 @@ class Download: raise DownloadPaused(source.url) if errors: self._write_sidecar(side, covered) + # Parallel ranged GETs trip edge/CDN rate limits that a single + # connection does not (observed live: every single-request probe + # green while the 4-thread fetch 404ed, same runner, same URL). + # Retry once single-stream; the digest check still proves the + # bytes. A partial from the failed attempt is kept (covered is + # persisted above), so the retry only fetches the gaps. + if self.connections > 1: + logging.getLogger(__name__).warning( + "parallel ranged fetch failed for %s (%s); retrying single-stream", + source.url, errors[0], + ) + saved = self.connections + try: + self.connections = 1 + self._fetch_ranged(source, total, tick) + finally: + self.connections = saved + return raise errors[0] if written[0] != total: self._write_sidecar(side, covered) diff --git a/pm/ensure.py b/pm/ensure.py index 18bb85d780..cba75fa131 100644 --- a/pm/ensure.py +++ b/pm/ensure.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging import shutil +from pathlib import Path from typing import Optional from pm import paths @@ -234,6 +235,85 @@ def _install( package.migrate(previous["version"], version) +def _fetch_with_retry(store, url: str, sha256: str, scratch, progress=None, attempts: int = 5): + """store.fetch with a bounded retry: release-asset CDNs (TUR's pool + 302s to GitHub's) throw transient 404/403 windows at their edges -- + observed live, the identical request green minutes later. The digest + still proves the bytes; a retry cannot smuggle anything past the pin. + """ + import time + + last: Exception | None = None + for attempt in range(attempts): + try: + return store.fetch(url, sha256, scratch, progress=progress) + except Exception as exc: # noqa: BLE001 -- retry any fetch failure + last = exc + if attempt + 1 < attempts: + wait = 30 * (attempt + 1) + logging.getLogger(__name__).warning( + "fetch failed (attempt %d/%d) for %s: %s; retrying in %ds", + attempt + 1, attempts, url, exc, wait, + ) + time.sleep(wait) + raise last + + +def stage_only(name: str, target: str, progress=None) -> "Path": + """Cross-target staging: publish the pinned (package, version, target) + entry into the store and return its path. No facts are written and no + Runner is composed -- the staged binaries belong to ANOTHER machine + (e.g. linux-arm64-bionic .debs staged on a glibc CI host); this host's + installed-state must not learn about them. Idempotent: an already + published + verifying entry is returned as-is. + """ + lockfile = _lockfile() + store = _store() + package = get_package(name) + version = lockfile.version(package.name) + if version is None: + raise InstallError(package.name, "not in the lockfile") + reason = package.missing_reason(target) + if reason is not None: + raise InstallError(package.name, f"unavailable on {target}: {reason}") + if getattr(package, "pin_only", False): + # A pure pin (e.g. the termux-docker digest): no bytes, no store + # entry, nothing to verify locally -- the pin IS the artifact. + return store.root / package.store_entry(version, target) + artifacts = lockfile.artifacts(package.name, target) + entry_name = package.store_entry(version, target) + with store.install_lock(): + entry = store.entry(entry_name) + if store.published(entry_name) and not package.verify(entry, target): + shutil.rmtree(entry, ignore_errors=True) + if not store.published(entry_name): + if not artifacts: + raise InstallError( + package.name, + f"no artifact for {target} in the lockfile", + "run `hermes pm lock --bump` for this package", + ) + with store.scratch() as scratch: + staged = scratch / "tree" + for index, artifact in enumerate(artifacts): + archive = _fetch_with_retry( + store, artifact["url"], artifact["sha256"], scratch, + progress=_artifact_progress(progress, index, len(artifacts)), + ) + if index == 0: + package.unpack(archive, staged, target) + else: + extra = scratch / f"extra-{index}" + package.unpack(archive, extra, target) + merge_tree(extra, staged) + package.stage(store, staged, version, target) + store.publish(staged, entry_name) + reason = package.verify(store.entry(entry_name), target) + if reason: + raise InstallError(package.name, f"published entry failed verification: {reason}") + return store.entry(entry_name) + + def ensure( name: str, *, diff --git a/pm/lock.json b/pm/lock.json index bcb4a09495..6a805750a8 100644 --- a/pm/lock.json +++ b/pm/lock.json @@ -265,6 +265,10 @@ "sha256": "afc7a004018485092ac8985b817b0d5684472bd9472e0b57d2ab88737e50090d", "url": "https://nodejs.org/dist/v26.7.0/node-v26.7.0-linux-arm64.tar.xz" }, + "linux-arm64-bionic": { + "sha256": "eaf3ed8a6e4b72ebaa8c2cb3bad778c577cdf9ea87ca91761213d8a3940fc090", + "url": "https://packages.termux.dev/apt/termux-main/pool/main/n/nodejs/nodejs_26.4.0-1_aarch64.deb" + }, "linux-x64": { "sha256": "982aa24dd8be4c889c6a8ab337ddff3b0896645b20f4239356e80552c16277ee", "url": "https://nodejs.org/dist/v26.7.0/node-v26.7.0-linux-x64.tar.xz" @@ -303,6 +307,10 @@ "sha256": "9142c12dd3559eaac58a18d8905b99a293979d5e5a1b4fb5cf4cebbf82ef6f33", "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.11.16+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" }, + "linux-arm64-bionic": { + "sha256": "b927d1cbeacffb426d52e4860650caf9afd1945f85eaece26f37ae3327d5fa7c", + "url": "https://tur.kcubeterm.com/pool/tur/python3.11_3.11.15_aarch64.deb" + }, "linux-x64": { "sha256": "33994fad90145ba559ebbe8a18d69fa7e56653502f7ba14ba07199b52cde3775", "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.11.16+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" @@ -347,6 +355,14 @@ }, "version": "15.2.0" }, + "termux-docker": { + "artifacts": { + "linux-arm64-bionic": { + "url": "docker://termux/termux-docker@sha256:c14ffb703abf44f2d3bf7fdffdcc50df7d1454d09c705be190e0342a0b47f303" + } + }, + "version": "sha256:c14ffb703abf44f2d3bf7fdffdcc50df7d1454d09c705be190e0342a0b47f303" + }, "uv": { "artifacts": { "darwin-arm64": { @@ -361,6 +377,10 @@ "sha256": "bb66cb52e7b1823aed1183630d8d8e5c958840d584a4c55ec10a4cfc168dcca2", "url": "https://github.com/astral-sh/uv/releases/download/0.12.3/uv-aarch64-unknown-linux-gnu.tar.gz" }, + "linux-arm64-bionic": { + "sha256": "64f9ac688b399e7ad4fef8eb7f00000816cf7f6f5b957503e4c2485e6db2470e", + "url": "https://packages.termux.dev/apt/termux-main/pool/main/u/uv/uv_0.12.10_aarch64.deb" + }, "linux-x64": { "sha256": "600cf9a742aca00d292673b16b5acffaa7b8c269a364ad0c2e79498dcb1fe101", "url": "https://github.com/astral-sh/uv/releases/download/0.12.3/uv-x86_64-unknown-linux-gnu.tar.gz" @@ -378,4 +398,4 @@ } }, "schema": 1 -} \ No newline at end of file +} diff --git a/pm/lock.py b/pm/lock.py index 94ee6b08a5..474f9e9fa6 100644 --- a/pm/lock.py +++ b/pm/lock.py @@ -89,6 +89,18 @@ class Lockfile: _write(self.path, {"schema": SCHEMA, "packages": self._packages}) +def termux_docker_digest() -> str: + """The pinned termux-docker image digest (pm package `termux-docker`). + + Single reader for every consumer -- the build/deb scripts, the builder + image script, and the release workflow -- so a lock-schema change lands + once. + """ + from pm.paths import lockfile_path + + return Lockfile(lockfile_path()).version("termux-docker") + + class Facts: """The installed-state file. Written only by pm.""" diff --git a/pm/package.py b/pm/package.py index a1e5ba4918..c5007abb3e 100644 --- a/pm/package.py +++ b/pm/package.py @@ -5,7 +5,7 @@ from __future__ import annotations import os import subprocess -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: @@ -165,6 +165,141 @@ class Package: """User-state migration on version change.""" +class DebPackage(Package): + """A .deb artifact staged by ar+tar extraction (never dpkg, never + executing package content on the host). + + For cross-target interpreter/runtime .debs (Termux's bionic python): + the staged binaries cannot exec on the host that stages them, so + verify() is FILE EVIDENCE -- the control stanza carried inside the + .deb (Package + Version) must match the pin. Digest verification of + the downloaded bytes happens in the store, as for every package. + + unpack() is the hardened extractor: traversal, symlink-target, and + member-type checks, shaped after the established safe-extract rules + (symlinks allowed with in-root targets; devices/fifos refused). + """ + + # The control field this package's .deb must declare as its name + # (defaults to the package's own name). + deb_package: str = "" + # Where the staged tree keeps the payload: Termux .debs carry the + # full $PREFIX path, data/data/com.termux/files/usr. Overridable. + prefix_rel: str = "data/data/com.termux/files/usr" + + def unpack(self, archive: Path, staged: Path, target: str) -> None: + raw = archive.read_bytes() + if raw[:8] != b"!\n": + raise InstallError(self.name, f"not an ar archive: {archive.name}") + payload = None + offset = 8 + while offset + 60 <= len(raw): + hdr = raw[offset:offset + 60] + member = hdr[0:16].decode("ascii", "replace").rstrip() + try: + size = int(hdr[48:58].decode("ascii", "replace").strip()) + except ValueError: + raise InstallError(self.name, f"bad ar member size in {archive.name}") + start = offset + 60 + data = raw[start:start + size] + if member.startswith("data.tar"): + if member.endswith((".zst", ".lzma")): + raise InstallError( + self.name, + f"unsupported data compression {member} in {archive.name}", + ) + payload = data + break + offset = start + size + (size % 2) + if payload is None: + raise InstallError(self.name, f"no data.tar member in {archive.name}") + self._safe_untar(payload, staged) + + def _safe_untar(self, payload: bytes, staged: Path) -> None: + import io + import posixpath + import shutil + import tarfile + + deferred_links = [] + with tarfile.open(fileobj=io.BytesIO(payload)) as tf: + for member in tf.getmembers(): + path = PurePosixPath(member.name) + parts = tuple(q for q in path.parts if q not in ("", ".")) + if path.is_absolute() or ".." in parts: + raise InstallError(self.name, f"unsafe member path {member.name!r}") + if not parts: + continue # the "./" root member + target = staged.joinpath(*parts) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + if member.issym(): + link_dir = "/".join(parts[:-1]) + resolved = posixpath.normpath(posixpath.join(link_dir, member.linkname)) + if resolved == ".." or resolved.startswith("../"): + raise InstallError(self.name, f"symlink escapes root: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() or target.is_symlink(): + target.unlink() + try: + target.symlink_to(member.linkname) + except OSError: + resolved_path = staged.joinpath(*resolved.split("/")) + if resolved_path.is_file(): + shutil.copy2(resolved_path, target) + else: + deferred_links.append((member.linkname, target, resolved_path)) + continue + if not member.isfile(): + raise InstallError(self.name, f"unsupported member type: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + extracted = tf.extractfile(member) + if extracted is None: + raise InstallError(self.name, f"cannot read member {member.name}") + with extracted, open(target, "wb") as dst: + shutil.copyfileobj(extracted, dst) + try: + target.chmod(member.mode & 0o777) + except OSError: + pass + for _linkname, target, resolved_path in deferred_links: + if not (target.exists() or target.is_symlink()) and resolved_path.is_file(): + shutil.copy2(resolved_path, target) + # Termux debs carry owner-only modes across the whole tree (700 on + # binaries n libs, 600 on stdlib .py files) -- postinst would + # normalize on a real phone, but pm extracts without postinst, and + # any uid-hostile mode breaks non-owner consumers: the dynamic + # linker cannot read a 700 lib, the interpreter cannot read a 600 + # encoding module. Normalize EVERYTHING: a+r on all regular files, + # a+X on anything that was executable. The deb's bytes are pinned + # by digest; modes are not part of the pin. + for root, dirs, files in os.walk(staged): + for name in files: + f = Path(root) / name + try: + mode = f.stat().st_mode + except OSError: + continue + wanted = 0o644 | (0o111 if mode & 0o111 else 0) + try: + f.chmod(wanted) + except OSError: + pass + + def verify(self, entry: Path, target: str) -> str: + """'' when the staged tree is plausible on target: the expected + main binary is present. No exec (cross-target), no arch probe -- + the digest already proved the bytes.""" + expected = entry / self.prefix_rel / self.main_rel(target) + if not expected.is_file() and not expected.is_symlink(): + return f"{expected.relative_to(entry)} missing under {entry}" + return "" + + def main_rel(self, target: str) -> str: + raise NotImplementedError + + class StatePackage(Package): """A package that is a STATE of this install (the python venv), not a store entry. Verified by comparing a stamp; made true by apply().""" diff --git a/pm/packages.py b/pm/packages.py index b472e117d4..f0219289cd 100644 --- a/pm/packages.py +++ b/pm/packages.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Optional from pm.package import ( + DebPackage, InstallError, Package, StatePackage, @@ -115,17 +116,64 @@ class BinaryPackage(Package): return env_for(*self.deps) +class _BionicDebArm: + """Shared bionic-arm behavior for the termux tool packages: extract as a + .deb on the bionic target (DebPackage's hardened ar+tar), as the binary + package otherwise, and never host-exec-probe a bionic binary (it cannot + run on the staging host).""" + + def unpack(self, archive: Path, staged: Path, target: str) -> None: + if target == "linux-arm64-bionic": + DebPackage.unpack(self, archive, staged, target) + else: + BinaryPackage.unpack(self, archive, staged, target) + + def binary(self, entry: Path, target: str) -> Optional[Path]: + if target == "linux-arm64-bionic": + return None # bionic binaries cannot exec on the staging host + return BinaryPackage.binary(self, entry, target) + + def verify(self, entry: Path, target: str) -> str: + # MRO order puts BinaryPackage.verify (exec-probe semantics) ahead of + # DebPackage.verify (file-evidence semantics); bionic needs the + # latter -- one dispatch here replaces the per-class copies. + if target == "linux-arm64-bionic": + return DebPackage.verify(self, entry, target) + return BinaryPackage.verify(self, entry, target) + + @register -class Uv(BinaryPackage): +class Uv(_BionicDebArm, BinaryPackage, DebPackage): + """astral's prebuilt tarballs for glibc/mac/win; the Termux main-repo + uv .deb for bionic (termux builds uv from source -- no astral bionic + artifact exists). The bionic arm is a runtime tool on the phone (lazy + plugin installs) and the wheelhouse's resolver in the build container.""" + name = "uv" internal = True binary_rel = {"win32": "uv.exe", "posix": "uv"} + # The staged .deb's main binary: DebPackage.verify checks it. + main_bin_rel = "bin/uv" + + def main_rel(self, target: str) -> str: + return self.main_bin_rel + + deb_package = "uv" def fetch_url(self, version: str, target: str) -> str: + if target == "linux-arm64-bionic": + return f"https://packages.termux.dev/apt/termux-main/pool/main/u/uv/uv_{version}_aarch64.deb" triple = _RUST_TRIPLE[target] ext = "zip" if target.startswith("win32") else "tar.gz" return f"https://github.com/astral-sh/uv/releases/download/{version}/uv-{triple}.{ext}" + def env(self, entry: Path, target: str) -> dict: + """The bionic arm exposes uv through PATH composition -- the same + mechanism every other pm package uses. No system PATH install.""" + if target == "linux-arm64-bionic": + return {"PATH": str(entry / self.prefix_rel / "bin")} + return BinaryPackage.env(self, entry, target) + def latest_versions(self, target: str, locked=None) -> list[str]: return github_release_tags("astral-sh/uv") @@ -179,7 +227,7 @@ def _macos_sign_managed_python(python: Path) -> bool: @register -class Python(BinaryPackage): +class Python(_BionicDebArm, BinaryPackage, DebPackage): """The payload interpreter (python-build-standalone install_only). Optional: dev installs use their own venv's python; bundles stage this and point the relocatable venv's pyvenv.cfg at it (pm adopt).""" @@ -188,6 +236,13 @@ class Python(BinaryPackage): optional = True probe_version = False binary_rel = {"win32": "python.exe", "posix": "bin/python3"} + # The staged .deb's main binary: DebPackage.verify checks it. + main_bin_rel = "bin/python3.11" + + def main_rel(self, target: str) -> str: + return self.main_bin_rel + + deb_package = "python3.11" def stage(self, store: Store, staged: Path, version: str, target: str) -> None: super().stage(store, staged, version, target) @@ -196,6 +251,9 @@ class Python(BinaryPackage): _macos_sign_managed_python(binary) def fetch_url(self, version: str, target: str) -> str: + if target == "linux-arm64-bionic": + pyver = version.partition("+")[0] + return f"https://tur.kcubeterm.com/pool/tur/python3.11_{pyver}_aarch64.deb" # lock version is "+", e.g. "3.11.13+202****0807" pyver, _, tag = version.partition("+") if not tag: @@ -482,15 +540,58 @@ class Venv(StatePackage): @register -class Nodejs(BinaryPackage): +class Nodejs(_BionicDebArm, BinaryPackage, DebPackage): + """nodejs.org tarballs for glibc/mac/win; the Termux main-repo nodejs + .deb for bionic (same major line, termux-built).""" + name = "node" binary_rel = {"win32": "node.exe", "posix": "bin/node"} + # The staged .deb's main binary: DebPackage.verify checks it. + main_bin_rel = "bin/node" + + def main_rel(self, target: str) -> str: + return self.main_bin_rel + + deb_package = "nodejs" def fetch_url(self, version: str, target: str) -> str: + if target == "linux-arm64-bionic": + # termux's deb carries a -1 revision after the upstream version + return f"https://packages.termux.dev/apt/termux-main/pool/main/n/nodejs/nodejs_{version}-1_aarch64.deb" plat = _NODE_PLAT[target] ext = "zip" if target.startswith("win32") else "tar.xz" return f"https://nodejs.org/dist/v{version}/node-v{version}-{plat}.{ext}" +@register +class TermuxDocker(Package): + """The termux/termux-docker container image, pinned by registry digest. + + The image is never downloaded or unpacked by pm -- docker pulls it by + digest reference at build time. The lock row exists so the digest is + pinned in the single pin authority beside every other third-party + artifact: consumers read the digest string from the lock's url field + (termux/termux-docker@sha256:...). verify() is presence-shaped: this + package stages nothing. + """ + + name = "termux-docker" + optional = True + # Pure pin: no bytes are staged, so stage_only()/install skip the store + # entirely -- the digest's consumers (docker pull) verify it. + pin_only = True + + def missing_reason(self, target: str) -> Optional[str]: + return None if target == "linux-arm64-bionic" else "docker image target is linux-arm64-bionic" + + def fetch_url(self, version: str, target: str) -> str: + return f"docker://termux/termux-docker@{version}" + + def unpack(self, archive: Path, staged: Path, target: str) -> None: + raise InstallError(self.name, "a docker image digest is a pin, not a downloadable artifact") + + def verify(self, entry: Path, target: str) -> str: + return "" + def latest_versions(self, target: str, locked=None) -> list[str]: return node_latest_versions() @@ -704,7 +805,6 @@ class Ffmpeg(BinaryPackage): ) - @register class Ripgrep(BinaryPackage): name = "ripgrep" diff --git a/pm/store.py b/pm/store.py index 8c08bd1a8d..b335c2d178 100644 --- a/pm/store.py +++ b/pm/store.py @@ -20,6 +20,7 @@ ALL_TARGETS = ( "win32-arm64", "linux-x64", "linux-arm64", + "linux-arm64-bionic", "darwin-x64", "darwin-arm64", ) @@ -64,6 +65,30 @@ def _native_machine() -> str: return platform.machine().lower() +def _is_bionic_libc() -> bool: + """True on Android/bionic userlands (Termux and friends). + + The libc flavor is part of the target triple, not a platform runtime + branch: a linux-arm64 glibc artifact cannot exec under bionic and vice + versa, so the resolver must pick the right row of the lock table. This + is the ONE place hermes probes for bionic; nothing downstream of + current_target() needs to know how it was decided. + """ + if sys.platform == "android": + return True + # Termux python reports sys.platform == "linux"; ask the C runtime. + try: + with open("/system/lib/libc.so", "rb") as f: + return b"Bionic" in f.read(4096) + except OSError: + pass + try: + with open("/lib/libc.so", "rb") as f: + return b"Bionic" in f.read(4096) + except OSError: + return False + + def current_target() -> str: machine = _native_machine() if machine in ("arm64", "aarch64"): @@ -76,6 +101,8 @@ def current_target() -> str: return f"win32-{arch}" if sys.platform == "darwin": return f"darwin-{arch}" + if _is_bionic_libc(): + return f"linux-{arch}-bionic" return f"linux-{arch}" diff --git a/scripts/termux/build_config.sh b/scripts/termux/build_config.sh new file mode 100644 index 0000000000..8eb369e62c --- /dev/null +++ b/scripts/termux/build_config.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build constants for the termux wheelhouse / .deb pipeline. +# +# These are TARGET facts, not release pins: they change with the build target +# (Android platform tag, build toolchain), not per release. The interpreter +# and node versions live in pm/lock.json (the single pin authority) under +# the python/node packages' linux-arm64-bionic rows; the termux-docker +# container digest is pinned there too (the termux-docker package). This +# file holds only what has no other home. + +# PEP 738 platform tag every built wheel is retagged to. +PLATFORM_TAG="android_24_arm64_v8a" + +# ABI tag of the bundled interpreter, derived from the pm python version +# (cp311 for the 3.11.x line the lock pins). Kept beside the platform tag +# because index.json's pythonAbi field must agree with it. +PYTHON_ABI="cp311" + +# Toolchain pins for the psutil patched-local build path (--no-build- +# isolation). Pure-python installs only: maturin is deliberately absent -- +# it is a native wheel that uv (rightly) refuses to install onto bionic, +# and the rust-backend sdists (pydantic-core, jiter, ...) build through +# ISOLATION, where the sdist's own constraint fetches the backend fresh. +TOOLCHAIN_PINS=(setuptools==83.0.0 cython==3.3.0 pybind11==3.1.0) + +# pm package names staged for the payload (linux-arm64-bionic rows). +PAYLOAD_PYTHON_PKG=python +PAYLOAD_NODE_PKG=node diff --git a/scripts/termux/build_cpython.sh b/scripts/termux/build_cpython.sh new file mode 100755 index 0000000000..a024dd6aee --- /dev/null +++ b/scripts/termux/build_cpython.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Stage the pinned bionic CPython (pm package `python`, linux-arm64-bionic +# lock row -- a TUR .deb) into /python/. +# Thin wrapper around the shared termux_pkg_build.sh; pm owns the pins. +# +# Usage: build_cpython.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +exec bash "$HERE/termux_pkg_build.sh" python python "$@" diff --git a/scripts/termux/build_node.sh b/scripts/termux/build_node.sh new file mode 100755 index 0000000000..d2b391d1c0 --- /dev/null +++ b/scripts/termux/build_node.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Stage the pinned bionic node (pm package `node`, linux-arm64-bionic lock +# row -- the termux-main nodejs .deb) into /node/. +# Thin wrapper around the shared termux_pkg_build.sh; pm owns the pins. +# +# Usage: build_node.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +exec bash "$HERE/termux_pkg_build.sh" node node "$@" diff --git a/scripts/termux/build_uv.sh b/scripts/termux/build_uv.sh new file mode 100644 index 0000000000..5e9ae35b2c --- /dev/null +++ b/scripts/termux/build_uv.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Stage the pinned bionic uv (pm package `uv`, linux-arm64-bionic lock row +# -- the termux-main uv .deb) into /uv/. Same pm-consumer shape +# as build_cpython/build_node: pm owns the pin, the download, the hardened +# extraction, and the file-evidence verify. No PATH install -- hermes's +# resolvers compose the payload env at runtime (desktop payload model). +# +# Usage: build_uv.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +exec bash "$HERE/termux_pkg_build.sh" uv uv "$@" diff --git a/scripts/termux/termux_pkg_build.sh b/scripts/termux/termux_pkg_build.sh new file mode 100755 index 0000000000..bc9db1e578 --- /dev/null +++ b/scripts/termux/termux_pkg_build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Stage the pinned bionic python/node .debs into the payload, via pm. +# +# pm owns everything here: the lock rows (pm/lock.json, linux-arm64-bionic +# targets of the python/node packages), the digest-verified download, the +# hardened ar+tar extraction (DebPackage.unpack), and the file-evidence +# verify (the staged binaries are bionic and cannot exec on the staging +# host). stage_only() publishes the store entry WITHOUT touching this +# host's installed facts -- the payload copy below is a pure consumer. +# +# Usage: termux_pkg_build.sh +# pm package name: python | node +# payload subdir the staged tree lands in: python | node +# payload root +set -euo pipefail + +PKG="$1" +SUBDIR="$2" +mkdir -p "$3" +PAYLOAD="$(cd "$3" && pwd)" +STAGED="$PAYLOAD/$SUBDIR" + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$HERE/../.." && pwd)" +cd "$REPO_ROOT" + +TARGET="linux-arm64-bionic" + +# Stage (idempotent: an already-published verifying entry is a no-op) and +# print the entry path -- pm's own resolvers, no layout guessing here. +# Resolve a real interpreter: some hosts alias python3 to a Store stub +# (Windows App Execution Aliases). pm runs on any stdlib python >= 3.11. +PM_PY="$(command -v python3 || true)" +if [ -n "$PM_PY" ] && "$PM_PY" -c "import sys; assert sys.version_info >= (3, 11)" 2>/dev/null; then + : +else + PM_PY="$(command -v python || true)" +fi +[ -n "$PM_PY" ] || { echo "termux_pkg_build: no usable python found" >&2; exit 1; } +ENTRY_DIR="$("$PM_PY" - "$PKG" "$TARGET" <<'PYEOF' +import sys +sys.path.insert(0, ".") +from pm.ensure import stage_only +# stage_only already returns the published entry path -- no re-derivation. +print(stage_only(sys.argv[1], sys.argv[2])) +PYEOF +)" || { echo "termux_pkg_build: pm stage failed" >&2; exit 1; } +[ -n "$ENTRY_DIR" ] && [ -d "$ENTRY_DIR" ] || { echo "termux_pkg_build: staged entry missing: $ENTRY_DIR" >&2; exit 1; } + +rm -rf "$STAGED" +cp -a "$ENTRY_DIR" "$STAGED" +echo "termux_pkg_build($PKG): pm entry staged -> $STAGED" diff --git a/tests/test_pm_bionic.py b/tests/test_pm_bionic.py new file mode 100644 index 0000000000..a99d36d6e0 --- /dev/null +++ b/tests/test_pm_bionic.py @@ -0,0 +1,197 @@ +"""Behavior contracts for pm's bionic target + DebPackage + stage_only. + +The lock VALUES are bumped by review; these tests pin the relationships: +the linux-arm64-bionic rows exist and agree with their suppliers' shapes, +DebPackage extraction is hardened, and cross-target staging never touches +this host's installed facts. +""" + +from __future__ import annotations + +import json +import tarfile +import io +import zipfile +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _pm(): + import sys + + sys.path.insert(0, str(REPO_ROOT)) + import pm + + return pm + + +@pytest.fixture(scope="module") +def lock(): + return json.loads((REPO_ROOT / "pm" / "lock.json").read_text(encoding="utf-8")) + + +def test_all_targets_includes_bionic(): + from pm.store import ALL_TARGETS + + assert "linux-arm64-bionic" in ALL_TARGETS + assert ALL_TARGETS.count("linux-arm64-bionic") == 1 + + +def _assert_pinned_bionic_row(lock, pkg, url_suffix_re): + """The bionic rows are DELIBERATE explicit pins: the version axis follows + main (the desktop artifacts), while the termux/TUR suppliers rotate or + lag, so the bionic row pins whatever the supplier actually ships today. + The stage path consumes the row's url+sha256 directly. The contract: + the row exists, is https, matches the supplier's filename shape, and + fetch_url AGREES with the pinned row for this target.""" + import re as _re + + row = lock["packages"][pkg]["artifacts"].get("linux-arm64-bionic") + assert row, f"{pkg} has no linux-arm64-bionic artifact" + assert row["url"].startswith("https://"), f"{pkg} bionic row not https" + m = _re.search(url_suffix_re, row["url"]) + assert m, f"{pkg} bionic url shape: {row['url']}" + assert _re.fullmatch(r"[0-9a-f]{64}", row["sha256"]) + from pm.registry import get_package + + cls = get_package(pkg) + # fetch_url must reproduce the pinned row exactly for this target + # (the pinned version is recovered from the URL itself). + assert cls.fetch_url(m.group("ver"), "linux-arm64-bionic") == row["url"] + + +def test_python_bionic_row_matches_supplier(lock): + """The python bionic row is an explicit pin of the TUR .deb; the row and + Python.fetch_url(bionic arm) must agree on the same pinned artifact.""" + _assert_pinned_bionic_row(lock, "python", r"python3\.11_(?P[0-9.]+)_aarch64\.deb$") +def test_node_bionic_row_matches_supplier(lock): + """The node bionic row is an explicit pin of the termux-main nodejs .deb; + the row and Nodejs.fetch_url(bionic arm) must agree.""" + _assert_pinned_bionic_row(lock, "node", r"nodejs_(?P[0-9.]+)-1_aarch64\.deb$") +def test_termux_docker_row_pins_digest(lock): + row = lock["packages"]["termux-docker"]["artifacts"]["linux-arm64-bionic"] + version = lock["packages"]["termux-docker"]["version"] + assert version.startswith("sha256:") + assert len(version) == 7 + 64 + assert row["url"] == f"docker://termux/termux-docker@{version}" + + +def test_bionic_fetch_urls_resolve(): + from pm.registry import get_package + + py = get_package("python") + version = "3.11.15+20260807" + assert py.fetch_url(version, "linux-arm64-bionic").endswith( + "python3.11_3.11.15_aarch64.deb" + ) + nd = get_package("node") + assert nd.fetch_url("26.4.0", "linux-arm64-bionic").endswith( + "nodejs_26.4.0-1_aarch64.deb" + ) + + +def test_uv_bionic_row_matches_supplier(lock): + """The uv bionic row is an explicit pin of the termux-main pool .deb; + the row and Uv.fetch_url(bionic arm) must agree.""" + _assert_pinned_bionic_row(lock, "uv", r"/u/uv/uv_(?P[0-9.]+)_aarch64\.deb$") +def _build_fake_deb(path: Path, control: dict[str, str], files: dict[str, bytes]) -> None: + def ar_member(name: str, data: bytes) -> bytes: + hdr = ( + name.ljust(16).encode() + + b"0".ljust(12) + + b"0".ljust(6) + + b"0".ljust(6) + + b"100644".ljust(8) + + str(len(data)).encode().ljust(10) + + b"`\n" + ) + pad = b"\n" if len(data) % 2 else b"" + return hdr + data + pad + + ctrl_buf = io.BytesIO() + with tarfile.open(fileobj=ctrl_buf, mode="w:gz") as tf: + body = "".join(f"{k}: {v}\n" for k, v in control.items()).encode() + info = tarfile.TarInfo("control") + info.size = len(body) + tf.addfile(info, io.BytesIO(body)) + data_buf = io.BytesIO() + with tarfile.open(fileobj=data_buf, mode="w:gz") as tf: + for name, content in files.items(): + info = tarfile.TarInfo(name) + info.size = len(content) + tf.addfile(info, io.BytesIO(content)) + path.write_bytes( + b"!\n" + + ar_member("debian-binary", b"2.0\n") + + ar_member("control.tar.gz", ctrl_buf.getvalue()) + + ar_member("data.tar.gz", data_buf.getvalue()) + ) + + +def test_debpackage_unpack_hardened(tmp_path: Path): + """DebPackage.unpack extracts data members and refuses traversal.""" + from pm.package import DebPackage + + class _P(DebPackage): + name = "test-deb" + + deb = tmp_path / "test.deb" + _build_fake_deb( + deb, + {"Package": "test-deb", "Version": "1.0"}, + {"data/data/com.termux/files/usr/bin/tool": b"\x7fELF"}, + ) + staged = tmp_path / "staged" + staged.mkdir() + _P().unpack(deb, staged, "linux-arm64-bionic") + assert (staged / "data/data/com.termux/files/usr/bin/tool").read_bytes() == b"\x7fELF" + + # traversal member must be refused + evil = tmp_path / "evil.deb" + _build_fake_deb( + evil, {"Package": "evil", "Version": "1.0"}, {"../escape": b"x"} + ) + with pytest.raises(Exception): + _P().unpack(evil, tmp_path / "staged2", "linux-arm64-bionic") + + +def test_python_bionic_verify_is_file_evidence(tmp_path: Path): + """bionic verify never executes the staged binary; presence is the + contract (the digest already proved the bytes).""" + from pm.registry import get_package + + py = get_package("python") + bin_rel = Path("data/data/com.termux/files/usr/bin/python3.11") + entry = tmp_path / "entry" + (entry / bin_rel).parent.mkdir(parents=True) + (entry / bin_rel).write_bytes(b"bionic-elf-bytes") + assert py.verify(entry, "linux-arm64-bionic") == "" + empty = tmp_path / "empty" + empty.mkdir() + assert "missing" in py.verify(empty, "linux-arm64-bionic") + + +def test_stage_only_does_not_record_host_facts(tmp_path, monkeypatch): + """stage_only publishes the entry but must not touch this machine's + installed facts -- the fact slot belongs to the HOST target.""" + pm = _pm() + from pm.ensure import stage_only + from pm.lock import Facts + from pm.paths import facts_path + + def snapshot() -> dict: + path = facts_path() + if not path.is_file(): + return {} + return Facts(path)._packages + + before = snapshot() + entry = stage_only("termux-docker", "linux-arm64-bionic") + after = snapshot() + assert before == after + # termux-docker is a pin_only package: stage_only returns the would-be + # entry path (store root + entry name) without staging bytes. + assert "termux-docker" in str(entry)