From 842e3b0eed0eafbfb3d7f219ea465bd92d055cc2 Mon Sep 17 00:00:00 2001 From: ethernet Date: Sun, 6 Sep 2026 12:06:22 -0400 Subject: [PATCH] test(termux): isolate native Python linkage on an ARM runner --- .github/workflows/termux-verify.yml | 64 ++++++++++++++ scripts/termux/probe_python_linkage.py | 116 +++++++++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 .github/workflows/termux-verify.yml create mode 100644 scripts/termux/probe_python_linkage.py diff --git a/.github/workflows/termux-verify.yml b/.github/workflows/termux-verify.yml new file mode 100644 index 0000000000..c32e871d1e --- /dev/null +++ b/.github/workflows/termux-verify.yml @@ -0,0 +1,64 @@ +name: Termux verification + +on: + push: + branches: [ethie/cli-bundles] + paths: + - 'scripts/termux/**' + - 'pm/**' + - 'tests/test_termux*' + - '.github/workflows/termux-verify.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: termux-verify-${{ github.ref }} + cancel-in-progress: true + +jobs: + native-linkage: + name: Reproduce and verify bionic Python linkage + runs-on: ubuntu-24.04-arm + timeout-minutes: 45 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: '0.12.3' + enable-cache: false + - name: Stage the pinned bionic runtimes + run: | + bash scripts/termux/build_cpython.sh termux-build/payload + bash scripts/termux/build_uv.sh termux-build/payload + python3 scripts/termux/stage_runtime_libs.py termux-build/payload + - name: Run the native reproduction + run: | + image=ghcr.io/ethernet8023/hermes-termux-builder:c14ffb703abf + docker pull "$image" + image=$(docker image inspect --format '{{index .RepoDigests 0}}' "$image") + printf 'Builder image: %s\n' "$image" + mkdir -p termux-build/linkage + chmod 0777 termux-build/linkage + docker run --rm --platform linux/arm64 --user 1000:1000 \ + -v "$PWD:/repo:ro" -v "$PWD/termux-build/payload:/payload:ro" \ + -v "$PWD/termux-build/linkage:/evidence" \ + "$image" bash -c ' + set -euo pipefail + export PREFIX=/data/data/com.termux/files/usr + export LD_LIBRARY_PATH="/payload/python$PREFIX/lib:/payload/runtime-libs/lib:$PREFIX/lib" + exec "/payload/python$PREFIX/bin/python3.11" \ + /repo/scripts/termux/probe_python_linkage.py \ + --repo /repo --payload /payload --output /evidence + ' + - name: Preserve the built wheel and linker evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: termux-linkage-${{ github.sha }} + path: termux-build/linkage/ + if-no-files-found: warn + retention-days: 7 diff --git a/scripts/termux/probe_python_linkage.py b/scripts/termux/probe_python_linkage.py new file mode 100644 index 0000000000..d8d0fa02cc --- /dev/null +++ b/scripts/termux/probe_python_linkage.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Diagnose the actual anydoc wheel against the pinned bionic interpreter. + +This standalone CI diagnostic builds the locked sdist, inspects the ELF +linkage, and tests the same extension with an explicit libpython dependency. +It does not change any package source or publish release artifacts. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import tomllib +import zipfile + + +def run(argv: list[str], *, env: dict[str, str] | None = None) -> subprocess.CompletedProcess: + print("+", " ".join(argv), flush=True) + return subprocess.run(argv, check=True, env=env) + + +def import_anydoc(python: Path, site: Path, env: dict[str, str]) -> subprocess.CompletedProcess: + return subprocess.run( + [str(python), "-c", "import anydoc; print('ANYDOC_IMPORT_OK', anydoc.__file__)"], + env={**env, "PYTHONPATH": str(site)}, + capture_output=True, + text=True, + ) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--repo", type=Path, required=True) + ap.add_argument("--payload", type=Path, required=True) + ap.add_argument("--output", type=Path, required=True) + args = ap.parse_args() + prefix = Path("/data/data/com.termux/files/usr") + python = args.payload / "python" / prefix.relative_to("/") / "bin/python3.11" + uv = args.payload / "uv" / prefix.relative_to("/") / "bin/uv" + pylib = python.parent.parent / "lib" + runtime_libs = args.payload / "runtime-libs/lib" + env = {**os.environ, "LD_LIBRARY_PATH": f"{pylib}:{runtime_libs}:{prefix}/lib"} + args.output.mkdir(parents=True, exist_ok=True) + lock = tomllib.loads((args.repo / "uv.lock").read_text(encoding="utf-8")) + package = next(p for p in lock["package"] if p["name"] == "firecrawl-anydoc") + requirement = f"firecrawl-anydoc=={package['version']}" + with tempfile.TemporaryDirectory(prefix="hermes-linkage-", dir=prefix / "tmp") as tmp: + venv = Path(tmp) / "venv" + run([str(uv), "venv", "--python", str(python), str(venv)], env=env) + vp = venv / "bin/python" + run([str(uv), "pip", "install", "--python", str(vp), "pip", "packaging"], env=env) + build_env = { + **env, + "CARGO_BUILD_JOBS": "1", + "MAKEFLAGS": "-j1", + "ANDROID_API_LEVEL": "24", + "RUSTFLAGS": f"-L{pylib}", + "LDFLAGS": f"-L{pylib}", + } + run([ + str(vp), "-m", "pip", "wheel", "--no-deps", "--no-cache-dir", + "--no-binary", ":all:", "-w", str(args.output), requirement, + ], env=build_env) + wheels = sorted(args.output.glob("firecrawl_anydoc-*.whl")) + if len(wheels) != 1: + raise RuntimeError(f"expected one anydoc wheel, found {wheels}") + site = Path(tmp) / "site" + with zipfile.ZipFile(wheels[0]) as wheel: + wheel.extractall(site) + extension = next(site.glob("anydoc/_anydoc*.so")) + library = next(pylib.glob("libpython3.11.so.*")) + readelf = shutil.which("llvm-readelf") or shutil.which("readelf") + if not readelf: + raise RuntimeError("builder has no ELF inspection tool") + run([readelf, "-d", str(extension)]) + symbols = subprocess.run([readelf, "--dyn-syms", "--wide", str(library)], check=True, capture_output=True, text=True) + print("libpython export:", *[line for line in symbols.stdout.splitlines() if "_Py_NoneStruct" in line], sep="\n") + baseline = import_anydoc(vp, site, env) + print("BASELINE_EXIT", baseline.returncode, flush=True) + print(baseline.stdout, baseline.stderr, flush=True) + soname = subprocess.run(["patchelf", "--print-soname", str(library)], check=True, capture_output=True, text=True).stdout.strip() + needed = subprocess.run(["patchelf", "--print-needed", str(extension)], check=True, capture_output=True, text=True).stdout.splitlines() + if soname not in needed: + run(["patchelf", "--add-needed", soname, str(extension)]) + explicit = import_anydoc(vp, site, env) + print("EXPLICIT_LINK_EXIT", explicit.returncode, flush=True) + print(explicit.stdout, explicit.stderr, flush=True) + evidence = { + "requirement": requirement, + "wheel": wheels[0].name, + "libpython_soname": soname, + "original_needed": needed, + "baseline_exit": baseline.returncode, + "baseline_error": baseline.stderr, + "explicit_link_exit": explicit.returncode, + "explicit_link_error": explicit.stderr, + } + (args.output / "linkage-evidence.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8") + if baseline.returncode == 0: + raise RuntimeError("baseline did not reproduce the native import failure") + if "_Py_NoneStruct" not in baseline.stderr: + raise RuntimeError("baseline failed for a different reason") + if explicit.returncode: + raise RuntimeError("explicit libpython dependency did not fix the import") + print("LINKAGE_CAUSE_PROVEN", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())