a41aabff6c
- pm.environment owns _RESOLVER_MARKERS: the streaming uv runner imported
pm.workspace, whose tomllib import fails on the 3.10 system python that
bootstraps the Docker arm64 image (No module named 'tomllib'). Invariant test
proves runtime staging needs neither tomllib nor pm.workspace.
- run_tests.sh forwards the MSVC/SDK/Rust/OpenSSL toolchain variables through
its env -i scrub so PM tests that compile ruamel-yaml-clib on Windows arm64
find cl.exe (previously 'Visual C++ 14.0 or greater is required').
- nix desktop-backend check: the spawned backend outlives cage's process group
and kept writing under the temp HERMES_HOME during rmtree; stop every
process bound to the throwaway HOME before cleanup.
- windows: test_launcher_runtime_selection imports runtime_state from
hermes_cli (moved in bbec973514); the ' spaced ' suffix row loses its
trailing space on win32 (the filesystem strips it).
- macOS: test_sealed_worker_command copies the interpreter into the payload
(the escape guard resolves symlinks) and links the host lib tree.
219 lines
11 KiB
Bash
Executable File
219 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Canonical test runner for hermes-agent. Run this instead of calling
|
|
# `pytest` directly to guarantee your local run matches CI behavior.
|
|
#
|
|
# What this script enforces:
|
|
# * Per-file isolation via scripts/run_tests_parallel.py — each test
|
|
# file runs in its own freshly-spawned `python -m pytest <file>`
|
|
# subprocess. No xdist, no shared workers, no module-level leakage
|
|
# between files.
|
|
# * TZ=UTC, LANG=C.UTF-8, PYTHONHASHSEED=0 (deterministic)
|
|
# * Env vars blanked (conftest.py also does this, but this
|
|
# is belt-and-suspenders for anyone running pytest outside our
|
|
# conftest path — e.g. on a single file)
|
|
# * Proper venv activation (probes .venv, venv, then ~/.hermes/...)
|
|
#
|
|
# Usage:
|
|
# scripts/run_tests.sh # full suite
|
|
# scripts/run_tests.sh -j 4 # cap parallelism
|
|
# scripts/run_tests.sh tests/agent/ # discover only here
|
|
# scripts/run_tests.sh tests/agent/ tests/acp_adapter/ # multiple roots
|
|
# scripts/run_tests.sh tests/foo.py # single file
|
|
# scripts/run_tests.sh tests/foo.py -q # path + bare pytest flag
|
|
# scripts/run_tests.sh tests/foo.py -v --tb=long # bare flags "just work"
|
|
# scripts/run_tests.sh -k 'pattern' # value flags pass through too
|
|
# scripts/run_tests.sh tests/foo.py -- --tb=long # explicit '--' still works
|
|
#
|
|
# Bare pytest flags (anything starting with '-' that isn't one of this
|
|
# runner's own options: -j/--jobs, --paths, --slice, --file-timeout, etc.)
|
|
# are forwarded to each per-file pytest invocation automatically — no '--'
|
|
# separator required. The explicit '--' form still works and stacks with
|
|
# bare flags. Positional path arguments override the default discovery
|
|
# root (tests/).
|
|
|
|
set -euo pipefail
|
|
|
|
# ── Locate repo root ────────────────────────────────────────────────────────
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
# ── Locate python ───────────────────────────────────────────────────────────
|
|
# Probe local venvs first; fall back to the Nix devShell's editable venv
|
|
# (HERMES_PYTHON is exported by the devShell hook and ships [dev] extras:
|
|
# pytest, pytest-asyncio, pytest-timeout, ruff, ty).
|
|
#
|
|
# A candidate must have pytest INSTALLED, not merely exist. The release venv
|
|
# at ~/.hermes/hermes-agent/venv has bin/activate but no pytest, so an
|
|
# existence-only probe selected it in checkouts/worktrees without a local
|
|
# .venv — every file then died with "No module named pytest" and the run
|
|
# reported "0 tests passed" (which reads green at a glance even though the
|
|
# exit code is 1). Skip such a venv and keep probing instead.
|
|
VENV=""
|
|
VENV_PYTHON=""
|
|
SKIPPED_VENVS=""
|
|
for candidate in "$REPO_ROOT/.venv" "$REPO_ROOT/venv" "$HOME/.hermes/hermes-agent/venv"; do
|
|
if [ -f "$candidate/bin/activate" ]; then
|
|
if "$candidate/bin/python" -c 'import pytest' 2>/dev/null; then
|
|
VENV="$candidate"
|
|
VENV_PYTHON="$candidate/bin/python"
|
|
break
|
|
fi
|
|
SKIPPED_VENVS="$SKIPPED_VENVS $candidate"
|
|
fi
|
|
# Native Windows venv layout: python.exe and activate live under
|
|
# Scripts/, and there is no bin/. Anyone running this script from
|
|
# Git Bash / MSYS with a `python -m venv`- or uv-created venv hits
|
|
# this branch — without it the canonical runner refuses to start.
|
|
if [ -f "$candidate/Scripts/activate" ]; then
|
|
if "$candidate/Scripts/python.exe" -c 'import pytest' 2>/dev/null; then
|
|
VENV="$candidate"
|
|
VENV_PYTHON="$candidate/Scripts/python.exe"
|
|
break
|
|
fi
|
|
SKIPPED_VENVS="$SKIPPED_VENVS $candidate"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$SKIPPED_VENVS" ]; then
|
|
for skipped in $SKIPPED_VENVS; do
|
|
echo "▶ skipping venv without pytest: $skipped" >&2
|
|
done
|
|
fi
|
|
|
|
if [ -n "$VENV" ]; then
|
|
PYTHON="$VENV_PYTHON"
|
|
elif [ -n "${HERMES_PYTHON:-}" ] && [ -x "$HERMES_PYTHON" ] \
|
|
&& "$HERMES_PYTHON" -c 'import pytest' 2>/dev/null; then
|
|
# Guard with an import check: HERMES_PYTHON may point at the RELEASE
|
|
# venv (no pytest) when inherited from a wrapped `hermes` binary rather
|
|
# than the devShell hook.
|
|
PYTHON="$HERMES_PYTHON"
|
|
echo "▶ no local venv — using Nix dev venv via HERMES_PYTHON: $PYTHON"
|
|
else
|
|
echo "error: no virtualenv with pytest found in $REPO_ROOT/.venv or $REPO_ROOT/venv," >&2
|
|
echo " and HERMES_PYTHON is not a python with pytest (enter the Nix devShell or create a venv)" >&2
|
|
if [ -n "$SKIPPED_VENVS" ]; then
|
|
echo " (skipped for missing pytest:$SKIPPED_VENVS — install dev extras there, or create $REPO_ROOT/.venv)" >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
|
|
# ── Live-gateway plugin (computed before we drop env) ───────────────────────
|
|
EXTRA_PYTHONPATH=""
|
|
EXTRA_PYTEST_PLUGINS=""
|
|
if [ -f "$HOME/.hermes/pytest_live_guard.py" ]; then
|
|
EXTRA_PYTHONPATH="$HOME/.hermes"
|
|
EXTRA_PYTEST_PLUGINS="pytest_live_guard"
|
|
fi
|
|
|
|
|
|
# ── Real-home PATH entries (computed before we drop env) ────────────────────
|
|
# An installed Hermes puts ~/.hermes/{bin,node/bin} on the shell PATH. Tests
|
|
# run under the real-home I/O tripwire, so a `shutil.which()` walking those
|
|
# entries fails as "I/O against the REAL hermes home" — CI has none of them.
|
|
TEST_PATH=""
|
|
_IFS_SAVE="$IFS"; IFS=':'
|
|
for _entry in $PATH; do
|
|
case "$_entry" in
|
|
"$HOME/.hermes"|"$HOME/.hermes/"*) ;;
|
|
*) TEST_PATH="${TEST_PATH:+$TEST_PATH:}$_entry" ;;
|
|
esac
|
|
done
|
|
IFS="$_IFS_SAVE"
|
|
|
|
|
|
# ── Windows location variables (computed before we drop env) ───────────────
|
|
# `env -i` forwards HOME, which is enough on POSIX. Native Windows CPython
|
|
# resolves Path.home() from USERPROFILE (or HOMEDRIVE+HOMEPATH), stdlib
|
|
# platform paths come from LOCALAPPDATA/APPDATA, ssl/sockets need SYSTEMROOT,
|
|
# and tempfile needs TEMP/TMP. Dropping them breaks collection on native
|
|
# Windows (issues #67385, #70813). These are location variables, not
|
|
# credentials, so forwarding them keeps the isolation intent intact. Each is
|
|
# only forwarded when actually set, so POSIX runs are byte-for-byte unchanged.
|
|
WIN_ENV=()
|
|
for _win_var in USERPROFILE HOMEDRIVE HOMEPATH LOCALAPPDATA APPDATA SYSTEMROOT TEMP TMP \
|
|
ComSpec PROGRAMFILES ProgramFiles PROGRAMDATA ProgramData; do
|
|
if [ -n "${!_win_var:-}" ]; then
|
|
WIN_ENV+=("$_win_var=${!_win_var}")
|
|
fi
|
|
done
|
|
# Native build toolchain (Windows arm64 has no wheels for every pinned C extension, so
|
|
# `uv sync` inside a PM test compiles ruamel-yaml-clib and friends). The MSVC developer
|
|
# environment is exported by scripts/build/windows-deps.ps1 into the job env; without
|
|
# INCLUDE/LIB/VSINSTALLDIR the build backend reports "Visual C++ 14.0 or greater is
|
|
# required". These describe compiler locations, not credentials.
|
|
for _tool_var in INCLUDE LIB LIBPATH VSINSTALLDIR VCINSTALLDIR VCToolsInstallDir VCToolsVersion \
|
|
VCToolsRedistDir WindowsSdkDir WindowsSDKVersion WindowsSdkBinPath WindowsSdkVerBinPath \
|
|
WindowsLibPath UCRTVersion UniversalCRTSdkDir VSCMD_ARG_HOST_ARCH VSCMD_ARG_TGT_ARCH VSCMD_VER \
|
|
DevEnvDir ExtensionSdkDir Platform CARGO_HOME RUSTUP_HOME RUSTUP_TOOLCHAIN \
|
|
CARGO_TARGET_AARCH64_PC_WINDOWS_MSVC_LINKER CC_aarch64_pc_windows_msvc CC CXX AR \
|
|
VCPKG_ROOT OPENSSL_DIR OPENSSL_STATIC OPENSSL_LIB_DIR OPENSSL_INCLUDE_DIR; do
|
|
if [ -n "${!_tool_var:-}" ]; then
|
|
WIN_ENV+=("$_tool_var=${!_tool_var}")
|
|
fi
|
|
done
|
|
|
|
# ── Test-runner knobs (computed before we drop env) ────────────────────────
|
|
# The runner's own documented environment knobs must survive the hermetic
|
|
# `env -i` below, or they are silent no-ops for anyone invoking this script:
|
|
#
|
|
# * HERMES_TEST_WORKERS / PATHS / FILE_TIMEOUT / FILE_RETRIES / SLICE are
|
|
# read by run_tests_parallel.py at argparse-default time — inside the
|
|
# stripped environment.
|
|
# * HERMES_TEST_IMAGE is read by tests/docker/conftest.py to skip its
|
|
# session-scoped `docker build`. CI's docker.yml sets it to the image
|
|
# the build step just loaded; stripping it made every per-file pytest
|
|
# subprocess rebuild the 5GB image from a cold builder cache instead
|
|
# (~4 min per worker per run, and the rebuilt image lacked the
|
|
# HERMES_GIT_SHA build-arg the workflow bakes in).
|
|
#
|
|
# These are test-infrastructure knobs, not credentials — same class as the
|
|
# HERMES_RUN_SLOW_PET_TESTS / HERMES_E2E_BROWSER opt-ins already forwarded.
|
|
# SSL_CERT_FILE/DIR are trust-store locations: the pinned interpreter's
|
|
# OpenSSL has no compiled-in bundle path on NixOS, so network tests (PM
|
|
# downloads, channel reads) need the host's pointer to verify TLS.
|
|
# Keep this an explicit allowlist (no HERMES_TEST_* glob) so the "no
|
|
# credential can leak" property stays auditable at a glance.
|
|
TEST_ENV=()
|
|
for _test_var in HERMES_TEST_IMAGE HERMES_TEST_WORKERS HERMES_TEST_PATHS \
|
|
HERMES_TEST_FILE_TIMEOUT HERMES_TEST_FILE_RETRIES HERMES_TEST_SLICE \
|
|
SSL_CERT_FILE SSL_CERT_DIR; do
|
|
if [ -n "${!_test_var:-}" ]; then
|
|
TEST_ENV+=("$_test_var=${!_test_var}")
|
|
fi
|
|
done
|
|
|
|
# ── Run in hermetic env ──────────────────────────────────────────────────────
|
|
# env -i: start with empty environment, opt-in only what we need.
|
|
# No credential var can leak — you'd have to explicitly add it here.
|
|
echo "▶ running per-file parallel test suite via run_tests_parallel.py"
|
|
echo " (TZ=UTC LANG=C.UTF-8 PYTHONHASHSEED=0; clean env)"
|
|
|
|
cd "$REPO_ROOT"
|
|
|
|
# ── Pre-compile .pyc bytecode cache ─────────────────────────────────────────
|
|
# Each test file runs in its own subprocess via run_tests_parallel.py.
|
|
# Pre-building the bytecode cache once here (instead of each subprocess
|
|
# compiling on first import) avoids redundant work across ~2000 processes.
|
|
# Uses git to list tracked .py files (skips venv, node_modules, etc).
|
|
echo "▶ pre-compiling bytecode cache"
|
|
"$PYTHON" -m compileall -q -j 0 -- $(git ls-files '*.py') >/dev/null 2>&1 || true
|
|
|
|
echo "▶ launching test runner"
|
|
exec env -i \
|
|
PATH="$TEST_PATH" \
|
|
HOME="$HOME" \
|
|
${WIN_ENV[@]+"${WIN_ENV[@]}"} \
|
|
${TEST_ENV[@]+"${TEST_ENV[@]}"} \
|
|
TZ=UTC \
|
|
LANG=C.UTF-8 \
|
|
LC_ALL=C.UTF-8 \
|
|
PYTHONHASHSEED=0 \
|
|
PYTHONUTF8=1 \
|
|
${HERMES_RUN_SLOW_PET_TESTS:+HERMES_RUN_SLOW_PET_TESTS="$HERMES_RUN_SLOW_PET_TESTS"} \
|
|
${HERMES_E2E_BROWSER:+HERMES_E2E_BROWSER="$HERMES_E2E_BROWSER"} \
|
|
${EXTRA_PYTHONPATH:+PYTHONPATH="$EXTRA_PYTHONPATH"} \
|
|
${EXTRA_PYTEST_PLUGINS:+PYTEST_PLUGINS="$EXTRA_PYTEST_PLUGINS"} \
|
|
"$PYTHON" "$SCRIPT_DIR/run_tests_parallel.py" "$@"
|