81f873b334
tests/e2e/core/upgrade/pm/: 22 cells in 7 files, one per failure class, each driving the real entry points (install.sh, hermes update, hermes pm repair/status/install, hermes doctor, hermes gateway run/install --force, hermes kanban dispatch, hermes -z through the loopback provider) inside the existing bwrap sandbox, seeded with an N-1 install and a dependency-changing release: - configured features survive update, repair and legacy-venv migration - SIGKILL at the stage/publish boundary of a dependency update recovers - stamps, receipts and doctor tell the truth (healthy, drifted, failed update) - no stray in-tree venv / repo-local 3.11 .venv reaches any interpreter; execute_code and Kanban workers spawned after an update boot and import deps - repeated updates/repairs/restarts don't accumulate generations - gateway install --force keeps the install bootable Every non-gated cell has a recorded sabotage proof; open bugs are message-gated with known_failure (gated on #124228, #122627, #122425, #124214, #124075, #124049, #124668). Harness: make_origin clones --single-branch and allows filters, so a blobless developer checkout can serve the installer's clone. CI: e2e-upgrade is sharded per suite directory (an e2e-upgrade-plan job lists "core" plus every subdirectory holding test_*.py); a shard that collects no files fails; HERMES_TEST_WORKERS=6.
362 lines
17 KiB
YAML
362 lines
17 KiB
YAML
name: Tests
|
||
|
||
on:
|
||
workflow_call:
|
||
|
||
permissions:
|
||
contents: read
|
||
|
||
# Cancel in-progress runs for the same ref
|
||
concurrency:
|
||
group: tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }}
|
||
cancel-in-progress: ${{ github.ref_type != 'tag' }}
|
||
|
||
jobs:
|
||
test:
|
||
name: Run tests
|
||
# One 96-core runner for the whole suite. There is no slicing. Slicing
|
||
# existed to spread the suite over 4-core runners. It cost a matrix job, a
|
||
# duration cache, a per-slice artifact and a merge job to do it.
|
||
#
|
||
# 96 cores clear the floor that the slowest single test file sets (about
|
||
# 82s). A second slice divides work that is already at that floor, and
|
||
# adds a second setup.
|
||
runs-on: ubuntu-latest-96-core
|
||
timeout-minutes: 30
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||
|
||
- name: Install ripgrep (prebuilt binary)
|
||
run: |
|
||
set -euo pipefail
|
||
RG_VERSION=15.1.0
|
||
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
||
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
||
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
||
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
||
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
||
tar -xzf "$RG_TARBALL"
|
||
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
||
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
||
rg --version
|
||
|
||
- name: Set up locked Python and test dependencies
|
||
uses: ./.github/actions/setup-pm
|
||
with:
|
||
extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "parallel-web"]'
|
||
test-environment: 'true'
|
||
prune-python-cache: true
|
||
|
||
- name: Check SQLite runs WAL
|
||
# Hermes picks DELETE journal mode on a SQLite with the WAL-reset
|
||
# bug. If the PM-pinned interpreter links one, every WAL test silently
|
||
# runs DELETE (or skips), so fail loudly instead. Tests that need DELETE
|
||
# select it explicitly (journal_mode config, a pinned vulnerable
|
||
# version), so they run on any interpreter.
|
||
run: |
|
||
"$HERMES_PYTHON" -c "import sqlite3, sys, hermes_state_wal as w; v = w.is_sqlite_wal_reset_vulnerable(); print(f'Python {sys.version.split()[0]}, SQLite {sqlite3.sqlite_version}, WAL-reset vulnerable: {v}'); sys.exit(1 if v else 0)"
|
||
|
||
- name: Restore per-file duration cache
|
||
# scripts/run_tests_parallel.py raises a file's timeout to
|
||
# 3x its last healthy duration (_effective_file_timeout) so a
|
||
# known-slow file dilated by load is not SIGKILL'd at the flat cap
|
||
# and laundered into a FLAKY retry. The scaler reads
|
||
# test_durations.json from the checkout; without this restore the
|
||
# file is absent on a fresh runner and the scaler is inert.
|
||
# Exact key never matches (run_id differs); restore-keys picks the
|
||
# most recent cache saved by a main push. PRs read, only main writes.
|
||
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||
with:
|
||
path: test_durations.json
|
||
key: test-durations-never-exact
|
||
restore-keys: |
|
||
test-durations-
|
||
|
||
- name: Run tests
|
||
# Per-file isolation via scripts/run_tests.sh: each test file runs
|
||
# in its own freshly-spawned `python -m pytest <file>` subprocess
|
||
# with bounded parallelism. No xdist, no shared workers, no
|
||
# module-level state leakage between files.
|
||
#
|
||
# No --files: the runner discovers the suite itself. The discovered
|
||
# set is identical to the list the removed matrix job used to pass in.
|
||
run: |
|
||
scripts/run_tests.sh
|
||
env:
|
||
# This is the maximum number of test FILES that run together.
|
||
# run_tests_parallel.py starts one pytest subprocess for each file
|
||
# from a single ThreadPoolExecutor, so this value IS the limit. The
|
||
# default is cpu_count*2, which is 192 here.
|
||
#
|
||
# Measured on this runner (96-core EPYC 7763, 377GB). Whole suite,
|
||
# two repetitions for each value. See run 32549672063:
|
||
#
|
||
# workers x cores mean
|
||
# 48 0.5x 138s
|
||
# 96 1.0x 126s <- fastest
|
||
# 144 1.5x 132s
|
||
# 192 2.0x 132s
|
||
# 240 2.5x 140s
|
||
# 288 3.0x 142s
|
||
#
|
||
# One worker for each core wins. The curve is shallow: 126s to 142s
|
||
# across a 6x range. The suite has sufficient concurrency at this
|
||
# size. The remaining time is the slowest files plus the setup.
|
||
# Workers above the core count only add contention.
|
||
HERMES_TEST_WORKERS: 96
|
||
# A healthy 44-test compression file takes ~197s alone and can exceed
|
||
# the 300s default under 96-worker load before its duration is cached.
|
||
# Match the runner's 3x known-slow headroom on the first PR run too.
|
||
HERMES_TEST_FILE_TIMEOUT: 600
|
||
# Ensure tests don't accidentally call real APIs
|
||
OPENROUTER_API_KEY: ""
|
||
OPENAI_API_KEY: ""
|
||
NOUS_API_KEY: ""
|
||
|
||
- name: Save per-file duration cache (main only)
|
||
# Only green first-attempt durations are written by the runner, so
|
||
# a hang on main cannot ratchet its own bound upward.
|
||
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && hashFiles('test_durations.json') != ''
|
||
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
||
with:
|
||
path: test_durations.json
|
||
key: test-durations-${{ github.run_id }}
|
||
|
||
e2e:
|
||
# The core suites spawn real serve / gateway / tui_gateway / MCP / SQLite
|
||
# writer processes per test; a 4-vCPU runner serialises them into timeouts.
|
||
runs-on: ubuntu-latest-32-core
|
||
timeout-minutes: 30
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||
|
||
- name: Set up locked Python, Node and test dependencies
|
||
# toolchain: all puts PM's locked Node in the runner's PM store, not
|
||
# only on PATH. The E2E sandboxes copy node from that store
|
||
# (tests/e2e/core/_pm_dependencies.py) and forbid lazy installs, and
|
||
# the dashboard's /api/pty resolves the TUI's node through PM only.
|
||
# A setup-node binary on PATH is invisible to PM, so every /api/pty
|
||
# chat refused to start. The locked Node also satisfies the
|
||
# boot-contract suite (backend-ready.ts needs Node >= 22.18).
|
||
uses: ./.github/actions/setup-pm
|
||
with:
|
||
toolchain: all
|
||
# `messaging`: tests/e2e/core/platforms runs the real Telegram/Discord/Slack
|
||
# adapters (their SDKs) against local stand-ins.
|
||
extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "parallel-web", "messaging"]'
|
||
test-environment: 'true'
|
||
prune-python-cache: true
|
||
|
||
- name: Install Node deps and build the Ink TUI
|
||
# tests/e2e/core/terminal drives the real `hermes --tui` over a PTY;
|
||
# HERMES_E2E_REQUIRE_TUI=1 below turns a missing build into a failure.
|
||
uses: ./.github/actions/retry
|
||
with:
|
||
command: npm ci --workspace ui-tui --ignore-scripts --no-audit --no-fund && npm run build --workspace ui-tui
|
||
|
||
- name: Install ripgrep (prebuilt binary)
|
||
run: |
|
||
set -euo pipefail
|
||
RG_VERSION=15.1.0
|
||
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
||
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
||
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
||
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
||
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
||
tar -xzf "$RG_TARBALL"
|
||
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
||
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
||
rg --version
|
||
|
||
- name: Require a WAL-capable SQLite
|
||
# The state.db suites skip on a WAL-reset-vulnerable SQLite; fail
|
||
# instead of reporting green over zero coverage.
|
||
run: |
|
||
"$HERMES_PYTHON" -c "import sqlite3, hermes_state_wal as w; print('sqlite', sqlite3.sqlite_version); assert not w.is_sqlite_wal_reset_vulnerable()"
|
||
|
||
- name: Run e2e tests
|
||
# One subprocess per file, in parallel: the core suites spawn real
|
||
# processes (serve, gateway, tui_gateway, MCP servers, SQLite writers)
|
||
# and must not share interpreter state.
|
||
# tests/e2e/core/upgrade runs in its own job (e2e-upgrade) below.
|
||
run: |
|
||
mapfile -t files < <(find tests/e2e -name 'test_*.py' -not -path 'tests/e2e/core/upgrade/*' | sort)
|
||
scripts/run_tests.sh --include-integration "${files[@]}"
|
||
env:
|
||
# An e2e file is a process tree, not one CPU-bound pytest worker. With
|
||
# 32 files at once the hosted run starved live children before their
|
||
# ready frames (empty stderr, still alive) and hit 60–120s deadlines
|
||
# across unrelated SQLite, tenancy, provider and terminal suites.
|
||
# Bound the trees, not the child readiness/turn timeouts. At 4 the
|
||
# PTY turn and serve-SIGTERM deadlines still starved intermittently.
|
||
HERMES_TEST_WORKERS: "3"
|
||
# Multi-process episodes (torture chamber, compaction kill -9,
|
||
# gateway liveness, delivery exactly-once through a real gateway,
|
||
# the cron virtual-clock soak) legitimately run past the 300 s default.
|
||
HERMES_TEST_FILE_TIMEOUT: "900"
|
||
# No automatic re-run of a failed file: the torture chamber and the
|
||
# exactly-once/compaction suites are race detectors, and a rare
|
||
# corruption that passes on retry is still a corruption.
|
||
HERMES_TEST_FILE_RETRIES: "0"
|
||
HERMES_E2E_REQUIRE_TUI: "1"
|
||
OPENROUTER_API_KEY: ""
|
||
OPENAI_API_KEY: ""
|
||
NOUS_API_KEY: ""
|
||
|
||
e2e-upgrade-plan:
|
||
# One e2e-upgrade shard per suite directory: the files directly under
|
||
# tests/e2e/core/upgrade ("core") plus every subdirectory that holds
|
||
# test_*.py (pm, git, handoff, network, hosts, ...). A new suite
|
||
# directory gets its own shard without editing this file, and no shard
|
||
# carries the whole tree's wall time against the 60-minute timeout.
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
outputs:
|
||
shards: ${{ steps.plan.outputs.shards }}
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||
with:
|
||
sparse-checkout: tests/e2e/core/upgrade
|
||
- name: List suite directories
|
||
id: plan
|
||
run: |
|
||
set -euo pipefail
|
||
shards=$(python3 - <<'PY'
|
||
import json, pathlib
|
||
root = pathlib.Path("tests/e2e/core/upgrade")
|
||
shards = ["core"] if any(root.glob("test_*.py")) else []
|
||
shards += sorted(d.name for d in root.iterdir()
|
||
if d.is_dir() and not d.name.startswith(("_", ".")) and any(d.rglob("test_*.py")))
|
||
print(json.dumps(shards))
|
||
PY
|
||
)
|
||
echo "shards=$shards" | tee -a "$GITHUB_OUTPUT"
|
||
|
||
e2e-upgrade:
|
||
# tests/e2e/core/upgrade: a real N-1 -> HEAD `hermes update` (clean,
|
||
# autostash, killed mid-pull / mid-deps, offline), fresh-process import and
|
||
# entrypoint smoke, the config round-trip property matrix, and the lane
|
||
# suites below it (pm: PM environment lifecycle across updates, ...).
|
||
# Its own job: it needs full history + tags, bubblewrap, and one file
|
||
# runs ~5-15 min. Sharded by suite directory (see e2e-upgrade-plan).
|
||
name: e2e-upgrade (${{ matrix.shard }})
|
||
needs: e2e-upgrade-plan
|
||
runs-on: ubuntu-latest-32-core
|
||
timeout-minutes: 60
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
shard: ${{ fromJSON(needs.e2e-upgrade-plan.outputs.shards) }}
|
||
steps:
|
||
- name: Checkout code
|
||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||
with:
|
||
# N-1 = `git describe --tags --abbrev=0 HEAD~1`.
|
||
fetch-depth: 0
|
||
fetch-tags: true
|
||
|
||
- name: Set up Node
|
||
# `hermes update` builds the web UI / TUI workspaces.
|
||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||
with:
|
||
node-version: 26
|
||
|
||
- name: Install ripgrep (prebuilt binary)
|
||
run: |
|
||
set -euo pipefail
|
||
RG_VERSION=15.1.0
|
||
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
||
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
||
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
||
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
||
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
||
tar -xzf "$RG_TARBALL"
|
||
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
||
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
||
rg --version
|
||
|
||
- name: Install uv
|
||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
||
with:
|
||
# Pin the uv version: unpinned, setup-uv resolves "latest" by
|
||
# fetching a manifest from raw.githubusercontent.com on EVERY job —
|
||
# a transient fetch failure fails the whole job (2026-07-28 slice-5
|
||
# incident). Pinned, the binary downloads directly; no manifest hop.
|
||
# Newer than the unit job's pin on purpose: 0.9.28 only knows CPython
|
||
# 3.11.14, whose bundled SQLite has the WAL-reset bug, so Hermes runs
|
||
# state.db in DELETE mode and the WAL torture chamber would skip.
|
||
version: "0.12.13"
|
||
# Persist uv's download/wheel cache (~/.cache/uv) across runs.
|
||
# Keyed on the dependency manifests, so the cache is reused until
|
||
# pyproject.toml or uv.lock changes. `uv sync` still runs every
|
||
# time, but resolves from the warm cache instead of re-downloading
|
||
# and re-building wheels.
|
||
enable-cache: true
|
||
cache-dependency-glob: |
|
||
pyproject.toml
|
||
uv.lock
|
||
|
||
- name: Set up Python 3.14
|
||
run: uv python install 3.14
|
||
|
||
- name: Install dependencies
|
||
# `uv sync --locked` installs the exact pinned set from uv.lock (and
|
||
# fails if the lock is out of sync with pyproject.toml), giving a
|
||
# reproducible env. It also creates .venv itself, so no separate
|
||
# `uv venv` step is needed.
|
||
#
|
||
# Same extras as the test job's sync above: the hermetic test env
|
||
# forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
|
||
# tests/conftest.py), so lazy-install SDKs exercised by tests must be
|
||
# in the venv up front.
|
||
uses: ./.github/actions/retry
|
||
with:
|
||
command: uv sync --locked --python 3.14 --extra all --group dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra parallel-web
|
||
|
||
- name: Install bubblewrap
|
||
# Every spawned updater runs in bwrap (own PID namespace, no user
|
||
# systemd bus, only the test tmp writable). Ubuntu 24.04 restricts
|
||
# unprivileged user namespaces, which bwrap needs.
|
||
run: |
|
||
sudo apt-get update -qq
|
||
sudo apt-get install -y -qq bubblewrap
|
||
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
|
||
# The suite's own probe (_bwrap_usable, same flags as every sandboxed
|
||
# run): without it the suite silently runs the updater unsandboxed.
|
||
source .venv/bin/activate
|
||
python -c "from tests.e2e.core.upgrade._helpers import BWRAP_OK; assert BWRAP_OK, 'bwrap unusable'" && echo "bwrap ok"
|
||
|
||
- name: Require a WAL-capable SQLite
|
||
# The state.db suites skip on a WAL-reset-vulnerable SQLite; fail
|
||
# instead of reporting green over zero coverage.
|
||
run: |
|
||
source .venv/bin/activate
|
||
python -c "import sqlite3, hermes_state_wal as w; print('sqlite', sqlite3.sqlite_version); assert not w.is_sqlite_wal_reset_vulnerable()"
|
||
|
||
- name: Run upgrade e2e tests (${{ matrix.shard }})
|
||
run: |
|
||
set -euo pipefail
|
||
source .venv/bin/activate
|
||
if [ "$SHARD" = core ]; then
|
||
mapfile -t files < <(find tests/e2e/core/upgrade -maxdepth 1 -name 'test_*.py' | sort)
|
||
else
|
||
mapfile -t files < <(find "tests/e2e/core/upgrade/$SHARD" -name 'test_*.py' | sort)
|
||
fi
|
||
# A shard that collects nothing is a wiring bug, not a pass.
|
||
[ "${#files[@]}" -gt 0 ] || { echo "no test files for shard $SHARD" >&2; exit 1; }
|
||
printf 'shard %s: %s\n' "$SHARD" "${files[*]}"
|
||
scripts/run_tests.sh --include-integration "${files[@]}"
|
||
env:
|
||
SHARD: ${{ matrix.shard }}
|
||
# Each file drives real installs/updaters (process trees plus uv and
|
||
# git I/O); cpu_count of them at once starves their deadlines. A
|
||
# shard holds at most ~7 files, so this runs each shard in one wave.
|
||
HERMES_TEST_WORKERS: "6"
|
||
HERMES_TEST_FILE_TIMEOUT: "3000"
|
||
OPENROUTER_API_KEY: ""
|
||
OPENAI_API_KEY: ""
|
||
NOUS_API_KEY: ""
|