146 lines
6.2 KiB
YAML
146 lines
6.2 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
workflow_call:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Cancel in-progress runs for the same ref
|
|
concurrency:
|
|
group: tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }}
|
|
cancel-in-progress: ${{ github.ref_type != 'tag' }}
|
|
|
|
jobs:
|
|
test:
|
|
name: Run tests
|
|
# One 96-core runner for the whole suite. There is no slicing. Slicing
|
|
# existed to spread the suite over 4-core runners. It cost a matrix job, a
|
|
# duration cache, a per-slice artifact and a merge job to do it.
|
|
#
|
|
# 96 cores clear the floor that the slowest single test file sets (about
|
|
# 82s). A second slice divides work that is already at that floor, and
|
|
# adds a second setup.
|
|
runs-on: ubuntu-latest-96-core
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Set up locked Python and test dependencies
|
|
uses: ./.github/actions/setup-pm
|
|
with:
|
|
extras: '["all", "dev", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]'
|
|
prune-python-cache: true
|
|
|
|
- name: Restore per-file duration cache
|
|
# scripts/run_tests_parallel.py raises a file's timeout to
|
|
# 3x its last healthy duration (_effective_file_timeout) so a
|
|
# known-slow file dilated by load is not SIGKILL'd at the flat cap
|
|
# and laundered into a FLAKY retry. The scaler reads
|
|
# test_durations.json from the checkout; without this restore the
|
|
# file is absent on a fresh runner and the scaler is inert.
|
|
# Exact key never matches (run_id differs); restore-keys picks the
|
|
# most recent cache saved by a main push. PRs read, only main writes.
|
|
uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: test_durations.json
|
|
key: test-durations-never-exact
|
|
restore-keys: |
|
|
test-durations-
|
|
|
|
- name: Run tests
|
|
# Per-file isolation via scripts/run_tests.sh: each test file runs
|
|
# in its own freshly-spawned `python -m pytest <file>` subprocess
|
|
# with bounded parallelism. No xdist, no shared workers, no
|
|
# module-level state leakage between files.
|
|
#
|
|
# No --files: the runner discovers the suite itself. The discovered
|
|
# set is identical to the list the removed matrix job used to pass in.
|
|
run: |
|
|
scripts/run_tests.sh
|
|
env:
|
|
# This is the maximum number of test FILES that run together.
|
|
# run_tests_parallel.py starts one pytest subprocess for each file
|
|
# from a single ThreadPoolExecutor, so this value IS the limit. The
|
|
# default is cpu_count*2, which is 192 here.
|
|
#
|
|
# Measured on this runner (96-core EPYC 7763, 377GB). Whole suite,
|
|
# two repetitions for each value. See run 32549672063:
|
|
#
|
|
# workers x cores mean
|
|
# 48 0.5x 138s
|
|
# 96 1.0x 126s <- fastest
|
|
# 144 1.5x 132s
|
|
# 192 2.0x 132s
|
|
# 240 2.5x 140s
|
|
# 288 3.0x 142s
|
|
#
|
|
# One worker for each core wins. The curve is shallow: 126s to 142s
|
|
# across a 6x range. The suite has sufficient concurrency at this
|
|
# size. The remaining time is the slowest files plus the setup.
|
|
# Workers above the core count only add contention.
|
|
HERMES_TEST_WORKERS: 96
|
|
# Ensure tests don't accidentally call real APIs
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
|
|
- name: Save per-file duration cache (main only)
|
|
# Only green first-attempt durations are written by the runner, so
|
|
# a hang on main cannot ratchet its own bound upward.
|
|
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && hashFiles('test_durations.json') != ''
|
|
uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: test_durations.json
|
|
key: test-durations-${{ github.run_id }}
|
|
|
|
e2e:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Set up locked Python and test dependencies
|
|
uses: ./.github/actions/setup-pm
|
|
with:
|
|
extras: '["all", "dev", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]'
|
|
prune-python-cache: true
|
|
|
|
- name: Run e2e tests
|
|
run: |
|
|
scripts/run_tests.sh tests/e2e/ -v --tb=short
|
|
env:
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|