121 lines
4.9 KiB
YAML
121 lines
4.9 KiB
YAML
name: Tests
|
|
|
|
on:
|
|
workflow_call:
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Cancel in-progress runs for the same ref
|
|
concurrency:
|
|
group: tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }}
|
|
cancel-in-progress: ${{ github.ref_type != 'tag' }}
|
|
|
|
jobs:
|
|
test:
|
|
name: Run tests
|
|
# One 96-core runner for the whole suite. There is no slicing. Slicing
|
|
# existed to spread the suite over 4-core runners. It cost a matrix job, a
|
|
# duration cache, a per-slice artifact and a merge job to do it.
|
|
#
|
|
# 96 cores clear the floor that the slowest single test file sets (about
|
|
# 82s). A second slice divides work that is already at that floor, and
|
|
# adds a second setup.
|
|
runs-on: ubuntu-latest-96-core
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Set up locked Python and test dependencies
|
|
uses: ./.github/actions/setup-pm
|
|
with:
|
|
extras: '["all", "dev", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]'
|
|
prune-python-cache: true
|
|
|
|
- name: Run tests
|
|
# Per-file isolation via scripts/run_tests.sh: each test file runs
|
|
# in its own freshly-spawned `python -m pytest <file>` subprocess
|
|
# with bounded parallelism. No xdist, no shared workers, no
|
|
# module-level state leakage between files.
|
|
#
|
|
# No --files: the runner discovers the suite itself. The discovered
|
|
# set is identical to the list the removed matrix job used to pass in.
|
|
run: |
|
|
scripts/run_tests.sh
|
|
env:
|
|
# This is the maximum number of test FILES that run together.
|
|
# run_tests_parallel.py starts one pytest subprocess for each file
|
|
# from a single ThreadPoolExecutor, so this value IS the limit. The
|
|
# default is cpu_count*2, which is 192 here.
|
|
#
|
|
# Measured on this runner (96-core EPYC 7763, 377GB). Whole suite,
|
|
# two repetitions for each value. See run 32549672063:
|
|
#
|
|
# workers x cores mean
|
|
# 48 0.5x 138s
|
|
# 96 1.0x 126s <- fastest
|
|
# 144 1.5x 132s
|
|
# 192 2.0x 132s
|
|
# 240 2.5x 140s
|
|
# 288 3.0x 142s
|
|
#
|
|
# One worker for each core wins. The curve is shallow: 126s to 142s
|
|
# across a 6x range. The suite has sufficient concurrency at this
|
|
# size. The remaining time is the slowest files plus the setup.
|
|
# Workers above the core count only add contention.
|
|
HERMES_TEST_WORKERS: 96
|
|
# Ensure tests don't accidentally call real APIs
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|
|
|
|
e2e:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install ripgrep (prebuilt binary)
|
|
run: |
|
|
set -euo pipefail
|
|
RG_VERSION=15.1.0
|
|
RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
|
|
RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
|
|
curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
|
|
"https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
|
|
echo "${RG_SHA256} ${RG_TARBALL}" | sha256sum -c -
|
|
tar -xzf "$RG_TARBALL"
|
|
sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
|
|
rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
|
|
rg --version
|
|
|
|
- name: Set up locked Python and test dependencies
|
|
uses: ./.github/actions/setup-pm
|
|
with:
|
|
extras: '["all", "dev", "anthropic", "mistral", "fal", "modal", "daytona", "hindsight", "parallel-web"]'
|
|
prune-python-cache: true
|
|
|
|
- name: Run e2e tests
|
|
run: |
|
|
scripts/run_tests.sh tests/e2e/ -v --tb=short
|
|
env:
|
|
OPENROUTER_API_KEY: ""
|
|
OPENAI_API_KEY: ""
|
|
NOUS_API_KEY: ""
|