ci: add macos and windows test lanes for the os-marked tests
the markers from the previous commit skip off-host. without a host to run them on, every marked test is a silent skip. this commit adds the hosts. - tests-os.yml runs -m macos_only on macos-latest and -m windows_only on windows-latest. ci.yml requires both lanes in all-checks-pass. - a lane fails on pytest exit code 5 (zero tests selected). a renamed marker cannot produce a green job that ran nothing. - each lane repeats 'not integration' because a command-line -m replaces the addopts filter. - scripts/ci/list_os_marked_tests.py selects which files each lane imports. -m filters after collection, and collection imports every module. without this helper, one unrelated ImportError on the foreign host fails a job whose own tests passed. the helper exits non-zero when a marker matches no file, and writes bytes with explicit lf so windows crlf translation cannot corrupt the bash file list. it has its own tests in tests/ci/. - the local runner now reports the skipped count and prints a note: macos_only/windows_only tests were skipped on this host, and this ci lane runs them. a green local run on linux no longer reads as coverage of the other hosts. - the runner default job count is now #cpu, not #cpu*2.
This commit is contained in:
ethernet
11
.github/workflows/ci.yml
vendored
11
.github/workflows/ci.yml
vendored
@@ -73,6 +73,16 @@ jobs:
|
||||
with:
|
||||
slice_count: 12
|
||||
|
||||
# macOS + Windows lanes. The main `tests` lane above is Linux-only, and
|
||||
# the OS-marked tests it collects are skipped there by design (see the
|
||||
# `_OS_MARKS` comment in tests/conftest.py) — this is where they run.
|
||||
# Same `python` lane gate: if no Python changed, neither runs.
|
||||
tests-os:
|
||||
name: OS-specific tests
|
||||
needs: detect
|
||||
if: needs.detect.outputs.python == 'true'
|
||||
uses: ./.github/workflows/tests-os.yml
|
||||
|
||||
lint:
|
||||
name: Python lints
|
||||
needs: detect
|
||||
@@ -192,6 +202,7 @@ jobs:
|
||||
needs:
|
||||
- detect
|
||||
- tests
|
||||
- tests-os
|
||||
- lint
|
||||
- js-tests
|
||||
- installer-tests
|
||||
|
||||
147
.github/workflows/tests-os.yml
vendored
Normal file
147
.github/workflows/tests-os.yml
vendored
Normal file
@@ -0,0 +1,147 @@
|
||||
name: OS-specific tests
|
||||
|
||||
# Runs the tests that can only be trusted on their own host OS.
|
||||
#
|
||||
# The main Python suite (.github/workflows/tests.yml) runs on
|
||||
# ubuntu-latest and covers everything that is either platform-agnostic or
|
||||
# genuinely Linux-specific. Tests whose subject is macOS- or
|
||||
# Windows-specific behaviour carry a marker (see the ``_OS_MARKS`` block
|
||||
# comment in tests/conftest.py) and are SKIPPED on Linux, because faking
|
||||
# ``sys.platform`` on a Linux runner selects the branch under test without
|
||||
# reproducing any of the OS behaviour that branch exists for. This workflow
|
||||
# is where those markers actually execute:
|
||||
#
|
||||
# macos → ``-m macos_only`` on macos-latest
|
||||
# windows → ``-m windows_only`` on windows-latest
|
||||
#
|
||||
# Deliberately NOT sliced. The marked set is small (tens of tests, not
|
||||
# thousands), so one plain ``pytest`` process per OS is both faster and far
|
||||
# less machinery than the LPT-sliced per-file runner the Linux lane needs.
|
||||
# If either lane grows past its timeout, that is the signal to reach for
|
||||
# scripts/run_tests.sh --slice here too.
|
||||
#
|
||||
# Each lane FAILS when it selects zero tests (pytest exit code 5). Without
|
||||
# that guard, a renamed marker or a bad selector would report a green job
|
||||
# that ran nothing — the exact silent-coverage-loss failure this workflow
|
||||
# exists to prevent.
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: tests-os-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
os-tests:
|
||||
name: ${{ matrix.name }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: macOS-only tests
|
||||
runner: macos-latest
|
||||
marker: macos_only
|
||||
- name: Windows-only tests
|
||||
runner: windows-latest
|
||||
marker: windows_only
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
||||
with:
|
||||
# Pinned for the same reason as the Linux lane: unpinned, setup-uv
|
||||
# resolves "latest" by fetching a manifest on every job and a
|
||||
# transient fetch failure fails the whole job.
|
||||
version: "0.9.28"
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Set up Python 3.11
|
||||
run: uv python install 3.11
|
||||
|
||||
- name: Install dependencies
|
||||
# Same extras as the Linux test lane so an OS-marked test can import
|
||||
# anything its Linux siblings can. ``[all]`` is deliberately
|
||||
# Windows/macOS-installable (see the policy comment on the extra in
|
||||
# pyproject.toml — matrix/python-olm was removed from it precisely
|
||||
# because it could not build here).
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra hindsight --extra parallel-web
|
||||
|
||||
- name: Minimize uv cache
|
||||
run: uv cache prune --ci
|
||||
|
||||
- name: Run ${{ matrix.marker }} tests
|
||||
# Two-step selection:
|
||||
#
|
||||
# 1. scripts/ci/list_os_marked_tests.py narrows WHICH FILES are
|
||||
# imported. ``-m`` filters after collection, and collection
|
||||
# imports every module under tests/ — on this host that would
|
||||
# drag ~900 unrelated test modules through import, where a
|
||||
# single unrelated ImportError would fail a job whose own
|
||||
# subject is fine. The helper exits non-zero if the marker
|
||||
# matches no file at all.
|
||||
# 2. ``-m`` decides WHICH TESTS run, and stays authoritative.
|
||||
# Passing it on the command line REPLACES pyproject's
|
||||
# ``-m 'not integration'`` addopts (same option, last wins) —
|
||||
# hence repeating ``not integration``, or the integration
|
||||
# suite would return through the side door.
|
||||
#
|
||||
# ``--timeout-method`` needs no override: tests/conftest.py's
|
||||
# pytest_configure already downgrades the signal-based timer on
|
||||
# Windows, which has no SIGALRM.
|
||||
shell: bash
|
||||
run: |
|
||||
set -uo pipefail
|
||||
|
||||
LIST="${RUNNER_TEMP:-.}/selected-tests.txt"
|
||||
|
||||
# Process substitution would hide the helper's exit status, so write
|
||||
# to a file and check it explicitly.
|
||||
if ! uv run --no-sync python scripts/ci/list_os_marked_tests.py \
|
||||
"${{ matrix.marker }}" > "$LIST"; then
|
||||
echo "::error::could not enumerate ${{ matrix.marker }} test files"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$LIST" ]; then
|
||||
echo "::error::empty ${{ matrix.marker }} file list"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Deliberately NOT `mapfile`: that is a bash 4 builtin and the macOS
|
||||
# runner's /bin/bash is 3.2. Word-splitting is safe here because the
|
||||
# helper emits repo-relative test paths, which contain no spaces.
|
||||
# shellcheck disable=SC2046
|
||||
set -- $(cat "$LIST")
|
||||
echo "selected $# file(s) for ${{ matrix.marker }}:"
|
||||
cat "$LIST"
|
||||
|
||||
uv run --no-sync python -m pytest \
|
||||
"$@" \
|
||||
-m "${{ matrix.marker }} and not integration" \
|
||||
-v --tb=short
|
||||
status=$?
|
||||
if [ "$status" -eq 5 ]; then
|
||||
echo "::error::No tests matched -m ${{ matrix.marker }}. Either the" \
|
||||
"marker was renamed/dropped or selection is broken — this job" \
|
||||
"must never pass without running its OS's tests."
|
||||
exit 1
|
||||
fi
|
||||
exit "$status"
|
||||
env:
|
||||
# Belt-and-suspenders with tests/conftest.py's env blanking: no
|
||||
# test may reach a real provider API.
|
||||
OPENROUTER_API_KEY: ""
|
||||
OPENAI_API_KEY: ""
|
||||
NOUS_API_KEY: ""
|
||||
Reference in New Issue
Block a user