hermes-agent/.github/workflows/tests-os.yml

name: OS-specific tests

# Runs the tests that can only be trusted on their own host OS.
#
# The main Python suite (.github/workflows/tests.yml) runs on
# ubuntu-latest and covers everything that is either platform-agnostic or
# genuinely Linux-specific. Tests whose subject is macOS- or
# Windows-specific behaviour carry a marker (see the ``_OS_MARKS`` block
# comment in tests/conftest.py) and are SKIPPED on Linux, because faking
# ``sys.platform`` on a Linux runner selects the branch under test without
# reproducing any of the OS behaviour that branch exists for. This workflow
# is where those markers actually execute:
#
#   macos    → ``-m platforms``    on macos-latest
#   windows  → ``-m platforms``    on windows-latest-32-core and windows-latest-32-arm-core
#
# (tests/conftest.py's ``pytest_collection_modifyitems`` hook skips
# foreign-OS ``platforms(...)`` markers on each host, so one shared ``-m
# platforms`` expression selects "this host's own marked tests" on either
# lane — the helper narrows WHICH FILES get imported first.)
#
# Deliberately NOT sliced. The marked set is small (tens of tests, not
# thousands), so one plain ``pytest`` process per OS is both faster and far
# less machinery than the per-file parallel runner the Linux lane uses.
# Each lane FAILS when it selects zero tests (pytest exit code 5) — a
# renamed marker or bad selector can never report a green job that ran
# nothing.

on:
  workflow_call:
    inputs:
      desktop_updater:
        description: >-
          Run the Windows desktop-update hand-off integration tests
          (tests/scripts/desktop_update/test_desktop_update_windows_*.py). These spawn the real
          scripts/desktop-update/windows.ps1 and poll its loopback server, so
          they carry process-timing noise a shared runner amplifies; the
          caller gates them on the classifier's desktop_updater lane so a PR
          that never touched that surface cannot be failed by it. Push /
          dispatch runs fail open (classifier sets every lane true).
        type: boolean
        required: false
        default: true

permissions:
  contents: read

concurrency:
  group: tests-os-${{ github.ref_type == 'tag' && github.run_id || github.ref }}
  cancel-in-progress: ${{ github.ref_type != 'tag' }}

jobs:
  os-tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    timeout-minutes: ${{ matrix.timeout || 30 }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: macOS-only tests
            runner: macos-latest
            marker: macos
          - name: Windows-only tests
            runner: windows-latest-32-core
            marker: windows
          - name: Windows-only tests (arm64)
            runner: windows-latest-32-arm-core
            marker: windows
            timeout: 60
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # No arm64 wheels for some of the extras below: PM builds them from sdist.
      - name: Set up native ARM64 build dependencies
        if: runner.os == 'Windows' && runner.arch == 'ARM64'
        uses: ./.github/actions/setup-windows-build-deps

      - name: Set up locked Python and test dependencies
        uses: ./.github/actions/setup-pm
        with:
          # Installer stage tests inherit setup-pm's HERMES_RUNTIME_DIR. Prepare
          # the verified Git pin once, before per-file parallelism; otherwise
          # each isolated PowerShell stage races to download/extract the archive.
          packages: ${{ matrix.marker == 'windows' && 'git,ripgrep' || 'ripgrep' }}
          extras: '["all", "telegram", "anthropic", "mistral", "fal", "modal", "daytona", "parallel-web"]'
          test-environment: 'true'
          prune-python-cache: true

      - name: Run ${{ matrix.marker }} tests
        # scripts/run_tests.sh — the canonical runner, same as every other
        # lane: per-file subprocess isolation (run_tests_parallel.py). It
        # natively tolerates per-file empty collections (a platform-gated
        # file collects nothing after -m filtering, exit 5, treated as a
        # pass for that file) and fails the run itself when NOTHING was
        # collected across all files (exit 2) — the zero-tests guard the
        # bare-pytest variant implemented by hand with an exit-5 branch.
        #
        # Selection still narrows WHICH FILES run:
        # scripts/ci/list_os_marked_tests.py emits the file list (it exits
        # non-zero when the marker matches no file at all); the list rides
        # to the runner via ``--files`` so an unrelated ImportError fails
        # only ITS file, red and visible, without poisoning the lane's
        # other files. ``-m`` stays authoritative for which TESTS run —
        # passed after ``--`` so the runner routes it to every per-file
        # pytest invocation (the flag REPLACES pyproject's addopts, hence
        # repeating ``not integration``).
        #
        # ``--timeout-method`` needs no override: tests/conftest.py's
        # pytest_configure already downgrades the signal-based timer on
        # Windows, which has no SIGALRM.
        shell: bash
        run: |
          set -uo pipefail

          LIST="${RUNNER_TEMP:-.}/selected-tests.txt"

          # Process substitution would hide the helper's exit status, so write
          # to a file and check it explicitly.
          if ! python scripts/ci/list_os_marked_tests.py \
                 "${{ matrix.marker }}" > "$LIST"; then
            echo "::error::could not enumerate ${{ matrix.marker }} test files"
            exit 1
          fi
          if [ ! -s "$LIST" ]; then
            echo "::error::empty ${{ matrix.marker }} file list"
            exit 1
          fi

          echo "selected file(s) for ${{ matrix.marker }}:"
          cat "$LIST"

          # Desktop-update hand-off integration tests spawn the real
          # windows.ps1; deselect them unless the PR touched that surface
          # (see the workflow_call input). ``--ignore-glob`` keeps the file
          # list above intact, so a renamed test file still trips the
          # zero-tests guard rather than silently vanishing.
          # (bash 3.2 on the macOS runner: an empty array under ``set -u`` is
          # an unbound-variable error, hence the ``${arr[@]+...}`` idiom.)
          EXTRA_ARGS=()
          if [ "${{ inputs.desktop_updater }}" != "true" ]; then
            echo "desktop_updater lane off: skipping tests/scripts/desktop_update/test_desktop_update_windows_*.py"
            EXTRA_ARGS+=(--ignore-glob='*test_desktop_update_windows_*.py')
          fi

          # ``tr -d '\r'``: on Windows the helper's redirected stdout gains
          # CRLF line endings; a stray \r would corrupt the path. The list
          # uses the native path-list separator consumed by --files.
          #
          # Any non-zero exit propagates red: real test failures, or the
          # runner's own zero-run guard (every file filtered to empty by
          # ``-m`` — "must never pass without running its OS's tests").
          SEPARATOR="$(python -c 'import os, sys; sys.stdout.write(os.pathsep)')"
          FILES="$(tr -d '\r' < "$LIST" | paste -sd "$SEPARATOR" -)"
          scripts/run_tests.sh --files "$FILES" -- \
            ${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"} \
            -m "platforms and not integration" \
            -v --tb=short
        env:
          # These files spawn nested PowerShell trees. CPU-count parallelism
          # starves cold child startup before the fixture's idle deadline.
          HERMES_TEST_WORKERS: ${{ matrix.marker == 'windows' && (runner.arch == 'ARM64' && '2' || '8') || '' }}
          # Belt-and-suspenders with tests/conftest.py's env blanking: no
          # test may reach a real provider API.
          OPENROUTER_API_KEY: ''
          OPENAI_API_KEY: ''
          NOUS_API_KEY: ''

  e2e-windows:
    # Real Hermes processes on a real Windows host (tests/e2e/core/windows): hermes.exe,
    # `hermes serve`, `hermes gateway run/stop`, the tui_gateway stdio process, cron script
    # jobs, the terminal tool's Git Bash / PowerShell / cmd children and a ConPTY console,
    # all against the loopback fake provider (tests/fakes/fake_llm_provider.py). Separate
    # from os-tests above: those are unit-level Windows tests, these spawn process
    # trees per test. Every file there is also marked `integration`, so os-tests'
    # `-m "platforms and not integration"` never runs them a second time.
    #
    # One pytest process per FILE, in parallel (same model as the Linux `e2e` job): the
    # files spawn their own trees and must not share interpreter state. No retries: a
    # Windows race that passes on retry is still a race. Tracked Windows bugs are strict
    # xfails (per-file KNOWN tables), so a fix turns this job red until the entry goes.
    name: Windows E2E (real processes)
    # 32-core like os-tests' Windows row: seven files run in parallel and each spawns
    # process trees; on the 4-vCPU image the suite step took 153-171 s (job 4.7-5.6 min),
    # here 124 s (job 3.1 min), with headroom against scheduling-starved timeouts.
    runs-on: windows-latest-32-core
    timeout-minutes: 25
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up locked Python and test dependencies
        uses: ./.github/actions/setup-pm
        with:
          packages: git,ripgrep
          extras: '["all", "telegram", "anthropic", "mistral", "fal", "modal", "daytona", "parallel-web"]'
          test-environment: 'true'
          prune-python-cache: true

      - name: Run Windows E2E suite
        shell: bash
        run: |
          scripts/run_tests.sh tests/e2e/core/windows/ -- \
            -m "platforms and integration" -v --tb=short -rA
        env:
          HERMES_TEST_WORKERS: '6'
          HERMES_TEST_FILE_TIMEOUT: '900'
          HERMES_TEST_FILE_RETRIES: '0'
          OPENROUTER_API_KEY: ''
          OPENAI_API_KEY: ''
          NOUS_API_KEY: ''

      - name: No leftover Python / Hermes processes
        # Backstop for the per-test ownership checks: a file killed by its 900 s timeout,
        # or a detached grandchild some test's cleanup missed, leaves an interpreter behind
        # (Windows never re-parents, so nothing reaps it). The runner itself runs no
        # python.exe / hermes.exe once the suite is done.
        if: always()
        shell: pwsh
        run: |
          $names = 'python.exe', 'pythonw.exe', 'hermes.exe'
          $deadline = (Get-Date).AddSeconds(20)
          do {
            $left = @(Get-CimInstance Win32_Process | Where-Object { $names -contains $_.Name.ToLower() })
            if ($left.Count -eq 0) { 'no leftover python/hermes processes'; exit 0 }
            Start-Sleep -Milliseconds 500
          } while ((Get-Date) -lt $deadline)
          foreach ($p in $left) {
            "::error::leftover pid=$($p.ProcessId) ppid=$($p.ParentProcessId): $($p.CommandLine)"
          }
          exit 1