hermes-agent/.github/workflows/tests.yml

name: Tests

on:
  workflow_call:
    inputs:
      # The E2E jobs below are slow lanes (scripts/ci/classify_changes.py).
      # ci.yaml passes the classifier's answer; a caller that passes nothing
      # gets every job.
      e2e:
        description: Run the e2e job (tests/e2e except the upgrade suite).
        type: boolean
        required: false
        default: true
      e2e_upgrade:
        description: Run the e2e-upgrade shards (tests/e2e/core/upgrade).
        type: boolean
        required: false
        default: true
      strict_acceptance:
        description: >-
          Acceptance runs only: known_failure owners whose gaps FAIL this run instead of xfailing ('upd-txn', a comma list, or '1' for every gap). Sets HERMES_E2E_STRICT_ACCEPTANCE for the E2E suites (tests/e2e/core/_pending_fixes.py).
        type: string
        required: false
        default: ''

permissions:
  contents: read

# Cancel in-progress runs for the same ref
concurrency:
  group: tests-${{ github.ref_type == 'tag' && github.run_id || github.ref }}
  cancel-in-progress: ${{ github.ref_type != 'tag' }}

jobs:
  test:
    # Two 32-core slices instead of one box: on one runner the suite is ~10 min of
    # nearly fully packed work, and a single bigger runner (96-core) scaled ~1.8x for
    # 3x the price because files contend for the shared machine. Separate machines
    # scale linearly. run_tests_parallel.py partitions from the checkout alone, so
    # both slices agree on the split without a planning job.
    name: Run tests (${{ matrix.slice }}/2)
    runs-on: ubuntu-latest-32-core
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        slice: [1, 2]
    steps:
      - name: Checkout code
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1

      - name: Install ripgrep (prebuilt binary)
        run: |
          set -euo pipefail
          RG_VERSION=15.1.0
          RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
          RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
          curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
            "https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
          echo "${RG_SHA256}  ${RG_TARBALL}" | sha256sum -c -
          tar -xzf "$RG_TARBALL"
          sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
          rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
          rg --version

      - name: Set up locked Python and test dependencies
        uses: ./.github/actions/setup-pm
        with:
          extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "parallel-web"]'
          test-environment: 'true'
          prune-python-cache: true

      - name: Check SQLite runs WAL
        # Hermes picks DELETE journal mode on a SQLite with the WAL-reset
        # bug. If the PM-pinned interpreter links one, every WAL test silently
        # runs DELETE (or skips), so fail loudly instead. Tests that need DELETE
        # select it explicitly (journal_mode config, a pinned vulnerable
        # version), so they run on any interpreter.
        run: |
          "$HERMES_PYTHON" -c "import sqlite3, sys, hermes_state_wal as w; v = w.is_sqlite_wal_reset_vulnerable(); print(f'Python {sys.version.split()[0]}, SQLite {sqlite3.sqlite_version}, WAL-reset vulnerable: {v}'); sys.exit(1 if v else 0)"

      - name: Restore per-file duration cache
        # scripts/run_tests_parallel.py raises a file's timeout to
        # 3x its last healthy duration (_effective_file_timeout) so a
        # known-slow file dilated by load is not SIGKILL'd at the flat cap
        # and laundered into a FLAKY retry. The scaler reads
        # test_durations.json from the checkout; without this restore the
        # file is absent on a fresh runner and the scaler is inert.
        # Exact key never matches (run_id differs); restore-keys picks the
        # most recent cache saved by a main push. PRs read, only main writes.
        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: test_durations.json
          key: test-durations-never-exact
          restore-keys: |
            test-durations-

      - name: Run tests
        # Per-file isolation via scripts/run_tests.sh: each test file runs
        # in its own freshly-spawned `python -m pytest <file>` subprocess
        # with bounded parallelism. No xdist, no shared workers, no
        # module-level state leakage between files.
        #
        # No --files: the runner discovers the suite itself. The discovered
        # set is identical to the list the removed matrix job used to pass in.
        run: |
          scripts/run_tests.sh
        env:
          HERMES_TEST_SLICE: ${{ matrix.slice }}/2
          HERMES_TEST_WORKERS: 32
          HERMES_TEST_FILE_TIMEOUT: 600
          # Ensure tests don't accidentally call real APIs
          OPENROUTER_API_KEY: ""
          OPENAI_API_KEY: ""
          NOUS_API_KEY: ""

      - name: Save per-file duration cache (main only)
        # Only green first-attempt durations are written by the runner, so
        # a hang on main cannot ratchet its own bound upward.
        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && hashFiles('test_durations.json') != ''
        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: test_durations.json
          # Per-slice key: each slice's file keeps the restored entries for the other
          # slice's files and refreshes its own, so the cache stays complete.
          key: test-durations-${{ github.run_id }}-${{ matrix.slice }}

  e2e:
    # The core suites spawn real serve / gateway / tui_gateway / MCP / SQLite
    # writer processes per test; a 4-vCPU runner serialises them into timeouts.
    if: inputs.e2e
    runs-on: ubuntu-latest-32-core
    timeout-minutes: 30
    steps:
      - name: Checkout code
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1

      - name: Set up locked Python, Node and test dependencies
        # toolchain: all puts PM's locked Node in the runner's PM store, not
        # only on PATH. The E2E sandboxes copy node from that store
        # (tests/e2e/core/_pm_dependencies.py) and forbid lazy installs, and
        # the dashboard's /api/pty resolves the TUI's node through PM only.
        # A setup-node binary on PATH is invisible to PM, so every /api/pty
        # chat refused to start. The locked Node also satisfies the
        # boot-contract suite (backend-ready.ts needs Node >= 22.18).
        uses: ./.github/actions/setup-pm
        with:
          toolchain: all
          # `messaging`: tests/e2e/core/platforms runs the real Telegram/Discord/Slack
          # adapters (their SDKs) against local stand-ins.
          extras: '["all", "anthropic", "bedrock", "mistral", "fal", "modal", "daytona", "parallel-web", "messaging"]'
          test-environment: 'true'
          prune-python-cache: true

      - name: Install Node deps and build the Ink TUI
        # tests/e2e/core/terminal drives the real `hermes --tui` over a PTY;
        # HERMES_E2E_REQUIRE_TUI=1 below turns a missing build into a failure.
        uses: ./.github/actions/retry
        with:
          command: npm ci --workspace ui-tui --ignore-scripts --no-audit --no-fund && npm run build --workspace ui-tui

      - name: Install ripgrep (prebuilt binary)
        run: |
          set -euo pipefail
          RG_VERSION=15.1.0
          RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
          RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
          curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
            "https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
          echo "${RG_SHA256}  ${RG_TARBALL}" | sha256sum -c -
          tar -xzf "$RG_TARBALL"
          sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
          rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
          rg --version

      - name: Require a WAL-capable SQLite
        # The state.db suites skip on a WAL-reset-vulnerable SQLite; fail
        # instead of reporting green over zero coverage.
        run: |
          "$HERMES_PYTHON" -c "import sqlite3, hermes_state_wal as w; print('sqlite', sqlite3.sqlite_version); assert not w.is_sqlite_wal_reset_vulnerable()"

      - name: Run e2e tests
        # One subprocess per file, in parallel: the core suites spawn real
        # processes (serve, gateway, tui_gateway, MCP servers, SQLite writers)
        # and must not share interpreter state.
        # tests/e2e/core/upgrade runs in its own job (e2e-upgrade) below.
        run: |
          mapfile -t files < <(find tests/e2e -name 'test_*.py' -not -path 'tests/e2e/core/upgrade/*' | sort)
          scripts/run_tests.sh --include-integration "${files[@]}"
        env:
          HERMES_E2E_STRICT_ACCEPTANCE: ${{ inputs.strict_acceptance }}
          # An e2e file is a process tree, not one CPU-bound pytest worker. With
          # 32 files at once the hosted run starved live children before their
          # ready frames (empty stderr, still alive) and hit 60–120s deadlines
          # across unrelated SQLite, tenancy, provider and terminal suites.
          # Bound the trees, not the child readiness/turn timeouts. At 4 the
          # PTY turn and serve-SIGTERM deadlines still starved intermittently.
          HERMES_TEST_WORKERS: "3"
          # Multi-process episodes (torture chamber, compaction kill -9,
          # gateway liveness, delivery exactly-once through a real gateway,
          # the cron virtual-clock soak) legitimately run past the 300 s default.
          HERMES_TEST_FILE_TIMEOUT: "900"
          # No automatic re-run of a failed file: the torture chamber and the
          # exactly-once/compaction suites are race detectors, and a rare
          # corruption that passes on retry is still a corruption.
          HERMES_TEST_FILE_RETRIES: "0"
          HERMES_E2E_REQUIRE_TUI: "1"
          OPENROUTER_API_KEY: ""
          OPENAI_API_KEY: ""
          NOUS_API_KEY: ""

  e2e-upgrade-plan:
    # One e2e-upgrade shard per suite directory: the files directly under
    # tests/e2e/core/upgrade ("core") plus every subdirectory that holds
    # test_*.py (pm, git, handoff, network, hosts, ...). A new suite
    # directory gets its own shard without editing this file, and no shard
    # carries the whole tree's wall time against the 60-minute timeout.
    # Skipping this job skips every e2e-upgrade shard with it.
    if: inputs.e2e_upgrade
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      shards: ${{ steps.plan.outputs.shards }}
    steps:
      - name: Checkout code
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1
        with:
          sparse-checkout: tests/e2e/core/upgrade
      - name: List suite directories
        id: plan
        run: |
          set -euo pipefail
          shards=$(python3 - <<'PY'
          import json, pathlib
          root = pathlib.Path("tests/e2e/core/upgrade")
          shards = ["core"] if any(root.glob("test_*.py")) else []
          shards += sorted(d.name for d in root.iterdir()
                           if d.is_dir() and not d.name.startswith(("_", ".")) and any(d.rglob("test_*.py")))
          print(json.dumps(shards))
          PY
          )
          echo "shards=$shards" | tee -a "$GITHUB_OUTPUT"

  e2e-upgrade:
    # tests/e2e/core/upgrade: a real N-1 -> HEAD `hermes update` (clean,
    # autostash, killed mid-pull / mid-deps, offline), fresh-process import and
    # entrypoint smoke, the config round-trip property matrix, and the lane
    # suites below it (pm: PM environment lifecycle across updates, ...).
    # Its own job: it needs full history + tags, bubblewrap, and one file
    # runs ~5-15 min. Sharded by suite directory (see e2e-upgrade-plan).
    name: e2e-upgrade (${{ matrix.shard }})
    needs: e2e-upgrade-plan
    runs-on: ubuntu-latest-32-core
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        shard: ${{ fromJSON(needs.e2e-upgrade-plan.outputs.shards) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7.0.1
        with:
          # N-1 = `git describe --tags --abbrev=0 HEAD~1`.
          fetch-depth: 0
          fetch-tags: true

      - name: Set up Node
        # `hermes update` builds the web UI / TUI workspaces.
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # 7.0.0
        with:
          node-version: 26

      - name: Install ripgrep (prebuilt binary)
        run: |
          set -euo pipefail
          RG_VERSION=15.1.0
          RG_SHA256=1c9297be4a084eea7ecaedf93eb03d058d6faae29bbc57ecdaf5063921491599
          RG_TARBALL=ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz
          curl -sSfL --retry 3 --retry-delay 5 -o "$RG_TARBALL" \
            "https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${RG_TARBALL}"
          echo "${RG_SHA256}  ${RG_TARBALL}" | sha256sum -c -
          tar -xzf "$RG_TARBALL"
          sudo mv "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg
          rm -rf "$RG_TARBALL" "ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl"
          rg --version

      - name: Install uv
        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
        with:
          # Pin the uv version: unpinned, setup-uv resolves "latest" by
          # fetching a manifest from raw.githubusercontent.com on EVERY job —
          # a transient fetch failure fails the whole job (2026-07-28 slice-5
          # incident). Pinned, the binary downloads directly; no manifest hop.
          # Newer than the unit job's pin on purpose: 0.9.28 only knows CPython
          # 3.11.14, whose bundled SQLite has the WAL-reset bug, so Hermes runs
          # state.db in DELETE mode and the WAL torture chamber would skip.
          version: "0.12.13"
          # Persist uv's download/wheel cache (~/.cache/uv) across runs.
          # Keyed on the dependency manifests, so the cache is reused until
          # pyproject.toml or uv.lock changes. `uv sync` still runs every
          # time, but resolves from the warm cache instead of re-downloading
          # and re-building wheels.
          enable-cache: true
          cache-dependency-glob: |
            pyproject.toml
            uv.lock

      - name: Set up Python 3.14
        run: uv python install 3.14

      - name: Install dependencies
        # `uv sync --locked` installs the exact pinned set from uv.lock (and
        # fails if the lock is out of sync with pyproject.toml), giving a
        # reproducible env. It also creates .venv itself, so no separate
        # `uv venv` step is needed.
        #
        # Same extras as the test job's sync above: the hermetic test env
        # forbids mid-run pip installs (HERMES_DISABLE_LAZY_INSTALLS=1 in
        # tests/conftest.py), so lazy-install SDKs exercised by tests must be
        # in the venv up front.
        uses: ./.github/actions/retry
        with:
          command: uv sync --locked --python 3.14 --extra all --group dev --extra anthropic --extra mistral --extra fal --extra modal --extra daytona --extra parallel-web

      - name: Install bubblewrap
        # Every spawned updater runs in bwrap (own PID namespace, no user
        # systemd bus, only the test tmp writable). Ubuntu 24.04 restricts
        # unprivileged user namespaces, which bwrap needs.
        run: |
          sudo apt-get update -qq
          sudo apt-get install -y -qq bubblewrap
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
          # The suite's own probe (_bwrap_usable, same flags as every sandboxed
          # run): without it the suite silently runs the updater unsandboxed.
          source .venv/bin/activate
          python -c "from tests.e2e.core.upgrade._helpers import BWRAP_OK; assert BWRAP_OK, 'bwrap unusable'" && echo "bwrap ok"

      - name: Require a WAL-capable SQLite
        # The state.db suites skip on a WAL-reset-vulnerable SQLite; fail
        # instead of reporting green over zero coverage.
        run: |
          source .venv/bin/activate
          python -c "import sqlite3, hermes_state_wal as w; print('sqlite', sqlite3.sqlite_version); assert not w.is_sqlite_wal_reset_vulnerable()"

      - name: Run upgrade e2e tests (${{ matrix.shard }})
        run: |
          set -euo pipefail
          source .venv/bin/activate
          if [ "$SHARD" = core ]; then
            mapfile -t files < <(find tests/e2e/core/upgrade -maxdepth 1 -name 'test_*.py' | sort)
          else
            mapfile -t files < <(find "tests/e2e/core/upgrade/$SHARD" -name 'test_*.py' | sort)
          fi
          # A shard that collects nothing is a wiring bug, not a pass.
          [ "${#files[@]}" -gt 0 ] || { echo "no test files for shard $SHARD" >&2; exit 1; }
          printf 'shard %s: %s\n' "$SHARD" "${files[*]}"
          scripts/run_tests.sh --include-integration "${files[@]}"
        env:
          SHARD: ${{ matrix.shard }}
          HERMES_E2E_STRICT_ACCEPTANCE: ${{ inputs.strict_acceptance }}
          # Each file drives real installs/updaters (process trees plus uv and
          # git I/O); cpu_count of them at once starves their deadlines. A
          # shard holds at most ~7 files, so this runs each shard in one wave.
          HERMES_TEST_WORKERS: "6"
          HERMES_TEST_FILE_TIMEOUT: "3000"
          OPENROUTER_API_KEY: ""
          OPENAI_API_KEY: ""
          NOUS_API_KEY: ""