diff --git a/.github/workflows/live-providers.yml b/.github/workflows/live-providers.yml new file mode 100644 index 0000000000..6ea23f0edc --- /dev/null +++ b/.github/workflows/live-providers.yml @@ -0,0 +1,128 @@ +name: Live provider canaries + +# Secrets-gated LIVE canary against real LLM provider APIs (tests/e2e/core/live, +# pytest marker `live`). Catches what mocks cannot: vendor-side request-schema +# drift, reasoning-replay rules, streaming shape changes, prompt-cache hits and +# real credential routing / `/models` parsing. Cheap models, <=3 turns per case, +# a hard per-test token and dollar guard; typical full run is well under $0.50. +# +# Each case skips cleanly when its secret is absent, so the job is safe to run +# with any subset configured. Use dedicated, spend-capped keys: +# LIVE_OPENROUTER_API_KEY, LIVE_ANTHROPIC_API_KEY, LIVE_NOUS_API_KEY, +# LIVE_OPENAI_API_KEY, LIVE_GEMINI_API_KEY, LIVE_XAI_API_KEY + +on: + schedule: + - cron: "17 7 * * *" # nightly, 07:17 UTC + push: + tags: + - "v*" # release gate + workflow_dispatch: + inputs: + filter: + description: "pytest -k expression (e.g. 'openrouter-anthropic or models_listing')" + required: false + default: "" + +permissions: + contents: read + +# One live run per ref at a time; never cancel a release-tag gate half way. +concurrency: + group: live-providers-${{ github.ref }} + cancel-in-progress: false + +jobs: + live: + name: Live provider canaries + # Forks and PRs never see these secrets; only run where they exist. + if: github.repository == 'NousResearch/hermes-agent' + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.LIVE_ANTHROPIC_API_KEY }} + NOUS_API_KEY: ${{ secrets.LIVE_NOUS_API_KEY }} + OPENAI_API_KEY: ${{ secrets.LIVE_OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.LIVE_GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.LIVE_XAI_API_KEY }} + HERMES_LIVE_USAGE_FILE: ${{ github.workspace }}/live-usage.jsonl + steps: + - name: Check that at least one provider secret is configured + id: gate + run: | + n=0 + for v in OPENROUTER_API_KEY ANTHROPIC_API_KEY NOUS_API_KEY OPENAI_API_KEY GEMINI_API_KEY XAI_API_KEY; do + if [ -n "${!v}" ]; then n=$((n+1)); echo "configured: $v"; fi + done + echo "configured=$n" >> "$GITHUB_OUTPUT" + if [ "$n" -eq 0 ]; then + echo "::notice::No LIVE_* provider secrets configured; live canaries skipped." + fi + + - name: Checkout code + if: steps.gate.outputs.configured != '0' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install uv + if: steps.gate.outputs.configured != '0' + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0 + with: + version: "0.9.28" + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Set up Python 3.11 + if: steps.gate.outputs.configured != '0' + uses: ./.github/actions/retry + with: + command: uv python install 3.11 + + - name: Install dependencies + if: steps.gate.outputs.configured != '0' + uses: ./.github/actions/retry + with: + command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic + + - name: Run live canaries + if: steps.gate.outputs.configured != '0' + # Direct pytest, not scripts/run_tests.sh: the canonical runner starts from + # `env -i` precisely so no credential can reach a test, which would skip + # every case here. `-m live` overrides the default `not live` addopts. + env: + LIVE_FILTER: ${{ inputs.filter }} + run: | + source .venv/bin/activate + args=(tests/e2e/core/live -m live -rsxX -p no:cacheprovider --tb=short --junitxml=live-junit.xml) + if [ -n "$LIVE_FILTER" ]; then args+=(-k "$LIVE_FILTER"); fi + python -m pytest "${args[@]}" + + - name: Usage and cost summary + if: always() && steps.gate.outputs.configured != '0' + shell: python + run: | + import json, os + rows = ["### Live provider canary usage", "", + "| case | model | calls | input | output | cache read | cache write | est. $ | ceiling $ |", + "|---|---|---|---|---|---|---|---|---|"] + path = os.environ["HERMES_LIVE_USAGE_FILE"] + if os.path.exists(path): + for line in open(path, encoding="utf-8"): + u = json.loads(line) + rows.append("| {case} | {model} | {api_calls} | {input} | {output} | {cache_read} " + "| {cache_write} | {hermes_est_usd} | {list_price_ceiling_usd} |".format(**u)) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh: + fh.write("\n".join(rows) + "\n") + + - name: Upload results + if: always() && steps.gate.outputs.configured != '0' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: live-provider-canaries + path: | + live-junit.xml + live-usage.jsonl + if-no-files-found: ignore + retention-days: 30