ci: nightly + release-tag live provider canary workflow
Runs tests/e2e/core/live (`-m live`) nightly, on `v*` tags and on workflow_dispatch (optional -k filter). Secrets-gated on LIVE_*_API_KEY repo secrets (each case skips without its key; the job no-ops when none are configured), main-repo only, one run per ref (never cancels a release gate), 25-minute timeout. Uses direct pytest because scripts/run_tests.sh starts from `env -i` so no credential can reach a test. Publishes a usage/cost table to the step summary and uploads junit + usage JSONL. (cherry picked from commit 75c5656ff3905512bf93c1fd887bbd5e85396f29)
This commit is contained in:
teknium1
128
.github/workflows/live-providers.yml
vendored
Normal file
128
.github/workflows/live-providers.yml
vendored
Normal file
@@ -0,0 +1,128 @@
|
||||
name: Live provider canaries
|
||||
|
||||
# Secrets-gated LIVE canary against real LLM provider APIs (tests/e2e/core/live,
|
||||
# pytest marker `live`). Catches what mocks cannot: vendor-side request-schema
|
||||
# drift, reasoning-replay rules, streaming shape changes, prompt-cache hits and
|
||||
# real credential routing / `/models` parsing. Cheap models, <=3 turns per case,
|
||||
# a hard per-test token and dollar guard; typical full run is well under $0.50.
|
||||
#
|
||||
# Each case skips cleanly when its secret is absent, so the job is safe to run
|
||||
# with any subset configured. Use dedicated, spend-capped keys:
|
||||
# LIVE_OPENROUTER_API_KEY, LIVE_ANTHROPIC_API_KEY, LIVE_NOUS_API_KEY,
|
||||
# LIVE_OPENAI_API_KEY, LIVE_GEMINI_API_KEY, LIVE_XAI_API_KEY
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 7 * * *" # nightly, 07:17 UTC
|
||||
push:
|
||||
tags:
|
||||
- "v*" # release gate
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
description: "pytest -k expression (e.g. 'openrouter-anthropic or models_listing')"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# One live run per ref at a time; never cancel a release-tag gate half way.
|
||||
concurrency:
|
||||
group: live-providers-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
live:
|
||||
name: Live provider canaries
|
||||
# Forks and PRs never see these secrets; only run where they exist.
|
||||
if: github.repository == 'NousResearch/hermes-agent'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.LIVE_ANTHROPIC_API_KEY }}
|
||||
NOUS_API_KEY: ${{ secrets.LIVE_NOUS_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.LIVE_OPENAI_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.LIVE_GEMINI_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.LIVE_XAI_API_KEY }}
|
||||
HERMES_LIVE_USAGE_FILE: ${{ github.workspace }}/live-usage.jsonl
|
||||
steps:
|
||||
- name: Check that at least one provider secret is configured
|
||||
id: gate
|
||||
run: |
|
||||
n=0
|
||||
for v in OPENROUTER_API_KEY ANTHROPIC_API_KEY NOUS_API_KEY OPENAI_API_KEY GEMINI_API_KEY XAI_API_KEY; do
|
||||
if [ -n "${!v}" ]; then n=$((n+1)); echo "configured: $v"; fi
|
||||
done
|
||||
echo "configured=$n" >> "$GITHUB_OUTPUT"
|
||||
if [ "$n" -eq 0 ]; then
|
||||
echo "::notice::No LIVE_* provider secrets configured; live canaries skipped."
|
||||
fi
|
||||
|
||||
- name: Checkout code
|
||||
if: steps.gate.outputs.configured != '0'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Install uv
|
||||
if: steps.gate.outputs.configured != '0'
|
||||
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # 8.2.0
|
||||
with:
|
||||
version: "0.9.28"
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Set up Python 3.11
|
||||
if: steps.gate.outputs.configured != '0'
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: uv python install 3.11
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.gate.outputs.configured != '0'
|
||||
uses: ./.github/actions/retry
|
||||
with:
|
||||
command: uv sync --locked --python 3.11 --extra all --extra dev --extra anthropic
|
||||
|
||||
- name: Run live canaries
|
||||
if: steps.gate.outputs.configured != '0'
|
||||
# Direct pytest, not scripts/run_tests.sh: the canonical runner starts from
|
||||
# `env -i` precisely so no credential can reach a test, which would skip
|
||||
# every case here. `-m live` overrides the default `not live` addopts.
|
||||
env:
|
||||
LIVE_FILTER: ${{ inputs.filter }}
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
args=(tests/e2e/core/live -m live -rsxX -p no:cacheprovider --tb=short --junitxml=live-junit.xml)
|
||||
if [ -n "$LIVE_FILTER" ]; then args+=(-k "$LIVE_FILTER"); fi
|
||||
python -m pytest "${args[@]}"
|
||||
|
||||
- name: Usage and cost summary
|
||||
if: always() && steps.gate.outputs.configured != '0'
|
||||
shell: python
|
||||
run: |
|
||||
import json, os
|
||||
rows = ["### Live provider canary usage", "",
|
||||
"| case | model | calls | input | output | cache read | cache write | est. $ | ceiling $ |",
|
||||
"|---|---|---|---|---|---|---|---|---|"]
|
||||
path = os.environ["HERMES_LIVE_USAGE_FILE"]
|
||||
if os.path.exists(path):
|
||||
for line in open(path, encoding="utf-8"):
|
||||
u = json.loads(line)
|
||||
rows.append("| {case} | {model} | {api_calls} | {input} | {output} | {cache_read} "
|
||||
"| {cache_write} | {hermes_est_usd} | {list_price_ceiling_usd} |".format(**u))
|
||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fh:
|
||||
fh.write("\n".join(rows) + "\n")
|
||||
|
||||
- name: Upload results
|
||||
if: always() && steps.gate.outputs.configured != '0'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: live-provider-canaries
|
||||
path: |
|
||||
live-junit.xml
|
||||
live-usage.jsonl
|
||||
if-no-files-found: ignore
|
||||
retention-days: 30
|
||||
Reference in New Issue
Block a user