Files
hermes-agent/agent/command_token_source.py
Hayden Moulds c111ede3e5 fix(picker): resolve key_cmd credentials for model discovery
`key_cmd` (#86891) authenticates a provider with a SHORT-LIVED bearer minted
by a command — SSO/OIDC brokers, cloud IAM, internal auth proxies. The
request path has honoured it since it landed, but the picker resolved probe
credentials from `api_key`/`key_env` ONLY, so a key_cmd provider probed
`/v1/models` with an EMPTY key.

Against an authenticated endpoint the probe 401s, discovery returns nothing,
and the provider falls back to its single configured default model. The
picker shows ONE model, indistinguishable from an endpoint that genuinely
serves one — while inference keeps working, because that path mints
correctly. Reproduced against a LiteLLM gateway behind Entra OIDC: 0 models
discovered with an empty key, 26 with the minted token.

Both picker probe sites already funnel through `_entry_credentials()`, so
the fix lands in one place: it now reports a `cmd:<key_cmd>` identity, and
each site falls back to `resolve_probe_token()` after api_key/key_env. An
explicit static key still wins, so existing configs are unaffected.

The identity is keyed on the COMMAND, never the minted token: the token
rotates on every refresh, so keying on its value would change the group
fingerprint constantly and force a re-probe on every open. Two entries on
one URL with different helpers still get distinct rows.

`resolve_probe_token()` lives in agent.command_token_source, which already
owns key_cmd minting, and shares the CommandTokenSource cache with the
request path — a cache read, not a fresh sign-in. Fail-closed: a helper
needing an interactive sign-in degrades to today's empty-key behaviour
rather than taking down every other provider's row.

`_model_flow_named_custom` (the `hermes model` setup flow) is the sibling
path — it builds its own `Authorization: Bearer` from the same incomplete
resolution — and is fixed the same way, with one ordering constraint: the
value persisted to config.yaml is computed BEFORE the mint, so a short-lived
bearer can never be written back to shadow the key_cmd meant to re-mint it.

Tests drive the real code paths and assert on the credential each probe
receives rather than on function source, so a semantics-preserving refactor
does not fail them. Verified they fail with the fix reverted.
2026-09-07 21:22:49 -07:00

165 lines
7.0 KiB
Python

"""Mint a provider API key by running a command (``key_cmd``).
Enterprise gateways (SSO/OIDC brokers, cloud IAM, auth proxies) issue SHORT-LIVED bearers; a key
copied into ``.env`` goes stale within the hour. ``key_cmd`` names a command that PRINTS a token
(the ``apiKeyHelper`` / ``gcloud auth print-access-token`` idiom). Both wire clients accept a
callable API key and invoke it per request; the token is cached until shortly before expiry.
Output contract: ONLY the token on stdout, bare or as JSON with an ``access_token`` field
(``expires_in`` / ISO ``expiry`` honoured). Precedence: explicit ``--api-key`` wins (one-off
recovery escape hatch); otherwise ``key_cmd`` beats a static ``api_key`` / ``key_env``.
"""
from __future__ import annotations
import json
import logging
import subprocess
import threading
import time
from typing import Callable, Optional
logger = logging.getLogger(__name__)
# Treat a token as spent slightly before expiry so a request can't be signed with one that dies in
# flight (60s = usual OAuth cache leeway).
_TOKEN_REFRESH_LEEWAY_SECONDS = 60.0
# Helpers answer from a local cache in milliseconds; this long means hung.
_MINT_TIMEOUT_SECONDS = 15
# No advertised expiry: nothing in the request path re-mints on 401 (the SDK retries 429/5xx only), so
# a process-lifetime cache would 401 forever once the token died. Re-mint on a bounded window instead.
_NO_TTL_REFRESH_SECONDS = 900.0
class CommandTokenError(RuntimeError):
"""A ``key_cmd`` failed to produce a usable token."""
def materialize_probe_api_key(api_key: object) -> str:
"""Best-effort probe credential; never send a callable's repr or log mint errors."""
try:
token = api_key() if callable(api_key) else api_key
except Exception:
return ""
return token.strip() if isinstance(token, str) else ""
def _mint(command: str, label: str) -> tuple[str, Optional[float]]:
"""Run *command*, returning ``(token, ttl_seconds_or_None)``."""
try:
completed = subprocess.run(
command, shell=True, capture_output=True, text=True, timeout=_MINT_TIMEOUT_SECONDS,
)
except subprocess.TimeoutExpired as exc:
raise CommandTokenError(
f"key_cmd for provider {label!r} timed out after {_MINT_TIMEOUT_SECONDS}s"
) from exc
except OSError as exc:
raise CommandTokenError(f"key_cmd for provider {label!r} could not be executed: {exc}") from exc
if completed.returncode != 0:
# NEVER include stdout/stderr (may hold a token) or the command string (may embed
# `--client-secret=…`); name the provider instead.
raise CommandTokenError(
f"key_cmd for provider {label!r} exited {completed.returncode}. "
f"Run that provider's key_cmd manually to see why "
f"(e.g. `databricks auth login` if its OAuth session expired)."
)
stdout = completed.stdout or ""
if not stdout.strip():
raise CommandTokenError(f"key_cmd for provider {label!r} produced no output")
# JSON payload — the shape `databricks auth token --output json` prints.
if stdout.lstrip().startswith("{"):
try:
payload = json.loads(stdout)
except json.JSONDecodeError:
payload = None
if isinstance(payload, dict):
token = str(payload.get("access_token") or "").strip()
if not token:
raise CommandTokenError(
f"key_cmd for provider {label!r} returned JSON without an 'access_token' field"
)
ttl = payload.get("expires_in")
if isinstance(ttl, (int, float)) and ttl > 0:
return token, float(ttl)
# CLI helpers often print an absolute ISO 8601 deadline instead of OAuth's relative
# lifetime; honour it or the token 401s once past. Lazy import: hermes_cli.auth imports agent.*.
from hermes_cli.auth import _parse_iso_timestamp
for field in ("expiry", "expiresOn"):
deadline = _parse_iso_timestamp(payload.get(field))
remaining = deadline - time.time() if deadline is not None else 0
if remaining > 0:
return token, remaining
return token, None
# Bare token: stdout carries the token and nothing else. Do NOT keep one line of several — that
# turns a misconfigured helper (banner, warning) into a corrupt-key 401 far harder to diagnose.
token = stdout.strip()
if "\n" in token:
raise CommandTokenError(
f"key_cmd for provider {label!r} printed multiple lines; it must "
"print only the token (or JSON with an 'access_token' field)"
)
return token, None
class CommandTokenSource:
"""Callable returning a bearer token, cached until shortly before expiry."""
def __init__(self, command: str, label: str = "custom") -> None:
self._command = command
self._label = label or "custom"
self._lock = threading.Lock()
self._token = ""
self._expires_at: float = 0.0
def __call__(self) -> str:
with self._lock:
if self._token and time.monotonic() < self._expires_at:
return self._token
token, ttl = _mint(self._command, self._label)
self._token = token
self._expires_at = time.monotonic() + (
max(ttl - _TOKEN_REFRESH_LEEWAY_SECONDS, 5.0) if ttl else _NO_TTL_REFRESH_SECONDS
)
logger.debug(
"Minted key_cmd token for provider %s (ttl=%s)",
self._label, f"{int(ttl)}s" if ttl else "unknown",
)
return token
def build_command_token_provider(key_cmd: str, provider_label: str = "custom") -> Optional[Callable[[], str]]:
"""A per-request token provider for *key_cmd*, or ``None`` when unset."""
command = str(key_cmd or "").strip()
return CommandTokenSource(command, provider_label) if command else None
def resolve_probe_token(entry: dict) -> str:
"""Mint a one-shot credential from a provider entry's ``key_cmd``, or "".
For callers needing a CONCRETE token rather than the per-request callable
``build_command_token_provider`` returns — the ``/models`` catalog probes,
which build their request by hand instead of going through a wire client.
Shares the ``CommandTokenSource`` cache with the request path, so this is
a cache read rather than a fresh sign-in.
Fail-closed: any error yields "". A helper that needs an interactive
sign-in (or is simply broken) must not take down a whole picker — the
caller degrades to the pre-existing empty-key behaviour and every other
provider still renders.
"""
if not isinstance(entry, dict):
return ""
command = str(entry.get("key_cmd", "") or "").strip()
if not command:
return ""
try:
provider = build_command_token_provider(command, str(entry.get("name", "") or "custom"))
return (provider() or "").strip() if provider is not None else ""
except Exception:
return ""