`key_cmd` (#86891) authenticates a provider with a SHORT-LIVED bearer minted by a command — SSO/OIDC brokers, cloud IAM, internal auth proxies. The request path has honoured it since it landed, but the picker resolved probe credentials from `api_key`/`key_env` ONLY, so a key_cmd provider probed `/v1/models` with an EMPTY key. Against an authenticated endpoint the probe 401s, discovery returns nothing, and the provider falls back to its single configured default model. The picker shows ONE model, indistinguishable from an endpoint that genuinely serves one — while inference keeps working, because that path mints correctly. Reproduced against a LiteLLM gateway behind Entra OIDC: 0 models discovered with an empty key, 26 with the minted token. Both picker probe sites already funnel through `_entry_credentials()`, so the fix lands in one place: it now reports a `cmd:<key_cmd>` identity, and each site falls back to `resolve_probe_token()` after api_key/key_env. An explicit static key still wins, so existing configs are unaffected. The identity is keyed on the COMMAND, never the minted token: the token rotates on every refresh, so keying on its value would change the group fingerprint constantly and force a re-probe on every open. Two entries on one URL with different helpers still get distinct rows. `resolve_probe_token()` lives in agent.command_token_source, which already owns key_cmd minting, and shares the CommandTokenSource cache with the request path — a cache read, not a fresh sign-in. Fail-closed: a helper needing an interactive sign-in degrades to today's empty-key behaviour rather than taking down every other provider's row. `_model_flow_named_custom` (the `hermes model` setup flow) is the sibling path — it builds its own `Authorization: Bearer` from the same incomplete resolution — and is fixed the same way, with one ordering constraint: the value persisted to config.yaml is computed BEFORE the mint, so a short-lived bearer can never be written back to shadow the key_cmd meant to re-mint it. Tests drive the real code paths and assert on the credential each probe receives rather than on function source, so a semantics-preserving refactor does not fail them. Verified they fail with the fix reverted.
165 lines
7.0 KiB
Python
165 lines
7.0 KiB
Python
"""Mint a provider API key by running a command (``key_cmd``).
|
|
|
|
Enterprise gateways (SSO/OIDC brokers, cloud IAM, auth proxies) issue SHORT-LIVED bearers; a key
|
|
copied into ``.env`` goes stale within the hour. ``key_cmd`` names a command that PRINTS a token
|
|
(the ``apiKeyHelper`` / ``gcloud auth print-access-token`` idiom). Both wire clients accept a
|
|
callable API key and invoke it per request; the token is cached until shortly before expiry.
|
|
Output contract: ONLY the token on stdout, bare or as JSON with an ``access_token`` field
|
|
(``expires_in`` / ISO ``expiry`` honoured). Precedence: explicit ``--api-key`` wins (one-off
|
|
recovery escape hatch); otherwise ``key_cmd`` beats a static ``api_key`` / ``key_env``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from typing import Callable, Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Treat a token as spent slightly before expiry so a request can't be signed with one that dies in
|
|
# flight (60s = usual OAuth cache leeway).
|
|
_TOKEN_REFRESH_LEEWAY_SECONDS = 60.0
|
|
# Helpers answer from a local cache in milliseconds; this long means hung.
|
|
_MINT_TIMEOUT_SECONDS = 15
|
|
# No advertised expiry: nothing in the request path re-mints on 401 (the SDK retries 429/5xx only), so
|
|
# a process-lifetime cache would 401 forever once the token died. Re-mint on a bounded window instead.
|
|
_NO_TTL_REFRESH_SECONDS = 900.0
|
|
|
|
|
|
class CommandTokenError(RuntimeError):
|
|
"""A ``key_cmd`` failed to produce a usable token."""
|
|
|
|
|
|
def materialize_probe_api_key(api_key: object) -> str:
|
|
"""Best-effort probe credential; never send a callable's repr or log mint errors."""
|
|
try:
|
|
token = api_key() if callable(api_key) else api_key
|
|
except Exception:
|
|
return ""
|
|
return token.strip() if isinstance(token, str) else ""
|
|
|
|
|
|
def _mint(command: str, label: str) -> tuple[str, Optional[float]]:
|
|
"""Run *command*, returning ``(token, ttl_seconds_or_None)``."""
|
|
try:
|
|
completed = subprocess.run(
|
|
command, shell=True, capture_output=True, text=True, timeout=_MINT_TIMEOUT_SECONDS,
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
raise CommandTokenError(
|
|
f"key_cmd for provider {label!r} timed out after {_MINT_TIMEOUT_SECONDS}s"
|
|
) from exc
|
|
except OSError as exc:
|
|
raise CommandTokenError(f"key_cmd for provider {label!r} could not be executed: {exc}") from exc
|
|
|
|
if completed.returncode != 0:
|
|
# NEVER include stdout/stderr (may hold a token) or the command string (may embed
|
|
# `--client-secret=…`); name the provider instead.
|
|
raise CommandTokenError(
|
|
f"key_cmd for provider {label!r} exited {completed.returncode}. "
|
|
f"Run that provider's key_cmd manually to see why "
|
|
f"(e.g. `databricks auth login` if its OAuth session expired)."
|
|
)
|
|
|
|
stdout = completed.stdout or ""
|
|
if not stdout.strip():
|
|
raise CommandTokenError(f"key_cmd for provider {label!r} produced no output")
|
|
|
|
# JSON payload — the shape `databricks auth token --output json` prints.
|
|
if stdout.lstrip().startswith("{"):
|
|
try:
|
|
payload = json.loads(stdout)
|
|
except json.JSONDecodeError:
|
|
payload = None
|
|
if isinstance(payload, dict):
|
|
token = str(payload.get("access_token") or "").strip()
|
|
if not token:
|
|
raise CommandTokenError(
|
|
f"key_cmd for provider {label!r} returned JSON without an 'access_token' field"
|
|
)
|
|
ttl = payload.get("expires_in")
|
|
if isinstance(ttl, (int, float)) and ttl > 0:
|
|
return token, float(ttl)
|
|
# CLI helpers often print an absolute ISO 8601 deadline instead of OAuth's relative
|
|
# lifetime; honour it or the token 401s once past. Lazy import: hermes_cli.auth imports agent.*.
|
|
from hermes_cli.auth import _parse_iso_timestamp
|
|
|
|
for field in ("expiry", "expiresOn"):
|
|
deadline = _parse_iso_timestamp(payload.get(field))
|
|
remaining = deadline - time.time() if deadline is not None else 0
|
|
if remaining > 0:
|
|
return token, remaining
|
|
return token, None
|
|
|
|
# Bare token: stdout carries the token and nothing else. Do NOT keep one line of several — that
|
|
# turns a misconfigured helper (banner, warning) into a corrupt-key 401 far harder to diagnose.
|
|
token = stdout.strip()
|
|
if "\n" in token:
|
|
raise CommandTokenError(
|
|
f"key_cmd for provider {label!r} printed multiple lines; it must "
|
|
"print only the token (or JSON with an 'access_token' field)"
|
|
)
|
|
return token, None
|
|
|
|
|
|
class CommandTokenSource:
|
|
"""Callable returning a bearer token, cached until shortly before expiry."""
|
|
|
|
def __init__(self, command: str, label: str = "custom") -> None:
|
|
self._command = command
|
|
self._label = label or "custom"
|
|
self._lock = threading.Lock()
|
|
self._token = ""
|
|
self._expires_at: float = 0.0
|
|
|
|
def __call__(self) -> str:
|
|
with self._lock:
|
|
if self._token and time.monotonic() < self._expires_at:
|
|
return self._token
|
|
token, ttl = _mint(self._command, self._label)
|
|
self._token = token
|
|
self._expires_at = time.monotonic() + (
|
|
max(ttl - _TOKEN_REFRESH_LEEWAY_SECONDS, 5.0) if ttl else _NO_TTL_REFRESH_SECONDS
|
|
)
|
|
logger.debug(
|
|
"Minted key_cmd token for provider %s (ttl=%s)",
|
|
self._label, f"{int(ttl)}s" if ttl else "unknown",
|
|
)
|
|
return token
|
|
|
|
|
|
def build_command_token_provider(key_cmd: str, provider_label: str = "custom") -> Optional[Callable[[], str]]:
|
|
"""A per-request token provider for *key_cmd*, or ``None`` when unset."""
|
|
command = str(key_cmd or "").strip()
|
|
return CommandTokenSource(command, provider_label) if command else None
|
|
|
|
|
|
def resolve_probe_token(entry: dict) -> str:
|
|
"""Mint a one-shot credential from a provider entry's ``key_cmd``, or "".
|
|
|
|
For callers needing a CONCRETE token rather than the per-request callable
|
|
``build_command_token_provider`` returns — the ``/models`` catalog probes,
|
|
which build their request by hand instead of going through a wire client.
|
|
Shares the ``CommandTokenSource`` cache with the request path, so this is
|
|
a cache read rather than a fresh sign-in.
|
|
|
|
Fail-closed: any error yields "". A helper that needs an interactive
|
|
sign-in (or is simply broken) must not take down a whole picker — the
|
|
caller degrades to the pre-existing empty-key behaviour and every other
|
|
provider still renders.
|
|
"""
|
|
if not isinstance(entry, dict):
|
|
return ""
|
|
command = str(entry.get("key_cmd", "") or "").strip()
|
|
if not command:
|
|
return ""
|
|
try:
|
|
provider = build_command_token_provider(command, str(entry.get("name", "") or "custom"))
|
|
return (provider() or "").strip() if provider is not None else ""
|
|
except Exception:
|
|
return ""
|