Co-authored-by: happy5318 <5318happy@users.noreply.github.com> (cherry picked from commit 83c0a8bb9c7d0e611bafe436d2e4da77144c26ec)
8256 lines
421 KiB
Python
8256 lines
421 KiB
Python
"""Shared auxiliary client router for side tasks (compression, search, vision, ...).
|
||
|
||
Text auto chain: main provider+model → OpenRouter → Nous Portal → custom endpoint →
|
||
native Anthropic → direct API-key providers → None. Vision auto chain: main
|
||
provider (if a supported vision backend) → OpenRouter → Nous → Anthropic → custom.
|
||
``auxiliary.free_only`` restricts the OpenRouter lane to ``:free`` SKUs. Codex OAuth is
|
||
in neither chain (undocumented, shifting allow-list): main provider or explicit
|
||
``auxiliary.<task>.provider`` only. HTTP 402 in call_llm() falls through the chain.
|
||
"""
|
||
|
||
import contextlib
|
||
import contextvars
|
||
import functools
|
||
import hashlib
|
||
import inspect
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from types import SimpleNamespace
|
||
from typing import Any, Callable, Dict, List, NamedTuple, Optional, Tuple, TYPE_CHECKING, Union
|
||
from urllib.parse import urlparse, parse_qs, urlunparse
|
||
|
||
from agent.error_classifier import (
|
||
_BILLING_PATTERNS,
|
||
_OVERLOADED_PATTERNS,
|
||
UNSUPPORTED_PARAM_MARKERS,
|
||
is_reasoning_field_rejection,
|
||
is_reasoning_required_rejection,
|
||
)
|
||
from agent.auxiliary_reasoning_floor import remember_reasoning_floor, with_reasoning_floor
|
||
from agent.auxiliary_structured_output import remember_structured_output_rejection
|
||
from agent.codex_headers import (
|
||
CODEX_AUX_BASE_URL as _CODEX_AUX_BASE_URL,
|
||
apply_required_codex_headers as _apply_required_codex_headers,
|
||
codex_cloudflare_headers as _codex_cloudflare_headers,
|
||
is_official_codex_base_url as _is_official_codex_base_url,
|
||
)
|
||
from agent.codex_runtime import _codex_event_has_content
|
||
from agent.sdk_transform_bypass import bypass_chat_sdk_request_transform
|
||
|
||
# `openai.OpenAI` is imported lazily (~240 ms cold); `OpenAI` below is a proxy
|
||
# so in-module calls, `auxiliary_client.OpenAI` reads and
|
||
# `patch("agent.auxiliary_client.OpenAI")` all keep working.
|
||
if TYPE_CHECKING:
|
||
from openai import OpenAI # noqa: F401 — type hints only
|
||
|
||
_OPENAI_CLS_CACHE: Optional[type] = None
|
||
|
||
|
||
def _load_openai_cls() -> type:
|
||
"""Import and cache ``openai.OpenAI``."""
|
||
global _OPENAI_CLS_CACHE
|
||
if _OPENAI_CLS_CACHE is None:
|
||
from openai import OpenAI as _cls
|
||
_OPENAI_CLS_CACHE = _cls
|
||
return _OPENAI_CLS_CACHE
|
||
|
||
|
||
class _OpenAIProxy:
|
||
"""Lazy stand-in for ``openai.OpenAI``: forwards calls and isinstance checks, importing on first use."""
|
||
__slots__ = ()
|
||
|
||
def __call__(self, *args, **kwargs):
|
||
return _load_openai_cls()(*args, **kwargs)
|
||
|
||
def __instancecheck__(self, obj):
|
||
return isinstance(obj, _load_openai_cls())
|
||
|
||
def __repr__(self):
|
||
return "<lazy openai.OpenAI proxy>"
|
||
|
||
|
||
OpenAI = _OpenAIProxy()
|
||
|
||
|
||
# Availability probe mode: check_fns only need to know whether a client is RESOLVABLE, so
|
||
# inside `aux_probe_mode()` constructors return a stub instead of importing openai + building
|
||
# httpx/SSL (~0.3s on CLI startup). Stubs are never cached (see _store_cached_client).
|
||
_aux_probe_state = threading.local()
|
||
|
||
|
||
class _AuxProbeClientStub:
|
||
"""Non-functional placeholder returned while `aux_probe_mode` is active."""
|
||
__slots__ = ("api_key", "base_url")
|
||
|
||
def __init__(self, api_key: str = "", base_url: str = "") -> None:
|
||
self.api_key = api_key
|
||
self.base_url = base_url
|
||
|
||
def __getattr__(self, name: str) -> Any:
|
||
# Loud failure if a probe stub ever leaks into a runtime call path.
|
||
raise RuntimeError(
|
||
f"_AuxProbeClientStub used as a real client (attribute {name!r}); "
|
||
"aux_probe_mode is for availability checks only")
|
||
|
||
def __repr__(self) -> str:
|
||
return "<aux availability-probe client stub>"
|
||
|
||
|
||
def _aux_probe_active() -> bool:
|
||
return bool(getattr(_aux_probe_state, "active", False))
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def aux_probe_mode():
|
||
"""Resolve provider availability without constructing real SDK clients."""
|
||
prev = getattr(_aux_probe_state, "active", False)
|
||
_aux_probe_state.active = True
|
||
try:
|
||
yield
|
||
finally:
|
||
_aux_probe_state.active = prev
|
||
|
||
|
||
from agent.credential_pool import load_pool
|
||
from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, get_model_context_length
|
||
from hermes_cli.config import get_hermes_home
|
||
from hermes_cli.config_providers import _canonical_api_mode
|
||
from agent.auxiliary_health import (
|
||
_custom_health_base_url, _unhealthy_cache_key, fallback_candidate_quarantine_ttl,
|
||
fallback_candidate_unavailable_reason,
|
||
)
|
||
from agent.auxiliary_unavailable import (
|
||
AuxiliaryClientUnavailable, clear_nous_credential_failure, missing_provider_credentials_message,
|
||
nous_credential_failure_detail, record_nous_credential_failure)
|
||
from hermes_constants import OPENROUTER_BASE_URL, hermes_home_key
|
||
from utils import base_url_host_matches, base_url_hostname, base_url_origin, env_float, is_truthy_value, model_forces_max_completion_tokens, normalize_proxy_env_vars
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# resolve_provider_client fall-through dedup: misconfigured-provider warnings fire on every
|
||
# retry, so only the first per process surfaces. Separate sets let tests clear each branch.
|
||
_LOGGED_UNKNOWN_PROVIDER_KEYS: set = set()
|
||
_LOGGED_UNHANDLED_AUTHTYPE_KEYS: set = set()
|
||
_LOGGED_UNSUPPORTED_EXTPROC_KEYS: set = set()
|
||
_LOGGED_UNSUPPORTED_OAUTH_KEYS: set = set()
|
||
|
||
|
||
def _resolve_aux_verify(base_url: Optional[str]) -> Any:
|
||
"""httpx ``verify`` for an aux base_url, mirroring the main client (per-provider ``ssl_ca_cert`` /
|
||
``ssl_verify``; otherwise the OS trust store); any failure → httpx default (``True``)."""
|
||
try:
|
||
from agent.ssl_verify import resolve_httpx_verify
|
||
from hermes_cli.config import get_custom_provider_tls_settings, load_config_readonly
|
||
tls = get_custom_provider_tls_settings(str(base_url or ""), config=load_config_readonly())
|
||
return resolve_httpx_verify(
|
||
ca_bundle=tls.get("ssl_ca_cert"), ssl_verify=tls.get("ssl_verify"), base_url=str(base_url or ""))
|
||
except Exception:
|
||
return True
|
||
|
||
|
||
_WARNED_KEEPALIVE_IMPORT_SKEW = False
|
||
|
||
|
||
def _openai_http_client_kwargs(base_url: Optional[str], *, async_mode: bool = False) -> Dict[str, Any]:
|
||
"""Inject keepalive httpx client with env-only proxy (not macOS system proxy)."""
|
||
try:
|
||
from agent.process_bootstrap import build_keepalive_http_client
|
||
client = build_keepalive_http_client(
|
||
str(base_url or ""), async_mode=async_mode, verify=_resolve_aux_verify(base_url))
|
||
except (ImportError, AttributeError):
|
||
# Version-skewed install (Desktop runtime lagging a git tree) lacks this helper:
|
||
# degrade to the SDK default httpx client rather than kill the job; warn once.
|
||
global _WARNED_KEEPALIVE_IMPORT_SKEW
|
||
if not _WARNED_KEEPALIVE_IMPORT_SKEW:
|
||
_WARNED_KEEPALIVE_IMPORT_SKEW = True
|
||
logger.warning(
|
||
"agent.process_bootstrap.build_keepalive_http_client is "
|
||
"unavailable — mixed/stale install detected (#64333). Falling "
|
||
"back to the SDK default HTTP client. Run `hermes update` (or "
|
||
"reinstall the Desktop app) to resync the runtime.")
|
||
client = None
|
||
return {"http_client": client} if client is not None else {}
|
||
|
||
|
||
def _create_openai_client(*, api_key: str, base_url: str, **kwargs: Any) -> Any:
|
||
if _aux_probe_active():
|
||
# Availability probe: resolved credentials/base_url are the answer.
|
||
return _AuxProbeClientStub(api_key=api_key, base_url=base_url)
|
||
kwargs = {**_openai_http_client_kwargs(base_url), **kwargs}
|
||
_apply_required_codex_headers(kwargs, access_token=api_key, base_url=base_url)
|
||
# Hermes owns aux retry/fallback policy; the SDK default (max_retries=2) would triple
|
||
# wall time on a hung endpoint before Hermes sees one failure.
|
||
# Hermes owns auxiliary retry + provider/model fallback policy (the same-provider transient retry in
|
||
# call_llm plus the except-chain fallback). The OpenAI SDK's own default (max_retries=2 → up to 3
|
||
# attempts) silently multiplies the effective wall time of every aux call by 3× on a slow/hung endpoint,
|
||
# so a 120s timeout can stall ~360s before Hermes sees a single failure (issue #54465). Disable
|
||
# SDK-internal retries by default and let Hermes control the budget; explicit callers can still override
|
||
# via kwargs.
|
||
kwargs.setdefault("max_retries", 0)
|
||
return OpenAI(api_key=api_key, base_url=base_url, **kwargs)
|
||
|
||
|
||
# Interrupt protection for atomic aux tasks: a compression summary killed by an ordinary
|
||
# gateway interrupt degrades to a static marker, so a thread-local flag marks such calls
|
||
# protected. Explicit host cancel (Ctrl+C, /stop) still overrides it, timeouts still fire.
|
||
# ── Interrupt protection for atomic auxiliary tasks ────────────────────── Some auxiliary tasks must NOT be
|
||
# aborted mid-flight by a gateway interrupt (e.g. an incoming user message while the agent is busy). Context
|
||
# compression is the prime case: if the summary LLM call is interrupted part-way, compression falls back to
|
||
# a static "summary unavailable" marker and the real handoff is lost (#23975). A thread-local flag lets such
|
||
# a task mark its in-flight LLM call as interrupt-protected; the Codex Responses stream's cancellation check
|
||
# honors it. TIMEOUTS still fire (a hung call must die), and all OTHER aux tasks (vision, web_extract,
|
||
# title_generation, …) remain freely interruptible.
|
||
_aux_interrupt_protection = threading.local()
|
||
|
||
|
||
class AuxiliaryExplicitCancellation(BaseException):
|
||
"""Frozen signal that an auxiliary attempt was explicitly hard-cancelled. ``BaseException`` so broad
|
||
``except Exception`` retry/fallback code never treats a host stop as a transport failure; ``cause``
|
||
is immutable class data so nothing re-queries a mutable host Event after the transport unwound."""
|
||
cause = "explicit_host_cancel"
|
||
|
||
def __init__(self) -> None:
|
||
super().__init__("auxiliary request explicitly cancelled by host")
|
||
|
||
|
||
def _aux_interrupt_protected() -> bool:
|
||
return bool(getattr(_aux_interrupt_protection, "active", False))
|
||
|
||
|
||
def _aux_interrupt_cancel_requested() -> bool:
|
||
"""Return whether an explicit host cancel overrides aux protection."""
|
||
check = _capture_aux_cancel_check()
|
||
return _captured_aux_cancel_requested(check) if check is not None else False
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def aux_interrupt_protection(active: bool = True, cancel_check=None, cancel_event=None):
|
||
"""Mark this thread's aux LLM call interrupt-protected (re-entrant-safe). ``cancel_check`` /
|
||
``cancel_event`` keep an explicit host hard-cancel path (Event preferred); nested scopes inherit both."""
|
||
prev = getattr(_aux_interrupt_protection, "active", False)
|
||
prev_cancel_check = getattr(_aux_interrupt_protection, "cancel_check", None)
|
||
prev_cancel_event = getattr(_aux_interrupt_protection, "cancel_event", None)
|
||
_aux_interrupt_protection.active = active
|
||
if callable(cancel_check):
|
||
_aux_interrupt_protection.cancel_check = cancel_check
|
||
if cancel_event is not None and callable(getattr(cancel_event, "is_set", None)):
|
||
_aux_interrupt_protection.cancel_event = cancel_event
|
||
try:
|
||
yield
|
||
finally:
|
||
_aux_interrupt_protection.active = prev
|
||
_aux_interrupt_protection.cancel_check = prev_cancel_check
|
||
_aux_interrupt_protection.cancel_event = prev_cancel_event
|
||
|
||
|
||
def _capture_aux_cancel_check() -> Optional[Callable[[], Any]]:
|
||
"""Capture the current explicit-cancel source on the owning request thread."""
|
||
is_set = getattr(getattr(_aux_interrupt_protection, "cancel_event", None), "is_set", None)
|
||
if callable(is_set):
|
||
return is_set
|
||
# Return the callable itself so attempt-local decision objects keep begin_timeout_cleanup().
|
||
check = getattr(_aux_interrupt_protection, "cancel_check", None)
|
||
return check if callable(check) else None
|
||
|
||
|
||
def _captured_aux_cancel_requested(cancel_check: Callable[[], Any]) -> bool:
|
||
"""Read a request-thread cancellation source without leaking its failures."""
|
||
try:
|
||
return bool(cancel_check())
|
||
except Exception:
|
||
logger.debug("captured aux cancel check failed", exc_info=True)
|
||
return False
|
||
|
||
|
||
class _AuxiliaryCancellationDecision:
|
||
"""Atomically choose explicit cancellation or provider timeout per attempt."""
|
||
|
||
def __init__(self, source_cancel_check: Callable[[], Any]) -> None:
|
||
self._source_cancel_check = source_cancel_check
|
||
self._lock = threading.Lock()
|
||
self._outcome = "active"
|
||
|
||
def __call__(self) -> bool:
|
||
with self._lock:
|
||
if self._outcome == "active" and _captured_aux_cancel_requested(self._source_cancel_check):
|
||
self._outcome = "cancelled"
|
||
return self._outcome == "cancelled"
|
||
|
||
def begin_timeout_cleanup(self) -> bool:
|
||
"""Return whether timeout won and destructive cleanup is permitted."""
|
||
with self._lock:
|
||
if self._outcome == "active":
|
||
cancelled = _captured_aux_cancel_requested(self._source_cancel_check)
|
||
self._outcome = "cancelled" if cancelled else "timed_out"
|
||
return self._outcome == "timed_out"
|
||
|
||
|
||
# Forward-progress hooks for streamed aux calls: a fixed host deadline kills a SLOW model
|
||
# streaming a big summary as hard as a HUNG one, so wire consumers tick the progress hook only
|
||
# for non-empty payloads and the host extends its deadline while tokens move. Thread-local:
|
||
# the call and its stream consumption run on the installing thread.
|
||
_aux_progress = threading.local()
|
||
_aux_dispatch = threading.local()
|
||
_aux_provider_response = threading.local()
|
||
# Absolute monotonic deadline of the waiting HOST. The stream's own ceiling
|
||
# (_aux_stream_total_ceiling, >= the host's and started later) would otherwise leave an
|
||
# orphaned stream still billing after every host-ceiling timeout.
|
||
# Absolute wall-clock deadline (time.monotonic) of the HOST waiting for this auxiliary call, when it has one
|
||
# (#99692). Liveness alone is not enough: a host also stops waiting at its own total ceiling, and the
|
||
# streamed consumer below bounds itself only by _aux_stream_total_ceiling() — a budget derived from the aux
|
||
# request timeout, which is >= the host ceiling for every configured value AND starts counting later. So the
|
||
# stream that outlives its abandoned host is not an edge case; it is the guaranteed outcome of every
|
||
# total-ceiling timeout.
|
||
_aux_stream_deadline = threading.local()
|
||
|
||
|
||
def _tick_hook(local: threading.local, label: str) -> None:
|
||
"""Call the thread-local hook installed on ``local``, if any. Never raises."""
|
||
hook = getattr(local, "hook", None)
|
||
if hook is None:
|
||
return
|
||
try:
|
||
hook()
|
||
except Exception:
|
||
logger.debug("aux %s hook failed", label, exc_info=True)
|
||
|
||
|
||
def _notify_aux_progress() -> None:
|
||
"""Tick the installed forward-progress hook, if any."""
|
||
_tick_hook(_aux_progress, "progress")
|
||
|
||
|
||
def _notify_aux_dispatch() -> None:
|
||
"""Record an actual provider dispatch without claiming response progress."""
|
||
_tick_hook(_aux_dispatch, "dispatch")
|
||
|
||
|
||
def _notify_aux_timing_response() -> None:
|
||
"""Record a content-free frame (keepalive/empty delta): counts toward
|
||
``time_to_first_progress_ms`` but must not reset a compression inactivity fence."""
|
||
_tick_hook(_aux_provider_response, "provider response")
|
||
|
||
|
||
def _notify_aux_provider_response() -> None:
|
||
"""Record a provider response/chunk, then preserve the liveness signal."""
|
||
_notify_aux_timing_response()
|
||
_notify_aux_progress()
|
||
|
||
|
||
def _aux_progress_active() -> bool:
|
||
return getattr(_aux_progress, "hook", None) is not None
|
||
|
||
|
||
def _field(obj: Any, key: str, default: Any = None) -> Any:
|
||
"""Field access for wire objects that may be dicts or SDK/SimpleNamespace objects."""
|
||
val = obj.get(key) if isinstance(obj, dict) else getattr(obj, key, None)
|
||
return default if val is None else val
|
||
|
||
|
||
def _anthropic_event_has_content(event: Any) -> bool:
|
||
"""Whether an Anthropic stream event carries a non-empty payload."""
|
||
event_type = _field(event, "type")
|
||
if event_type == "content_block_delta":
|
||
delta = _field(event, "delta")
|
||
return any(bool(_field(delta, f)) for f in ("text", "thinking", "partial_json", "signature", "citation"))
|
||
if event_type == "content_block_start":
|
||
block = _field(event, "content_block")
|
||
return _field(block, "type") == "tool_use" and any(bool(_field(block, f)) for f in ("id", "name"))
|
||
return False
|
||
|
||
|
||
def _anthropic_aux_stream_event_hook() -> Callable[[Any], None]:
|
||
"""Per-event callback for the Anthropic aux wire: progress only for substantive payloads
|
||
(keepalives must not keep a stalled summary alive), stop at the host deadline or explicit
|
||
cancel. The ``TimeoutError`` text must say "timed out" so ``_is_timeout_error`` classifies it."""
|
||
host_deadline = _current_aux_stream_deadline()
|
||
started = time.monotonic()
|
||
|
||
def _on_event(event: Any) -> None:
|
||
if _anthropic_event_has_content(event):
|
||
_notify_aux_provider_response()
|
||
else:
|
||
_notify_aux_timing_response()
|
||
if _aux_interrupt_cancel_requested():
|
||
raise AuxiliaryExplicitCancellation()
|
||
if host_deadline is not None and time.monotonic() >= host_deadline:
|
||
raise TimeoutError(
|
||
"Anthropic auxiliary stream timed out at the host compression "
|
||
f"deadline after {time.monotonic() - started:.0f}s (the caller already stopped waiting)")
|
||
|
||
return _on_event
|
||
|
||
|
||
# A dead stream fails at the no-progress window (first token AND between tokens); a live
|
||
# stream re-arms per event, bounded by _aux_stream_total_ceiling().
|
||
_AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS = 60.0
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _aux_thread_local_hook(local: threading.local, hook):
|
||
"""Install one thread-local hook, restoring the prior on exit (non-callable = passthrough)."""
|
||
previous = getattr(local, "hook", None)
|
||
local.hook = hook if callable(hook) else previous
|
||
try:
|
||
yield
|
||
finally:
|
||
local.hook = previous
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def aux_progress_hook(hook):
|
||
"""Install *hook* as the current thread's aux forward-progress callback (None = passthrough)."""
|
||
with _aux_thread_local_hook(_aux_progress, hook):
|
||
yield
|
||
|
||
|
||
def _current_aux_stream_deadline() -> Optional[float]:
|
||
"""The waiting host's absolute monotonic deadline, if one is installed."""
|
||
return getattr(_aux_stream_deadline, "value", None)
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def aux_stream_deadline(deadline: Optional[float]):
|
||
"""Publish the host's absolute ``time.monotonic()`` deadline to the stream consumer.
|
||
|
||
``None`` is a passthrough; re-entrant-safe. Host->worker return leg of the progress hook:
|
||
without it the isolated provider daemon streams to its own ceiling after the host stopped
|
||
waiting, billing a summary the commit fence refuses.
|
||
|
||
``8207862212`` releases the compression OWNER when the fence is cancelled, but the isolated provider
|
||
daemon (:func:`_run_protected_sync_provider_call`) that holds the socket keeps streaming to its own
|
||
``_aux_stream_total_ceiling`` budget — >= the host's ceiling by construction — billing an abandoned
|
||
summary the commit fence is already guaranteed to refuse, and stacking one fresh orphan per turn on a
|
||
session that compression never managed to shrink. See #99692.
|
||
"""
|
||
previous = getattr(_aux_stream_deadline, "value", None)
|
||
_aux_stream_deadline.value = deadline if isinstance(deadline, (int, float)) else previous
|
||
try:
|
||
yield
|
||
finally:
|
||
_aux_stream_deadline.value = previous
|
||
|
||
|
||
def _run_protected_sync_provider_call(callback: Callable[[dict[str, Any]], Any], kwargs: dict[str, Any]) -> Any:
|
||
"""Run one protected provider callback in an attempt-isolated daemon thread.
|
||
|
||
Aux clients are process-shared and cannot be closed to wake one request, so the callback (incl.
|
||
stream aggregation) runs in a daemon while the owner polls cancellation; on cancel the owner
|
||
unwinds at once and the daemon finishes under the provider timeout in ``kwargs`` (it owns no
|
||
transcript/commit state, never holds the session lock). Unprotected / no cancel source: direct.
|
||
"""
|
||
source_cancel_check = _capture_aux_cancel_check()
|
||
if not _aux_interrupt_protected() or not callable(source_cancel_check):
|
||
return callback(kwargs)
|
||
# One linearized outcome per attempt: the host Event is reused/cleared on later turns and
|
||
# the Codex timeout Timer may race owner polling — same lock for both.
|
||
cancel_check = _AuxiliaryCancellationDecision(source_cancel_check)
|
||
if cancel_check():
|
||
raise AuxiliaryExplicitCancellation()
|
||
# Thread-locals do not cross into the daemon: timing hooks fire from the thread running
|
||
# the callback, and the host deadline is inert unless carried along.
|
||
progress_hook = getattr(_aux_progress, "hook", None)
|
||
dispatch_hook = getattr(_aux_dispatch, "hook", None)
|
||
provider_response_hook = getattr(_aux_provider_response, "hook", None)
|
||
host_deadline = _current_aux_stream_deadline()
|
||
# #99692: the stream is consumed on the daemon below, and thread-locals do not cross that boundary — an
|
||
# owner-thread-only deadline would leave the fix inert on exactly the path large-session compression
|
||
# takes (protected call + hard-cancel source installed).
|
||
provider_context = contextvars.copy_context()
|
||
done = threading.Event()
|
||
outcome: dict[str, Any] = {}
|
||
|
||
def _provider_worker() -> None:
|
||
try:
|
||
with (
|
||
aux_progress_hook(progress_hook),
|
||
_aux_thread_local_hook(_aux_dispatch, dispatch_hook),
|
||
_aux_thread_local_hook(_aux_provider_response, provider_response_hook),
|
||
aux_stream_deadline(host_deadline),
|
||
aux_interrupt_protection(cancel_check=cancel_check),
|
||
):
|
||
outcome["result"] = callback(kwargs)
|
||
except BaseException as exc:
|
||
outcome["exception"] = exc
|
||
finally:
|
||
done.set()
|
||
|
||
threading.Thread(
|
||
target=provider_context.run, args=(_provider_worker,), name="hermes-protected-aux-provider",
|
||
daemon=True).start()
|
||
while True:
|
||
# Check cancel before AND after each wait so it wins when result publication and the
|
||
# host Event land in the same polling interval.
|
||
if _captured_aux_cancel_requested(cancel_check):
|
||
raise AuxiliaryExplicitCancellation()
|
||
if not done.wait(0.02):
|
||
continue
|
||
if _captured_aux_cancel_requested(cancel_check):
|
||
raise AuxiliaryExplicitCancellation()
|
||
exception = outcome.get("exception")
|
||
if exception is not None:
|
||
raise exception
|
||
return outcome.get("result")
|
||
|
||
|
||
def _client_declares(client_obj: Any, flag: str) -> bool:
|
||
"""Whether ``client_obj`` (or its class) sets ``flag`` truthy; absent → False. Capability declaration,
|
||
not isinstance, so out-of-tree clients can opt out of wrappers unimported (cf. SUPPORTS_HERMES_TOOL_CALLS)."""
|
||
try:
|
||
return bool(getattr(client_obj, flag, False))
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _safe_isinstance(obj: Any, maybe_type: Any) -> bool:
|
||
"""Return False instead of raising when a patched symbol is not a type."""
|
||
try:
|
||
return isinstance(obj, maybe_type)
|
||
except TypeError:
|
||
return False
|
||
|
||
|
||
def _extract_url_query_params(url: str):
|
||
"""Extract query params from URL, return (clean_url, default_query dict or None)."""
|
||
parsed = urlparse(url)
|
||
if parsed.query:
|
||
return urlunparse(parsed._replace(query="")), {k: v[0] for k, v in parse_qs(parsed.query).items()}
|
||
return url, None
|
||
|
||
|
||
# Warn only once per process about stale OPENAI_BASE_URL.
|
||
_stale_base_url_warned = False
|
||
|
||
# Local OpenAI-compatible servers (Ollama, vLLM, llama.cpp) route through the generic custom
|
||
# provider — mirrors hermes_cli.auth._PROVIDER_ALIASES. Without this group an explicit
|
||
# ``provider: ollama`` aux lane matches no registry entry and raises a misleading
|
||
# ``OLLAMA_API_KEY`` error instead of using the lane's base_url (#106010).
|
||
_LOCAL_SERVER_ALIASES = {
|
||
"ollama": "custom", "vllm": "custom", "llamacpp": "custom",
|
||
"llama.cpp": "custom", "llama-cpp": "custom",
|
||
}
|
||
|
||
_ALIAS_TABLE: Optional[Dict[str, str]] = None
|
||
|
||
|
||
def _provider_alias_table() -> Dict[str, str]:
|
||
"""The same alias table the main provider path resolves against (hermes_cli.auth).
|
||
|
||
A hand-copied mirror here rots silently every time auth grows a family — the local
|
||
servers (#106010) and the OpenCode entries were both missed that way. Local-server
|
||
names stay pinned on top: that set doubles as the guard for the /v1 tail and the
|
||
no-key-borrow rule in the custom branch below.
|
||
"""
|
||
global _ALIAS_TABLE
|
||
if _ALIAS_TABLE is None:
|
||
merged: Dict[str, str] = dict(_LOCAL_SERVER_ALIASES)
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli.auth import _PROVIDER_ALIASES as _auth_table
|
||
merged.update(_auth_table)
|
||
_ALIAS_TABLE = merged
|
||
return _ALIAS_TABLE
|
||
|
||
def _normalize_aux_provider(provider: Optional[str]) -> str:
|
||
normalized = (provider or "auto").strip().lower()
|
||
if normalized.startswith("custom:"):
|
||
suffix = normalized.split(":", 1)[1].strip()
|
||
if not suffix:
|
||
return "custom"
|
||
normalized = suffix
|
||
if normalized == "codex":
|
||
return "openai-codex"
|
||
if normalized == "main":
|
||
# Resolve to the actual main provider so named custom providers work.
|
||
main_prov = (_read_main_provider() or "").strip().lower()
|
||
if not main_prov or main_prov in {"auto", "main"}:
|
||
return "custom"
|
||
normalized = main_prov
|
||
return _provider_alias_table().get(normalized, normalized)
|
||
|
||
|
||
# Sentinel from _fixed_temperature_for_model(): callers strip ``temperature`` entirely.
|
||
# Kimi/Moonshot manage it server-side — any value can conflict with gateway mode selection.
|
||
OMIT_TEMPERATURE: object = object()
|
||
|
||
|
||
def _bare_model(model: Optional[str]) -> str:
|
||
"""Lowercased model slug with any ``vendor/`` prefix stripped."""
|
||
return (model or "").strip().lower().rsplit("/", 1)[-1]
|
||
|
||
|
||
def _is_kimi_model(model: Optional[str]) -> bool:
|
||
"""True for any Kimi / Moonshot model that manages temperature server-side."""
|
||
bare = _bare_model(model)
|
||
return bare.startswith("kimi-") or bare == "kimi"
|
||
|
||
|
||
def _is_arcee_trinity_thinking(model: Optional[str]) -> bool:
|
||
"""True for Arcee Trinity Large Thinking (direct or via OpenRouter)."""
|
||
return _bare_model(model) == "trinity-large-thinking"
|
||
|
||
|
||
# Codex OAuth hard-caps gpt-5.4/5.5/5.6 and gpt-6 Astra at 272K (raw API/OpenRouter expose 1.05M);
|
||
# the default 50% trigger would compact at ~136K, so raise to 85% (~231K).
|
||
_CODEX_GPT54_GPT55_COMPACTION_THRESHOLD = 0.85
|
||
# gpt-5.3-codex-spark: Codex-OAuth-only, native 128K; 70% (~90K) leaves summary headroom.
|
||
_CODEX_SPARK_COMPACTION_THRESHOLD = 0.70
|
||
|
||
|
||
def _is_codex_gpt54_or_gpt55(model: Optional[str], provider: Optional[str] = None) -> bool:
|
||
"""True for gpt-5.4/5.5/5.6, gpt-6 Sol/Terra/Luna, gpt-6 Astra (and the Daybreak Sol alias) on the Codex OAuth route only.
|
||
|
||
Other routes expose a larger window for the same slug and keep the user's threshold.
|
||
Prefix-matched so ``-pro`` and dated snapshots track every 272K-capped family; ``-900k``
|
||
picker variants are excluded. Astra is substring-matched (any slug containing ``astra``
|
||
without ``900k``). Name kept for the ``compression.codex_gpt55_autoraise`` key.
|
||
"""
|
||
bare = _codex_route_bare_model(model, provider)
|
||
if bare is None:
|
||
return False
|
||
from agent.model_metadata import is_codex_context_variant
|
||
if is_codex_context_variant(bare):
|
||
return False
|
||
if "astra" in bare:
|
||
return "900k" not in bare
|
||
return bare == "gpt-daybreak-blue-latest" or any(
|
||
bare == fam or bare.startswith(fam + "-") or bare.startswith(fam + ".")
|
||
for fam in ("gpt-5.4", "gpt-5.5", "gpt-5.6", "gpt-6-sol", "gpt-6-luna"))
|
||
|
||
|
||
def _codex_route_bare_model(model: Optional[str], provider: Optional[str]) -> Optional[str]:
|
||
"""Lowercased bare model slug when ``provider`` is the Codex OAuth route, else None."""
|
||
return _bare_model(model) if (provider or "").strip().lower() == "openai-codex" else None
|
||
|
||
|
||
def _is_codex_spark(model: Optional[str], provider: Optional[str] = None) -> bool:
|
||
"""True for ``gpt-5.3-codex-spark`` on the Codex OAuth route (the slug exists nowhere else)."""
|
||
return _codex_route_bare_model(model, provider) == "gpt-5.3-codex-spark"
|
||
|
||
|
||
def _is_openai_default_temperature_only(model: Optional[str]) -> bool:
|
||
"""True for OpenAI reasoning families that 400 (``unsupported_value``) on any non-default
|
||
``temperature``: gpt-5.x (incl. dated snapshots, ``-pro``, ``-codex``), o1/o3/o4. The
|
||
``gpt-5-chat`` non-reasoning line still accepts it (#51083)."""
|
||
bare = _bare_model(model)
|
||
return bare.startswith(("gpt-5", "o1", "o3", "o4")) and not bare.startswith("gpt-5-chat")
|
||
|
||
|
||
# Routes (host + model) that rejected ``temperature`` at runtime; the next call omits it up front
|
||
# instead of paying the 400 round-trip again (the retry alone left #51083's first call to time out).
|
||
_TEMPERATURE_REJECTED_ROUTES: set = set()
|
||
|
||
|
||
def remember_temperature_rejection(
|
||
provider: Optional[str], base_url: Optional[str], rejected_kwargs: Dict[str, Any], error: BaseException,
|
||
) -> None:
|
||
from agent.auxiliary_structured_output import _route_key
|
||
_TEMPERATURE_REJECTED_ROUTES.add((_route_key(provider, base_url), _bare_model(rejected_kwargs.get("model"))))
|
||
|
||
|
||
def _fixed_temperature_for_model(
|
||
model: Optional[str], base_url: Optional[str] = None, provider: Optional[str] = None,
|
||
) -> "Optional[float] | object":
|
||
"""``OMIT_TEMPERATURE`` (drop the key; Kimi/Moonshot, OpenAI reasoning families, routes that
|
||
already rejected it), a fixed ``float``, or ``None``."""
|
||
if _is_kimi_model(model):
|
||
logger.debug("Omitting temperature for Kimi model %r (server-managed)", model)
|
||
return OMIT_TEMPERATURE
|
||
if _is_openai_default_temperature_only(model):
|
||
logger.debug("Omitting temperature for %r (accepts only the default)", model)
|
||
return OMIT_TEMPERATURE
|
||
from agent.auxiliary_structured_output import _route_key
|
||
if (_route_key(provider, base_url), _bare_model(model)) in _TEMPERATURE_REJECTED_ROUTES:
|
||
logger.debug("Omitting temperature for %r (route rejected it earlier)", model)
|
||
return OMIT_TEMPERATURE
|
||
return 0.5 if _is_arcee_trinity_thinking(model) else None
|
||
|
||
|
||
def _compression_threshold_for_model(
|
||
model: Optional[str], provider: Optional[str] = None, *,
|
||
allow_codex_gpt55_autoraise: bool = True,
|
||
) -> Optional[float]:
|
||
"""Per-model/route compression threshold override (fraction of context used), or None.
|
||
|
||
Arcee Trinity Large Thinking → 0.75 (preserve reasoning context); Codex-route gpt-5.4/5.5/5.6/Astra
|
||
→ 0.85, gated by ``allow_codex_gpt55_autoraise``; Codex-route gpt-5.3-codex-spark → 0.70, ungated.
|
||
"""
|
||
if _is_arcee_trinity_thinking(model):
|
||
return 0.75
|
||
if allow_codex_gpt55_autoraise and _is_codex_gpt54_or_gpt55(model, provider):
|
||
return _CODEX_GPT54_GPT55_COMPACTION_THRESHOLD
|
||
if _is_codex_spark(model, provider):
|
||
return _CODEX_SPARK_COMPACTION_THRESHOLD
|
||
return None
|
||
|
||
|
||
# Aux "fast tier" families, fastest first (measured p50 titling latency). Matched as substrings
|
||
# against the LIVE /v1/models catalog because pinned ids rot; rolling "-latest" aliases lead.
|
||
_FAST_MODEL_FAMILIES: tuple = (
|
||
"gpt-mini-latest", "gpt-nano-latest", "claude-haiku-latest", "gemini-flash-latest",
|
||
"gpt-5.4-nano", "gpt-5.4-mini", "gpt-5-mini", "haiku-4.5", "gemini-3.6-flash", "flash-lite",
|
||
"-nano", "-mini", "-flash", "haiku",
|
||
)
|
||
|
||
# Disqualifiers: reasoning variants think before answering; ":batch" is a queue; ":free" tiers
|
||
# are rate-limited and slowest; embedders/modality endpoints match a rung but cannot answer.
|
||
_FAST_MODEL_EXCLUDE: tuple = (
|
||
"thinking", "reason", "-r1", "minilm", ":batch", ":free",
|
||
"o1-", "o3-", "o4-", "codex", "audio", "-vl", "embed",
|
||
"-tts", "-transcribe", "-realtime", "-image", "-search-preview",
|
||
)
|
||
|
||
|
||
def _model_recency_key(model_id: str) -> tuple:
|
||
"""Sort key putting a family's newest release first: digit runs compare numerically (plain
|
||
string order picks ``gpt-3.5-mini`` over ``gpt-5.4-mini`` and breaks at 9 vs 10)."""
|
||
# re.split with one capturing group alternates text, number, text, …
|
||
return tuple(
|
||
(1, float(part), "") if index % 2 else (0, 0.0, part)
|
||
for index, part in enumerate(re.split(r"(\d+(?:\.\d+)?)", model_id.lower())) if part)
|
||
|
||
|
||
def _fast_model_from_catalog(provider_id: str) -> str:
|
||
"""Newest ``_FAST_MODEL_FAMILIES`` match from the provider's live (cached) catalog.
|
||
|
||
"" when the catalog is unavailable or holds no small model (caller falls through to the
|
||
curated default). Never raises; the fetch is memory+disk cached.
|
||
"""
|
||
is_nous = provider_id.strip().lower() == "nous"
|
||
try:
|
||
from hermes_cli.auth import resolve_api_key_provider_credentials
|
||
from hermes_cli.models_pricing import fetch_models_with_pricing
|
||
from providers import get_provider_profile
|
||
# Most /v1/models endpoints are authenticated; an anonymous 401 would read as "no small
|
||
# model" and pin the curated default forever.
|
||
api_key, base_url = "", ""
|
||
try:
|
||
creds = resolve_api_key_provider_credentials(provider_id) or {}
|
||
api_key = str(creds.get("api_key", "")).strip()
|
||
base_url = str(creds.get("base_url", "")).strip()
|
||
except Exception:
|
||
# Not an API-key provider, or nothing configured; anonymous fetch may still work.
|
||
logger.debug("No credentials for %s catalog", provider_id, exc_info=True)
|
||
if not api_key and is_nous:
|
||
# Nous is OAuth (resolver raises); anonymous reads return the full catalog.
|
||
try:
|
||
from hermes_cli.models_pricing import _resolve_nous_pricing_credentials
|
||
api_key, base_url = _resolve_nous_pricing_credentials()
|
||
except Exception:
|
||
logger.debug("No Nous credentials for catalog", exc_info=True)
|
||
if not base_url:
|
||
base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "")
|
||
base_url = base_url.rstrip("/")
|
||
if not base_url:
|
||
return ""
|
||
if base_url.endswith("/v1"): # fetch_models_with_pricing appends /v1/models
|
||
base_url = base_url[:-3]
|
||
# Nous-only args must match the pickers' or the seeded cache loses sale chrome and
|
||
# policy-catalog expiry.
|
||
_nous_kwargs = {}
|
||
if is_nous:
|
||
from hermes_cli.models_pricing import _NOUS_CATALOG_TTL_SECONDS
|
||
_nous_kwargs = {"include_sale_original": True, "cache_ttl_seconds": _NOUS_CATALOG_TTL_SECONDS}
|
||
catalog = fetch_models_with_pricing(
|
||
api_key=api_key or None, base_url=base_url, timeout=3.0, **_nous_kwargs) or {}
|
||
except Exception:
|
||
logger.debug("Fast-model catalog lookup failed for %s", provider_id, exc_info=True)
|
||
return ""
|
||
ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True)
|
||
if is_nous:
|
||
# Narrow catalog ids by org policy, as the pickers do.
|
||
try:
|
||
from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
|
||
ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids())
|
||
except Exception:
|
||
logger.debug("Nous policy filter unavailable", exc_info=True)
|
||
for family in _FAST_MODEL_FAMILIES:
|
||
for model_id in ids:
|
||
lowered = model_id.lower()
|
||
if family in lowered and not any(x in lowered for x in _FAST_MODEL_EXCLUDE):
|
||
return model_id
|
||
return ""
|
||
|
||
|
||
# Default auxiliary models for direct API-key providers (cheap/fast for side tasks)
|
||
def _get_aux_model_for_provider(provider_id: str, *, prefer_fast: bool = False) -> str:
|
||
"""Cheap auxiliary model for a provider.
|
||
|
||
Ladder: (``prefer_fast`` only) live-catalog family match, then ``ProviderProfile.resolve_aux_model``;
|
||
then ``default_aux_model`` (curated); then the legacy dict. ``prefer_fast`` is opt-in (titling)
|
||
so other callers keep their static behaviour and cache keys.
|
||
"""
|
||
profile = None
|
||
with contextlib.suppress(Exception):
|
||
from providers import get_provider_profile
|
||
profile = get_provider_profile(provider_id)
|
||
picked = ""
|
||
if prefer_fast:
|
||
picked = _fast_model_from_catalog(provider_id)
|
||
if not picked and profile is not None:
|
||
try:
|
||
picked = profile.resolve_aux_model() or ""
|
||
except Exception:
|
||
logger.debug("resolve_aux_model failed for %s", provider_id, exc_info=True)
|
||
if not picked and profile is not None and profile.default_aux_model:
|
||
picked = profile.default_aux_model
|
||
if not picked:
|
||
picked = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK.get(provider_id, "")
|
||
# Rungs 2-4 are policy-blind; a blocked pick is refused at request time, so drop it and
|
||
# let the caller keep the main model.
|
||
if picked and provider_id.strip().lower() == "nous":
|
||
try:
|
||
from hermes_cli.models_pricing import nous_policy_allowed_ids, restrict_to_nous_policy
|
||
allowed = nous_policy_allowed_ids()
|
||
if allowed and not restrict_to_nous_policy([picked], allowed):
|
||
return ""
|
||
except Exception:
|
||
logger.debug("Nous policy check unavailable", exc_info=True)
|
||
return picked
|
||
|
||
|
||
# Fallback for providers without ProviderProfile.default_aux_model (plus some pinned here).
|
||
# New providers should set default_aux_model instead.
|
||
_API_KEY_PROVIDER_AUX_MODELS_FALLBACK: Dict[str, str] = {
|
||
"gemini": "gemini-3.6-flash", "zai": "glm-4.5-flash", "kimi-coding": "kimi-k2-turbo-preview",
|
||
"stepfun": "step-3.5-flash", "kimi-coding-cn": "kimi-k2-turbo-preview",
|
||
"gmi": "google/gemini-3.1-flash-lite-preview", "anthropic": "claude-haiku-4-5-20251001",
|
||
"ai-gateway": "google/gemini-3-flash", "opencode-zen": "gemini-3-flash", "opencode-go": "glm-5",
|
||
"kilocode": "google/gemini-3.6-flash", "ollama-cloud": "nemotron-3-nano:30b",
|
||
"tencent-tokenhub": "hy4-preview", "tencent-tokenplan": "hy4-preview",
|
||
# No "deepinfra": its aux model lives on the ProviderProfile (read first).
|
||
}
|
||
|
||
# Legacy alias for callers not yet using _get_aux_model_for_provider().
|
||
_API_KEY_PROVIDER_AUX_MODELS: Dict[str, str] = _API_KEY_PROVIDER_AUX_MODELS_FALLBACK
|
||
|
||
# Tasks that may opt into ``auxiliary.<task>.prefer_fast_model``.
|
||
_FAST_MODEL_TASKS: frozenset = frozenset({"title_generation"})
|
||
|
||
|
||
def _task_prefers_fast_model(task: Optional[str]) -> bool:
|
||
"""Return whether an eligible task explicitly opts into fast-model routing."""
|
||
return task in _FAST_MODEL_TASKS and is_truthy_value(
|
||
_get_auxiliary_task_config(task).get("prefer_fast_model"), default=False)
|
||
|
||
|
||
# Dedicated vision models for direct providers whose main chat model differs. zai: glm-5.3-flash
|
||
# is the only image-capable GLM id served on every Z.AI surface (pay-as-you-go and Coding Plan,
|
||
# global and CN); the former glm-5v-turbo pin 404s / 1211 "Unknown Model" on the coding endpoints
|
||
# (#111429). ZaiProfile has no default_vision_model(), so dropping the pin would route vision to
|
||
# the user's text-only chat model and skip Z.AI entirely.
|
||
_PROVIDER_VISION_MODELS: Dict[str, str] = {"xiaomi": "mimo-v2.5", "zai": "glm-5.3-flash"}
|
||
|
||
|
||
def _resolve_provider_vision_default(provider: str) -> Optional[str]:
|
||
"""Provider default vision model id, or None: static ``_PROVIDER_VISION_MODELS`` (vision-only
|
||
names absent from any catalog) win, else ``ProviderProfile.default_vision_model()``."""
|
||
static = _PROVIDER_VISION_MODELS.get(provider)
|
||
if static:
|
||
return static
|
||
try:
|
||
from providers import get_provider_profile
|
||
profile = get_provider_profile(provider)
|
||
return profile.default_vision_model() if profile is not None else None
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
# Endpoints that reject image input: vision auto-detect skips these to the aggregator chain
|
||
# instead of returning a client that 404s (Kimi Coding Plan Anthropic wire has no image_in).
|
||
_PROVIDERS_WITHOUT_VISION: frozenset = frozenset({"kimi-coding", "kimi-coding-cn"})
|
||
|
||
# OpenRouter app attribution (always sent). `X-Title` is what the dashboard reads.
|
||
_OR_HEADERS_BASE = {
|
||
"HTTP-Referer": "https://hermes-agent.nousresearch.com",
|
||
"X-Title": "Hermes Agent",
|
||
"X-OpenRouter-Categories": "productivity,cli-agent",
|
||
}
|
||
|
||
|
||
def _apply_user_default_headers(headers: dict | None) -> dict | None:
|
||
"""Merge user ``model.default_headers`` onto resolved headers (user wins; ``model.extra_headers``
|
||
alias wins over both). Mirrors ``AIAgent._apply_user_default_headers`` so a custom endpoint behind a
|
||
WAF rejecting ``User-Agent`` / ``X-Stainless-*`` works for aux calls. SECURITY: never log values."""
|
||
try:
|
||
from hermes_cli.config import cfg_get, load_config
|
||
_cfg = load_config()
|
||
user_headers = cfg_get(_cfg, "model", "default_headers")
|
||
alias_headers = cfg_get(_cfg, "model", "extra_headers")
|
||
if isinstance(alias_headers, dict) and alias_headers:
|
||
user_headers = {**(user_headers if isinstance(user_headers, dict) else {}), **alias_headers}
|
||
except Exception:
|
||
return headers
|
||
if not isinstance(user_headers, dict) or not user_headers:
|
||
return headers
|
||
merged = dict(headers or {})
|
||
merged.update({str(k): str(v) for k, v in user_headers.items() if v is not None})
|
||
return merged or headers
|
||
|
||
|
||
def build_or_headers(or_config: dict | None = None) -> dict:
|
||
"""OpenRouter headers, plus response-cache headers when enabled.
|
||
|
||
Precedence env > config > default: ``HERMES_OPENROUTER_CACHE`` overrides
|
||
``openrouter.response_cache``; ``HERMES_OPENROUTER_CACHE_TTL`` (1-86400 s) overrides
|
||
``openrouter.response_cache_ttl``. ``or_config=None`` reads from disk.
|
||
"""
|
||
headers = dict(_OR_HEADERS_BASE)
|
||
if or_config is None:
|
||
try:
|
||
from hermes_cli.config import load_config_readonly
|
||
or_config = load_config_readonly().get("openrouter", {})
|
||
except Exception:
|
||
or_config = {}
|
||
env_cache = os.environ.get("HERMES_OPENROUTER_CACHE", "").strip().lower()
|
||
if not (env_cache in {"1", "true", "yes", "on"} if env_cache else or_config.get("response_cache", False)):
|
||
return headers
|
||
headers["X-OpenRouter-Cache"] = "true"
|
||
env_ttl = os.environ.get("HERMES_OPENROUTER_CACHE_TTL", "").strip()
|
||
if env_ttl:
|
||
if env_ttl.isdigit() and 1 <= int(env_ttl) <= 86400:
|
||
headers["X-OpenRouter-Cache-TTL"] = str(int(env_ttl))
|
||
else:
|
||
ttl = or_config.get("response_cache_ttl", 300)
|
||
if isinstance(ttl, (int, float)) and 1 <= ttl <= 86400:
|
||
headers["X-OpenRouter-Cache-TTL"] = str(int(ttl))
|
||
return headers
|
||
|
||
|
||
# NVIDIA NIM cloud billing attribution; host-gated because NVIDIA_BASE_URL may be a local NIM.
|
||
_NVIDIA_NIM_CLOUD_HEADERS = {"X-BILLING-INVOKE-ORIGIN": "HermesAgent"}
|
||
|
||
|
||
def build_nvidia_nim_headers(base_url: str | None) -> dict:
|
||
"""Return NVIDIA NIM cloud attribution headers for build.nvidia.com traffic."""
|
||
return dict(_NVIDIA_NIM_CLOUD_HEADERS) if base_url_host_matches(str(base_url or ""), "integrate.api.nvidia.com") else {}
|
||
|
||
|
||
# Vercel AI Gateway attribution (HTTP-Referer → referrerUrl, X-Title → appName).
|
||
from hermes_cli.version_info import get_version_info
|
||
|
||
_AI_GATEWAY_HEADERS = {
|
||
"HTTP-Referer": "https://hermes-agent.nousresearch.com",
|
||
"X-Title": "Hermes Agent",
|
||
"User-Agent": f"HermesAgent/{get_version_info().base_version}",
|
||
}
|
||
|
||
# Nous Portal attribution extra_body. Tags come from agent.portal_tags so the client= marker
|
||
# tracks the canonical base version — never inline a literal here.
|
||
from agent.portal_tags import nous_portal_tags as _nous_portal_tags
|
||
|
||
|
||
def _nous_extra_body() -> dict:
|
||
"""Fresh Nous Portal ``extra_body`` (per call, so a hot-reloaded version is reflected)."""
|
||
return {"tags": _nous_portal_tags()}
|
||
|
||
|
||
# Set at resolve time — True if the auxiliary client points to Nous Portal
|
||
auxiliary_is_nous: bool = False
|
||
|
||
# _OPENROUTER_MODEL MUST stay a :free SKU (matching the free_only warning): this lane engages
|
||
# silently, and a paid default meant spend the user never opted into. User-configured values
|
||
# are honored untouched (_warn_paid_lane_once fires).
|
||
_OPENROUTER_MODEL = "nvidia/nemotron-3-ultra-550b-a55b:free"
|
||
_NOUS_MODEL = "google/gemini-3.6-flash"
|
||
_NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1"
|
||
_ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com"
|
||
_AUTH_JSON_PATH = get_hermes_home() / "auth.json"
|
||
_AUTH_JSON_PATH_AT_IMPORT = _AUTH_JSON_PATH
|
||
|
||
|
||
def _auth_json_path():
|
||
"""Active profile's ``auth.json`` at call time (a patched ``_AUTH_JSON_PATH`` still wins). The
|
||
import-time constant is the LAUNCH profile's; under multiplexing a secondary's auxiliary calls
|
||
would otherwise authenticate to Nous with the default profile's token."""
|
||
from hermes_cli.auth import _auth_file_path
|
||
return _AUTH_JSON_PATH if _AUTH_JSON_PATH != _AUTH_JSON_PATH_AT_IMPORT else _auth_file_path()
|
||
|
||
# Hosts exposing BOTH ``…/anthropic`` and a sibling OpenAI ``…/v1``. Matched on the URL *host*
|
||
# only: unconditional rewrites break Anthropic-only gateways.
|
||
_DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES = ("minimax.io", "minimax.chat", "minimaxi.com")
|
||
_DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES = ("api.minimax.",)
|
||
|
||
|
||
def _is_dual_surface_anthropic_host(url: str) -> bool:
|
||
"""True when the URL's host is a known dual-surface (MiniMax-family) host."""
|
||
try:
|
||
host = (urlparse(url).hostname or "").lower()
|
||
except ValueError:
|
||
return False
|
||
return any(
|
||
host == suffix or host.endswith("." + suffix) for suffix in _DUAL_SURFACE_ANTHROPIC_HOST_SUFFIXES
|
||
) or any(host.startswith(prefix) for prefix in _DUAL_SURFACE_ANTHROPIC_HOST_PREFIXES)
|
||
|
||
|
||
def _to_openai_base_url(base_url: str) -> str:
|
||
"""Normalize dual-surface Anthropic URLs to their OpenAI-compatible sibling.
|
||
|
||
MiniMax-family: ``/anthropic`` → ``/v1``; ZAI Coding Plan → ``/coding/paas/v4`` (the general
|
||
endpoint bills separately); Kimi Code ``/coding`` → ``/coding/v1`` (the OpenAI SDK path 404s
|
||
without it). Anthropic-only gateways keep their path.
|
||
"""
|
||
url = str(base_url or "").strip().rstrip("/")
|
||
if base_url_hostname(url) == "api.actual.inc":
|
||
from hermes_cli.auth import normalize_actual_base_url
|
||
return normalize_actual_base_url(url)
|
||
if url.endswith("/anthropic"):
|
||
if base_url_host_matches(url, "open.bigmodel.cn") or base_url_host_matches(url, "api.z.ai"):
|
||
rewritten = url[: -len("/anthropic")] + "/coding/paas/v4"
|
||
logger.debug("Auxiliary client: rewrote ZAI base URL %s → %s", url, rewritten)
|
||
return rewritten
|
||
if _is_dual_surface_anthropic_host(url):
|
||
rewritten = url[: -len("/anthropic")] + "/v1"
|
||
logger.debug("Auxiliary client: rewrote dual-surface base URL %s → %s", url, rewritten)
|
||
return rewritten
|
||
logger.debug(
|
||
"Auxiliary client: keeping Anthropic-only base URL %s (no dual-surface host match)", url)
|
||
return url
|
||
if base_url_host_matches(url, "api.kimi.com") and url.endswith("/coding"):
|
||
rewritten = url + "/v1"
|
||
logger.debug("Auxiliary client: rewrote Kimi base URL %s → %s", url, rewritten)
|
||
return rewritten
|
||
return url
|
||
|
||
|
||
def _load_pool_with_credentials(provider: str, note: str = "") -> Optional[Any]:
|
||
"""``load_pool(provider)`` when it has credentials, else None (never raises)."""
|
||
try:
|
||
pool = load_pool(provider)
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary client: could not load pool for %s%s: %s", provider, note, exc)
|
||
return None
|
||
return pool if pool and pool.has_credentials() else None
|
||
|
||
|
||
def _select_pool_entry(provider: str) -> Tuple[bool, Optional[Any]]:
|
||
"""Return (pool_exists_for_provider, selected_entry)."""
|
||
pool = _load_pool_with_credentials(provider)
|
||
if pool is None:
|
||
return False, None
|
||
try:
|
||
return True, pool.select()
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary client: could not select pool entry for %s: %s", provider, exc)
|
||
return True, None
|
||
|
||
|
||
def _peek_pool_entry(provider: str, pool: Any = None) -> Optional[Any]:
|
||
"""Best-effort current/next pool entry without mutating selection order.
|
||
|
||
``pool`` skips the disk re-read when the caller already loaded it.
|
||
"""
|
||
if pool is None:
|
||
pool = _load_pool_with_credentials(provider, " (peek)")
|
||
if pool is None:
|
||
return None
|
||
try:
|
||
current_fn = getattr(pool, "current", None)
|
||
current = current_fn() if callable(current_fn) else None
|
||
if current is not None:
|
||
return current
|
||
peek_fn = getattr(pool, "peek", None)
|
||
if callable(peek_fn):
|
||
return peek_fn()
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary client: could not peek pool entry for %s: %s", provider, exc)
|
||
return None
|
||
|
||
|
||
def _pool_runtime_api_key(entry: Any) -> str:
|
||
# runtime_api_key handles provider-specific fallback (e.g. agent_key for nous); None entry → "".
|
||
key = getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "")
|
||
return str(key or "").strip()
|
||
|
||
|
||
def _pool_runtime_base_url(entry: Any, fallback: str = "") -> str:
|
||
if entry is None:
|
||
return str(fallback or "").strip().rstrip("/")
|
||
if getattr(entry, "provider", None) == "nous":
|
||
# Canonical auth-layer reader so the env override shares one normalization path.
|
||
from hermes_cli.auth import _nous_inference_env_override
|
||
env_url = _nous_inference_env_override()
|
||
if env_url:
|
||
return env_url
|
||
# runtime_base_url is provider-aware; fall back for non-PooledCredential entries.
|
||
url = (getattr(entry, "runtime_base_url", None) or getattr(entry, "inference_base_url", None)
|
||
or getattr(entry, "base_url", None) or fallback)
|
||
return str(url or "").strip().rstrip("/")
|
||
|
||
|
||
# Hosts the aux Anthropic path may be pointed at via model.base_url; anything else falls back
|
||
# to the Anthropic default so a foreign host never leaks in.
|
||
_ANTHROPIC_COMPATIBLE_HOSTS = frozenset({"api.anthropic.com"})
|
||
|
||
|
||
def _is_anthropic_compatible_host(url: str) -> bool:
|
||
"""True for native Anthropic hosts and gateways serving Messages under a ``/anthropic`` path
|
||
(same convention as runtime_provider / ``_wrap_if_needed``), so a configured ``model.base_url``
|
||
whose gateway holds auth is not discarded. A bare non-Anthropic base_url is False."""
|
||
if not url:
|
||
return False
|
||
try:
|
||
parsed = urlparse(url)
|
||
if (parsed.hostname or "").strip().lower().rstrip(".") in _ANTHROPIC_COMPATIBLE_HOSTS:
|
||
return True
|
||
path = (parsed.path or "").rstrip("/").lower()
|
||
return path.endswith("/anthropic") or path.endswith("/anthropic/v1")
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _nous_min_key_ttl_seconds() -> int:
|
||
try:
|
||
return max(60, int(os.getenv("HERMES_NOUS_MIN_KEY_TTL_SECONDS", "1800")))
|
||
except (TypeError, ValueError):
|
||
return 1800
|
||
|
||
|
||
def _scoped_key_env(name: str) -> str:
|
||
"""Read a provider API key (or its paired base-URL) env var through the profile secret scope.
|
||
|
||
In agent turns the scope's verdict is authoritative (a scoped miss must not borrow another
|
||
profile's key); only the unscoped default-profile path (``UnscopedSecretError``) reads
|
||
``os.environ`` -- any other scope failure propagates instead of borrowing the ambient env.
|
||
"""
|
||
if not name:
|
||
return ""
|
||
from agent.secret_scope import UnscopedSecretError, get_secret
|
||
|
||
try:
|
||
return (get_secret(name) or "").strip()
|
||
except UnscopedSecretError:
|
||
return (os.getenv(name) or "").strip()
|
||
|
||
|
||
# Codex Responses → chat.completions adapter, so aux consumers need no changes.
|
||
def _parse_codex_final_response(final: Any) -> Tuple[List[str], List[Any], Any]:
|
||
"""Split a completed Responses object into (text_parts, tool_calls, usage) in chat.completions shape."""
|
||
text_parts: List[str] = []
|
||
tool_calls_raw: List[Any] = []
|
||
for item in (getattr(final, "output", None) or []):
|
||
item_type = _field(item, "type")
|
||
if item_type == "message":
|
||
for part in (_field(item, "content") or []):
|
||
part_type = _field(part, "type")
|
||
if part_type in {"output_text", "text"}:
|
||
text_parts.append(_field(part, "text", ""))
|
||
elif part_type == "refusal":
|
||
# A refusal part carries the model's explanation; dropping it turns a
|
||
# refusal-only turn into an empty response that gets retried.
|
||
text_parts.append(_field(part, "refusal", ""))
|
||
elif item_type == "function_call":
|
||
tool_calls_raw.append(SimpleNamespace(
|
||
id=_field(item, "call_id", ""), type="function",
|
||
function=SimpleNamespace(
|
||
name=_field(item, "name", ""), arguments=_field(item, "arguments", "{}"))))
|
||
usage = None
|
||
resp_usage = getattr(final, "usage", None)
|
||
if resp_usage:
|
||
def _u(key: str) -> int:
|
||
return getattr(resp_usage, key, 0) or (resp_usage.get(key, 0) if isinstance(resp_usage, dict) else 0)
|
||
usage = SimpleNamespace(
|
||
prompt_tokens=_u("input_tokens"), completion_tokens=_u("output_tokens"),
|
||
total_tokens=_u("total_tokens"))
|
||
return text_parts, tool_calls_raw, usage
|
||
|
||
|
||
def _close_quietly(target: Any, failure_note: Optional[str]) -> None:
|
||
"""Call ``target.close()`` if present; a failure is debug-logged under ``failure_note`` (silent when None)."""
|
||
close = getattr(target, "close", None)
|
||
if callable(close):
|
||
try:
|
||
close()
|
||
except Exception:
|
||
if failure_note:
|
||
logger.debug("Codex auxiliary: %s", failure_note, exc_info=True)
|
||
|
||
|
||
# The context compressor keys its retry-ladder classification on this text (#124077).
|
||
CODEX_STREAM_STALL_MARKER = "stream stalled"
|
||
|
||
|
||
class _CodexStreamGuard:
|
||
"""Progress-aware deadline + FD-safe timeout watchdog for one Codex aux stream attempt.
|
||
|
||
(1) The first substantive payload must arrive within ``no_progress_timeout`` or we fail fast into
|
||
the caller's retry/fallback chain (a dead or keepalive-only zombie must not hold the budget);
|
||
(2) each substantive event re-arms that window (keepalive/lifecycle frames do NOT, mirroring
|
||
commit-fence gating) so a live stream is never killed by an absolute total; (3) a hard ceiling
|
||
from ``_aux_stream_total_ceiling`` still terminates a pathological drip.
|
||
"""
|
||
|
||
def __init__(
|
||
self, client: Any, total_timeout: Optional[float],
|
||
no_progress_timeout: Optional[float] = None,
|
||
):
|
||
self._client = client
|
||
self.total_timeout = total_timeout
|
||
self._start = time.monotonic()
|
||
# Task-scoped override (auxiliary.<task>.no_progress_timeout, #108104); falls back to the
|
||
# built-in default when unset or not a positive number.
|
||
if isinstance(no_progress_timeout, (int, float)) and no_progress_timeout > 0:
|
||
self.no_progress_timeout = float(no_progress_timeout)
|
||
else:
|
||
self.no_progress_timeout = _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS
|
||
# Progress-aware stream deadlines (supersedes the old single absolute kill at ``total_timeout``).
|
||
# Three regimes: 1. First token: the stream must produce its first substantive payload within
|
||
# ``no_progress_timeout`` (60s default) or we fail fast and let the caller's normal retry/fallback
|
||
# chain run — a dead (or keepalive-only zombie) Codex stream no longer holds the full 300s
|
||
# compression budget before falling back (masoria report, Aug 2026: 3 stacked 300s waits -> 20+ min
|
||
# stuck on "Summarizing"). 2. Streaming: every substantive event re-arms the deadline by
|
||
# ``no_progress_timeout`` — a live stream is never killed by an absolute total, so a long reasoning
|
||
# summary that is actually producing tokens completes instead of timing out at 300s and falling back
|
||
# (#54915's original complaint, fixed properly). Keepalive/lifecycle frames do NOT re-arm, mirroring
|
||
# the commit-fence progress gating (#96707). 3. Hard ceiling: an absolute backstop from
|
||
# ``_aux_stream_total_ceiling`` (max(600s, 4x configured timeout) — the same bound the streamed
|
||
# chat.completions path uses) so a pathological one-token-per-59s drip still terminates.
|
||
if total_timeout is not None:
|
||
self.no_progress_timeout = min(self.no_progress_timeout, float(total_timeout))
|
||
self.hard_deadline = self._start + _aux_stream_total_ceiling(total_timeout)
|
||
# The waiting host's absolute deadline clamps the ceiling so the watchdog Timer severs
|
||
# the socket the instant the host stops waiting — a stream blocked between events
|
||
# can't be stopped by a per-event check.
|
||
host_deadline = _current_aux_stream_deadline()
|
||
if isinstance(host_deadline, (int, float)) and host_deadline < self.hard_deadline:
|
||
self.hard_deadline = float(host_deadline)
|
||
self._deadline_lock = threading.Lock()
|
||
self._progress_deadline = self._start + self.no_progress_timeout
|
||
self.saw_content = threading.Event()
|
||
self.timed_out = threading.Event()
|
||
# Set only when the timeout WON (not when the owner hard-cancelled first): tells the
|
||
# owner's ``finally`` the shared client's FDs still need a real close.
|
||
self.timeout_release_pending = threading.Event()
|
||
self.stream_finished = threading.Event()
|
||
self._timer = None
|
||
# The owner may return on hard cancel while this attempt is still blocked in the SDK
|
||
# stream. Timer threads don't inherit the worker's thread-local protection state, so
|
||
# freeze the hard-cancel source before creating the timer.
|
||
self._protected_cancel_check = _capture_aux_cancel_check() if _aux_interrupt_protected() else None
|
||
self._attempt_stream_lock = threading.Lock()
|
||
self._attempt_stream: Any = None
|
||
# The request-driving thread owns the transport FDs — see _close_client_on_timeout.
|
||
self._owner_tid = threading.get_ident()
|
||
|
||
def effective_deadline(self) -> float:
|
||
with self._deadline_lock:
|
||
return min(self.hard_deadline, self._progress_deadline)
|
||
|
||
def cancel_requested(self) -> bool:
|
||
"""True when the frozen hard-cancel source says the owner already cancelled."""
|
||
check = self._protected_cancel_check
|
||
return callable(check) and _captured_aux_cancel_requested(check)
|
||
|
||
def adopt_stream(self, stream: Any) -> None:
|
||
with self._attempt_stream_lock:
|
||
self._attempt_stream = stream
|
||
|
||
def release_stream(self, stream: Any) -> None:
|
||
"""Owner-side: close the attempt stream silently and forget it."""
|
||
_close_quietly(stream, None)
|
||
with self._attempt_stream_lock:
|
||
self._attempt_stream = None
|
||
|
||
def close_attempt_stream(self, failure_note: str) -> None:
|
||
"""Closes only this attempt's stream — never the process-shared client."""
|
||
with self._attempt_stream_lock:
|
||
stream = self._attempt_stream
|
||
_close_quietly(stream, failure_note)
|
||
|
||
def record_progress(self) -> None:
|
||
"""Substantive payload re-arms the no-progress window; the hard ceiling never moves."""
|
||
with self._deadline_lock:
|
||
self._progress_deadline = time.monotonic() + self.no_progress_timeout
|
||
|
||
def timeout_message(self) -> str:
|
||
elapsed = time.monotonic() - self._start
|
||
if time.monotonic() >= self.hard_deadline:
|
||
return f"Codex auxiliary Responses stream exceeded {self.hard_deadline - self._start:.1f}s hard ceiling"
|
||
if not self.saw_content.is_set():
|
||
return (
|
||
"Codex auxiliary Responses stream produced no output "
|
||
f"within {float(self.no_progress_timeout):.1f}s (no-progress timeout, {elapsed:.1f}s elapsed)")
|
||
return (
|
||
f"Codex auxiliary Responses {CODEX_STREAM_STALL_MARKER}: no new output "
|
||
f"for {float(self.no_progress_timeout):.1f}s ({elapsed:.1f}s elapsed)")
|
||
|
||
def _close_client_on_timeout(self) -> None:
|
||
begin_timeout_cleanup = getattr(self._protected_cancel_check, "begin_timeout_cleanup", None)
|
||
if callable(begin_timeout_cleanup):
|
||
timeout_won = bool(begin_timeout_cleanup())
|
||
else:
|
||
timeout_won = not self.cancel_requested()
|
||
# Publish transport timeout only after the attempt-local decision is fixed, so owner
|
||
# polling cannot observe completion in between.
|
||
self.timed_out.set()
|
||
if not timeout_won:
|
||
# Owner already hard-cancelled. The OpenAI client is process-shared, so never
|
||
# close/evict it here; wake only this attempt's stream if responses.create()
|
||
# returned one, else rely on the bounded SDK timeout.
|
||
self.close_attempt_stream("cancelled attempt stream close during timeout failed")
|
||
return
|
||
# FD-ownership contract: only the thread driving the request may ``close()`` this
|
||
# client's FDs. From a stranger thread (the watchdog Timer) only ``shutdown()`` is
|
||
# FD-safe — ``close()`` releases the raw TLS fd while the owner's OpenSSL BIO still
|
||
# caches it, the kernel recycles it (e.g. into a SQLite handle), and the owner's TLS
|
||
# flush corrupts that file. The owner does the real close in its ``finally``.
|
||
# This callback has two callers — ``_check_cancelled`` on the owning thread, and the daemon watchdog
|
||
# ``threading.Timer``, which is a stranger thread. The owning thread performs the real close in the
|
||
# ``finally`` below, which is where the FD release belongs. See #70773.
|
||
self.timeout_release_pending.set()
|
||
if threading.get_ident() == self._owner_tid:
|
||
_close_quietly(self._client, "client close during timeout failed")
|
||
else:
|
||
try:
|
||
from agent.agent_runtime_helpers import force_close_tcp_sockets
|
||
shutdown_count = force_close_tcp_sockets(self._client)
|
||
logger.info(
|
||
"Codex auxiliary client aborted (timeout, tcp_force_closed=%d, "
|
||
"deferred_close=stranger_thread)", shutdown_count)
|
||
except Exception:
|
||
logger.debug("Codex auxiliary: client abort during timeout failed", exc_info=True)
|
||
# Socket shutdown only wakes a reader on a REAL transport; the owner may be blocked
|
||
# inside the SDK's event stream (or a socketless test double). Closing the
|
||
# attempt-owned stream releases it without touching shared FDs.
|
||
self.close_attempt_stream("attempt stream close during stranger-thread timeout failed")
|
||
# The aux client cache wraps this same client; drop the entry so the next aux call
|
||
# doesn't reuse the dead transport and fail fast.
|
||
try:
|
||
# After we close the httpx transport above, the cache must drop that entry — otherwise the next
|
||
# auxiliary call (compression retry, memory flush, etc.) reuses the dead client and fails fast
|
||
# with a connection error. See issue #23432.
|
||
_evict_cached_client_instance(self._client)
|
||
except Exception:
|
||
logger.debug("Codex auxiliary: cache eviction on timeout failed", exc_info=True)
|
||
|
||
def check_cancelled(self) -> None:
|
||
if self.total_timeout is not None and time.monotonic() >= self.effective_deadline():
|
||
if not self.timed_out.is_set():
|
||
self._close_client_on_timeout()
|
||
raise TimeoutError(self.timeout_message())
|
||
try:
|
||
from tools.interrupt import is_interrupted
|
||
# Protected atomic aux tasks (compression) must not abort on a mid-flight gateway
|
||
# interrupt (degraded fallback marker); explicit host cancel has its own exception.
|
||
if _aux_interrupt_cancel_requested():
|
||
raise AuxiliaryExplicitCancellation()
|
||
# Explicit host cancellation has its own frozen exception; timeouts above still fire and other
|
||
# aux tasks remain interruptible. See #23975.
|
||
if is_interrupted() and not _aux_interrupt_protected():
|
||
raise InterruptedError("Codex auxiliary Responses stream interrupted")
|
||
except InterruptedError:
|
||
raise
|
||
except Exception:
|
||
# Interrupt state is best-effort UX; never a new failure mode.
|
||
pass
|
||
|
||
def _watchdog_fire(self) -> None:
|
||
# Re-armable: if progress moved the deadline forward, reschedule instead of killing a
|
||
# live stream.
|
||
remaining = self.effective_deadline() - time.monotonic()
|
||
if remaining > 0:
|
||
if not (self.timed_out.is_set() or self.stream_finished.is_set()):
|
||
self._arm_timer(remaining)
|
||
return
|
||
self._close_client_on_timeout()
|
||
|
||
def _arm_timer(self, delay: float) -> None:
|
||
self._timer = t = threading.Timer(delay, self._watchdog_fire)
|
||
t.daemon = True
|
||
t.start()
|
||
|
||
def start(self) -> None:
|
||
"""Arm the watchdog (when a total timeout exists) and run the first cancel check."""
|
||
if self.total_timeout:
|
||
self._arm_timer(max(self.effective_deadline() - time.monotonic(), 0.0))
|
||
self.check_cancelled()
|
||
|
||
def on_event(self, _event: Any) -> None:
|
||
# TTFP telemetry records every frame, but forward progress (compression commit fence,
|
||
# no-progress window) counts only substantive payloads — keepalives must not re-arm,
|
||
# so a zombie stream dies at the same window as a dead connection.
|
||
# #93650: keep bulk wire-format payload out of the SDK's GIL-holding request transform on auxiliary
|
||
# calls too.
|
||
if _codex_event_has_content(_event):
|
||
self.record_progress()
|
||
self.saw_content.set()
|
||
_notify_aux_provider_response()
|
||
else:
|
||
_notify_aux_timing_response()
|
||
self.check_cancelled()
|
||
|
||
def finish(self) -> None:
|
||
"""Owner ``finally``: stop the watchdog and release FDs a stranger-thread timeout only shut down."""
|
||
self.stream_finished.set()
|
||
if self._timer is not None:
|
||
self._timer.cancel()
|
||
# Gated on timeout_release_pending, NOT timed_out: after a hard-cancel the shared
|
||
# client must stay usable for other sessions.
|
||
if self.timeout_release_pending.is_set():
|
||
_close_quietly(self._client, "owner-thread close after timeout failed")
|
||
|
||
|
||
class _CodexCompletionsAdapter:
|
||
"""Drop-in shim routing chat.completions.create() kwargs through Codex Responses streaming."""
|
||
|
||
def __init__(self, real_client: OpenAI, model: str):
|
||
self._client = real_client
|
||
self._model = model
|
||
|
||
def _build_responses_kwargs(self, kwargs: Dict[str, Any]) -> Tuple[Dict[str, Any], str, Any]:
|
||
"""chat.completions kwargs → Responses API kwargs, ``(resp_kwargs, model, timeout)``; mirrors codex.py::build_kwargs."""
|
||
# Separate system/instructions from replayable conversation messages, then route the rest through
|
||
# the SINGLE shared chat->Responses converter used by the main agent transport
|
||
# (agent/transports/codex.py). Maintaining a private conversion loop here let chat-style messages
|
||
# with role="tool" leak straight into Responses input[] — which the Responses API rejects with
|
||
# "Invalid value: 'tool'. Supported values are: 'assistant', 'system', 'developer', and 'user'."
|
||
# (issue #5709, hit hard by flush_memories() / compression replaying real session history that
|
||
# includes assistant tool_calls + role="tool" results). The shared converter encodes assistant tool
|
||
# calls as `function_call` items and tool results as `function_call_output` items with a valid
|
||
# call_id, so every Responses path normalizes tool history identically and cannot drift.
|
||
from agent.codex_responses_adapter import (
|
||
_chat_messages_to_responses_input,
|
||
_classify_responses_issuer,
|
||
_responses_tools,
|
||
_wire_model_identity,
|
||
classify_responses_route,
|
||
)
|
||
from agent.transports.codex import _alias_wire_tools
|
||
model = kwargs.get("model", self._model)
|
||
wire_model = _wire_model_identity(model)
|
||
host = str(getattr(self._client, "base_url", "") or "")
|
||
is_copilot = base_url_host_matches(host, "githubcopilot.com")
|
||
# Same route classifier as the main transport, so the issuer stamp matches what it minted.
|
||
route = classify_responses_route(SimpleNamespace(provider=None, base_url=host))
|
||
is_xai = route.is_xai_responses
|
||
is_github = route.is_github_responses
|
||
tools = kwargs.get("tools")
|
||
if tools:
|
||
# xAI Responses rejects ``pattern``/``format`` JSON Schema keywords (400); strip for
|
||
# chat_completion_helpers.py parity. Deep-copy first — sanitizers mutate inner dicts
|
||
# in place and would strip the caller's tool registry.
|
||
try:
|
||
import copy as _copy
|
||
from tools.schema_sanitizer import strip_pattern_and_format, strip_slash_enum
|
||
tools = _copy.deepcopy(list(tools))
|
||
tools, _ = strip_pattern_and_format(tools)
|
||
tools, _ = strip_slash_enum(tools)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"Auxiliary client: failed to sanitize tool schemas for "
|
||
"Codex/xAI Responses path: %s", exc,
|
||
)
|
||
# Tool schemas go through the SAME converter (``strict: False``) and the SAME reserved-name
|
||
# aliasing (OpenCode / Perplexity / xAI) as agent/transports/codex.py::build_kwargs. A private
|
||
# list here sent raw names without ``strict``, so title/compression/MoA calls 400ed on routes
|
||
# where the main loop worked (#114260). The alias map is request-local: it rides on the payload
|
||
# (``_wire_aliases``, popped in ``create()``), never on the instance — aux adapters are shared.
|
||
wire_tools, wire_aliases = _alias_wire_tools(
|
||
_responses_tools(tools),
|
||
{"provider": getattr(self._client, "_hermes_aux_effective_provider", "") or None, "base_url": host},
|
||
is_xai,
|
||
)
|
||
renamed = {original: alias for alias, original in wire_aliases.items()}
|
||
# System → ``instructions``; the rest goes through the SINGLE shared chat→Responses
|
||
# converter (a private loop here once let role="tool" leak into input[]; the shared one
|
||
# encodes tool history as function_call/function_call_output).
|
||
instructions = "You are a helpful assistant."
|
||
replay_messages: List[Dict[str, Any]] = []
|
||
for msg in kwargs.get("messages", []):
|
||
content = msg.get("content") or ""
|
||
if msg.get("role", "user") == "system":
|
||
instructions = content if isinstance(content, str) else str(content)
|
||
continue
|
||
# Replayed history names each tool the way THIS request declares it.
|
||
if renamed and msg.get("tool_calls"):
|
||
msg = {**msg, "tool_calls": [
|
||
{**tc, "function": {**tc["function"], "name": renamed[tc["function"]["name"]]}}
|
||
if isinstance(tc, dict) and (tc.get("function") or {}).get("name") in renamed else tc
|
||
for tc in msg["tool_calls"]
|
||
]}
|
||
replay_messages.append(msg)
|
||
# Copilot binds replayed codex_message_items ids to a backend connection that doesn't
|
||
# survive credential rotation (401 on replay) — same guard as build_kwargs. Aux calls
|
||
# never send ``context_management`` (main-turn feature): no compaction checkpoint.
|
||
# Auxiliary calls (context compression, flush_memories, MoA aggregation) go through this adapter
|
||
# instead of agent/transports/codex.py's build_kwargs, so they need the same guard applied
|
||
# independently. See #32716.
|
||
# Aux requests run their own model; stamp/filter reasoning provenance against it, not the main agent's.
|
||
input_items = _chat_messages_to_responses_input(
|
||
replay_messages, is_github_responses=is_copilot,
|
||
current_issuer_kind=_classify_responses_issuer(base_url=host, **route._asdict()),
|
||
current_issuer_model=wire_model, native_compaction_eligible=False,
|
||
)
|
||
resp_kwargs: Dict[str, Any] = {
|
||
# Codex only knows the base slug; strip the Hermes ``-900k`` picker suffix.
|
||
"model": wire_model, "instructions": instructions,
|
||
"input": input_items or [{"role": "user", "content": ""}], "store": False,
|
||
}
|
||
# Forward the chat.completions timeout; otherwise a Codex stream can sit behind a
|
||
# dead-looking CLI until the user force-interrupts.
|
||
timeout = kwargs.get("timeout")
|
||
if timeout is not None:
|
||
resp_kwargs["timeout"] = timeout
|
||
# Per-request HTTP headers (OpenCode session affinity, Copilot x-initiator) map to real
|
||
# headers via the SDK kwarg — forward them.
|
||
if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
|
||
resp_kwargs["extra_headers"] = dict(kwargs["extra_headers"])
|
||
# The Codex endpoint rejects max_output_tokens/temperature (400) — omit.
|
||
extra_body = kwargs.get("extra_body") or {}
|
||
if isinstance(extra_body, dict):
|
||
# service_tier (fast mode) is a top-level Responses field; xAI's endpoint rejects it.
|
||
service_tier = extra_body.get("service_tier")
|
||
if isinstance(service_tier, str) and service_tier.strip() and not is_xai:
|
||
resp_kwargs["service_tier"] = service_tier.strip()
|
||
reasoning_cfg = extra_body.get("reasoning")
|
||
if isinstance(reasoning_cfg, dict):
|
||
# Shared per-model vocabulary with the main transport ("max" only where the model publishes it; "minimal"/"ultra"
|
||
# clamp to a listed level; ``()`` = the model takes no ``reasoning`` field at all — gpt-4o/4.1 on api.openai.com,
|
||
# #76255). ``enabled: False`` goes on the wire as ``effort: none`` where the vocabulary has it,
|
||
# since an omitted field leaves the model's default effort on (#75227).
|
||
from agent.reasoning_effort import clamp_effort
|
||
from agent.transports.codex import _codex_efforts_for_route
|
||
supported = _codex_efforts_for_route(model, host, is_codex_backend=route.is_codex_backend)
|
||
if supported and reasoning_cfg.get("enabled") is not False:
|
||
# Truthy-only: Codex 400s on e.g. {"effort": null}, so falsy → default.
|
||
effort = clamp_effort(reasoning_cfg.get("effort") or "medium", supported)
|
||
resp_kwargs["reasoning"] = {"effort": effort, "summary": "auto"}
|
||
resp_kwargs["include"] = ["reasoning.encrypted_content"]
|
||
elif "none" in supported and not is_xai:
|
||
resp_kwargs["reasoning"] = {"effort": "none"}
|
||
if wire_tools:
|
||
resp_kwargs["tools"] = wire_tools
|
||
if wire_aliases:
|
||
resp_kwargs["_wire_aliases"] = wire_aliases
|
||
# Stable prompt-cache routing: key is content-addressed from the static prefix
|
||
# (instructions + tool schemas) so it survives across turns, scoped by the owning
|
||
# conversation (rotation-stable logical scope, else the physical session id). Skip the
|
||
# key where the main transport does: xAI takes it in extra_body, GitHub opts out.
|
||
try:
|
||
# Reuse the Responses transport's single authoritative hash algorithm and session-scope
|
||
# normalization so equivalent static prefixes route to the same cache bucket across modes,
|
||
# without concentrating unrelated sessions into one shared bucket (see #78941).
|
||
from agent.transports.codex import _cache_scope_from_session_id, _content_cache_key
|
||
from agent.transports.codex import _default_prompt_cache_retention_for_request
|
||
if not (is_xai or is_github) and "prompt_cache_key" not in resp_kwargs:
|
||
scope = _cache_scope_from_session_id(
|
||
_runtime_main_value("cache_scope") or _runtime_main_value("session_id")
|
||
)
|
||
cache_key = _content_cache_key(resp_kwargs["instructions"], resp_kwargs.get("tools"), scope)
|
||
if cache_key:
|
||
resp_kwargs["prompt_cache_key"] = cache_key
|
||
if "prompt_cache_retention" not in resp_kwargs:
|
||
cache_retention = _default_prompt_cache_retention_for_request(model, host)
|
||
if cache_retention:
|
||
resp_kwargs["prompt_cache_retention"] = cache_retention
|
||
except Exception:
|
||
logger.debug("Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True)
|
||
# Last, like the main transport: caller extra_body must not put a rejected Astra field back.
|
||
from agent.transports.codex import _sanitize_astra_request_kwargs
|
||
_sanitize_astra_request_kwargs(resp_kwargs, model, host)
|
||
return resp_kwargs, model, timeout
|
||
|
||
def create(self, **kwargs) -> Any:
|
||
from hermes_cli.providers import is_actual_route
|
||
|
||
if is_actual_route(
|
||
getattr(self._client, "_hermes_aux_effective_provider", ""),
|
||
str(getattr(self._client, "base_url", "") or ""),
|
||
):
|
||
raise ValueError(
|
||
"Actual requests require Chat Completions; refusing to call /responses."
|
||
)
|
||
# Low-level ``responses.create(stream=True)`` and assemble the final response ourselves
|
||
# from ``response.output_item.done``: the high-level ``responses.stream()`` rebuilds from
|
||
# ``response.completed.response.output``, which Codex returns as ``null`` (SDK crash).
|
||
resp_kwargs, model, timeout = self._build_responses_kwargs(kwargs)
|
||
wire_aliases = resp_kwargs.pop("_wire_aliases", None) or {}
|
||
total_timeout = timeout if isinstance(timeout, (int, float)) and timeout > 0 else None
|
||
guard = _CodexStreamGuard(self._client, total_timeout, no_progress_timeout=kwargs.get("no_progress_timeout"))
|
||
try:
|
||
guard.start()
|
||
from agent.codex_runtime import _consume_codex_event_stream
|
||
from agent.sdk_transform_bypass import bypass_sdk_request_transform
|
||
# Keep bulk wire payload out of the SDK's GIL-holding request transform.
|
||
stream_kwargs = bypass_sdk_request_transform({**resp_kwargs, "stream": True})
|
||
event_stream = self._client.responses.create(**stream_kwargs)
|
||
guard.adopt_stream(event_stream)
|
||
# The timer may fire while responses.create() is blocked; if the cancelled attempt
|
||
# had no stream to close then, close it now that it is attempt-owned — never the shared client.
|
||
if guard.timed_out.is_set() and guard.cancel_requested():
|
||
guard.close_attempt_stream("late cancelled attempt stream close failed")
|
||
try:
|
||
# Some Codex-compatible hosts accept ``stream=True`` but return a completed
|
||
# Responses object (not iterable) — don't hand it to the consumer.
|
||
if hasattr(event_stream, "output"):
|
||
final = event_stream
|
||
else:
|
||
final = _consume_codex_event_stream(
|
||
event_stream, model=str(resp_kwargs.get("model") or model), on_event=guard.on_event
|
||
)
|
||
finally:
|
||
guard.release_stream(event_stream)
|
||
if final is None:
|
||
raise RuntimeError("Codex auxiliary Responses stream did not return a final response")
|
||
text_parts, tool_calls_raw, usage = _parse_codex_final_response(final)
|
||
# Undo only the aliases THIS request emitted, before the call reaches Hermes dispatch.
|
||
for tc in tool_calls_raw or ():
|
||
if tc.function.name in wire_aliases:
|
||
tc.function.name = wire_aliases[tc.function.name]
|
||
except Exception as exc:
|
||
if guard.timed_out.is_set():
|
||
raise TimeoutError(guard.timeout_message()) from exc
|
||
logger.debug("Codex auxiliary Responses API call failed: %s", exc)
|
||
raise
|
||
finally:
|
||
guard.finish()
|
||
# Shape the result like chat.completions.
|
||
message = SimpleNamespace(
|
||
role="assistant", content="".join(text_parts).strip() or None,
|
||
tool_calls=tool_calls_raw or None,
|
||
)
|
||
choice = SimpleNamespace(
|
||
index=0, message=message, finish_reason="stop" if not tool_calls_raw else "tool_calls"
|
||
)
|
||
return SimpleNamespace(choices=[choice], model=model, usage=usage)
|
||
|
||
|
||
class _ChatShim:
|
||
"""Exposes ``client.chat.completions.create()`` over a sync or async adapter."""
|
||
|
||
def __init__(self, adapter: Any):
|
||
self.completions = adapter
|
||
|
||
|
||
class _AsyncCompletionsAdapter:
|
||
"""Async adapter: runs the sync adapter's ``create`` via asyncio.to_thread()."""
|
||
|
||
def __init__(self, sync_adapter: Any):
|
||
self._sync = sync_adapter
|
||
|
||
async def create(self, **kwargs) -> Any:
|
||
import asyncio
|
||
return await asyncio.to_thread(self._sync.create, **kwargs)
|
||
|
||
|
||
class _AsyncAuxiliaryClientBase:
|
||
"""Async-compatible wrapper matching AsyncOpenAI.chat.completions.create().
|
||
|
||
Mirrors ``_real_client`` (when the sync wrapper has one) so cache eviction by
|
||
leaf OpenAI client drops this async entry too instead of reusing a closed transport.
|
||
"""
|
||
|
||
def __init__(self, sync_wrapper: Any):
|
||
self.chat = _ChatShim(_AsyncCompletionsAdapter(sync_wrapper.chat.completions))
|
||
self.api_key = sync_wrapper.api_key
|
||
self.base_url = sync_wrapper.base_url
|
||
if hasattr(sync_wrapper, "_real_client"):
|
||
# Mirror the sync wrapper's _real_client so cache eviction by leaf OpenAI client (e.g.
|
||
# _close_client_on_timeout in #23482) drops this async entry too. Without this, sync and async
|
||
# cache entries diverge on poisoning: the sync entry is evicted but the async entry keeps
|
||
# reusing the closed transport, failing every subsequent async aux call with 'Connection error'
|
||
# until the gateway restarts.
|
||
self._real_client = sync_wrapper._real_client
|
||
|
||
|
||
_AsyncAnthropicCompletionsAdapter = _AsyncCompletionsAdapter # imported by tests
|
||
|
||
|
||
class CodexAuxiliaryClient:
|
||
"""OpenAI-client-compatible wrapper routing through the Codex Responses API (.api_key/.base_url for introspection)."""
|
||
|
||
def __init__(self, real_client: OpenAI, model: str):
|
||
self._real_client = real_client
|
||
self.chat = _ChatShim(_CodexCompletionsAdapter(real_client, model))
|
||
self.api_key = real_client.api_key
|
||
self.base_url = real_client.base_url
|
||
|
||
def close(self):
|
||
self._real_client.close()
|
||
|
||
|
||
class AsyncCodexAuxiliaryClient(_AsyncAuxiliaryClientBase):
|
||
pass
|
||
|
||
|
||
def _translate_anthropic_response_format(anthropic_kwargs: Dict[str, Any], response_format: Any) -> None:
|
||
"""Merge an OpenAI response format into Anthropic ``output_config``."""
|
||
if not isinstance(response_format, dict):
|
||
return
|
||
format_type = response_format.get("type")
|
||
if format_type == "json_schema":
|
||
json_schema = response_format.get("json_schema")
|
||
if not isinstance(json_schema, dict) or "schema" not in json_schema:
|
||
return
|
||
schema = json_schema["schema"]
|
||
elif format_type == "json_object":
|
||
# Anthropic SDK has no schema-less JSON mode; only ``json_schema``.
|
||
schema = {"type": "object"}
|
||
else:
|
||
return
|
||
output_config = anthropic_kwargs.get("output_config")
|
||
if not isinstance(output_config, dict):
|
||
output_config = {}
|
||
anthropic_kwargs["output_config"] = output_config
|
||
output_config["format"] = {"type": "json_schema", "schema": schema}
|
||
|
||
|
||
class _AnthropicCompletionsAdapter:
|
||
"""OpenAI-client-compatible adapter for Anthropic Messages API."""
|
||
|
||
def __init__(self, real_client: Any, model: str, is_oauth: bool = False, base_url: str | None = None):
|
||
self._client = real_client
|
||
self._model = model
|
||
self._is_oauth = is_oauth
|
||
# Caller URL first; fall back to the SDK client's host only for Nous Portal — a blanket
|
||
# fallback would flip MiniMax/Zhipu aux adapters to third-party handling (strips thinking sigs).
|
||
self._base_url = base_url or None
|
||
if not self._base_url:
|
||
candidate = str(getattr(real_client, "base_url", "") or "") or None
|
||
if candidate:
|
||
with contextlib.suppress(Exception):
|
||
from agent.anthropic_endpoints import _is_nous_portal_endpoint
|
||
if _is_nous_portal_endpoint(candidate):
|
||
self._base_url = candidate
|
||
|
||
def create(self, **kwargs) -> Any:
|
||
from agent.anthropic_adapter import build_anthropic_kwargs, create_anthropic_message
|
||
from agent.transports import get_transport
|
||
model = kwargs.get("model", self._model)
|
||
# ZAI's Anthropic endpoint rejects max_tokens on vision models (code 1210);
|
||
# callers signal this via _skip_zai_max_tokens.
|
||
if kwargs.pop("_skip_zai_max_tokens", False):
|
||
max_tokens = None
|
||
else:
|
||
max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
|
||
temperature = kwargs.get("temperature")
|
||
# Reasoning priority: explicit per-call _reasoning_config (MoA per-slot) wins over
|
||
# extra_body.reasoning; build_anthropic_kwargs translates to ``thinking``.
|
||
reasoning_cfg = kwargs.get("_reasoning_config")
|
||
if reasoning_cfg is None:
|
||
_eb = kwargs.get("extra_body")
|
||
_rc = _eb.get("reasoning") if isinstance(_eb, dict) else None
|
||
if isinstance(_rc, dict):
|
||
reasoning_cfg = _rc
|
||
# OpenAI tool_choice (str or dict) → Anthropic-style name/mode string.
|
||
tool_choice = kwargs.get("tool_choice")
|
||
if isinstance(tool_choice, dict):
|
||
choice_type = str(tool_choice.get("type", "")).lower()
|
||
if choice_type == "function":
|
||
tool_choice = tool_choice.get("function", {}).get("name")
|
||
else:
|
||
tool_choice = choice_type if choice_type in {"auto", "required", "none"} else None
|
||
elif not isinstance(tool_choice, str):
|
||
tool_choice = None
|
||
anthropic_kwargs = build_anthropic_kwargs(
|
||
model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
|
||
max_tokens=max_tokens, reasoning_config=reasoning_cfg, tool_choice=tool_choice,
|
||
is_oauth=self._is_oauth,
|
||
# Portal routes on ``anthropic/<slug>`` ids and replays signed thinking
|
||
# keyed off base_url; omitting it breaks Portal model resolution.
|
||
base_url=self._base_url,
|
||
)
|
||
# Opus 4.7+ rejects non-default temperature/top_p/top_k; build_anthropic_kwargs
|
||
# also strips these as a safety net — keep both layers.
|
||
if temperature is not None:
|
||
from agent.anthropic_adapter import _forbids_sampling_params
|
||
if not _forbids_sampling_params(model):
|
||
anthropic_kwargs["temperature"] = temperature
|
||
# Per-request HTTP headers (OpenCode session affinity) — the Anthropic SDK accepts
|
||
# ``extra_headers`` on messages.create/stream too.
|
||
if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
|
||
anthropic_kwargs["extra_headers"] = {
|
||
**(anthropic_kwargs.get("extra_headers") or {}),
|
||
**kwargs["extra_headers"],
|
||
}
|
||
# response_format: top-level gets the same translation as the extra_body form; when both
|
||
# are present the extra_body form wins. Passthrough excludes ``reasoning``/``response_format``
|
||
# (already TRANSLATED to native fields — raw would 400 on strict gateways) and ``_`` Hermes plumbing.
|
||
# The adapter builds the Messages body from a fixed allow-list of kwargs, so before this an
|
||
# unrecognized top-level kwarg was dropped on the floor: the request succeeded but the schema
|
||
# contract silently became prompt compliance (#85626 review, point 2).
|
||
top_level_response_format = kwargs.get("response_format")
|
||
if top_level_response_format is not None:
|
||
_translate_anthropic_response_format(anthropic_kwargs, top_level_response_format)
|
||
caller_extra_body = kwargs.get("extra_body")
|
||
if caller_extra_body and isinstance(caller_extra_body, dict):
|
||
_translate_anthropic_response_format(anthropic_kwargs, caller_extra_body.get("response_format"))
|
||
passthrough = {
|
||
k: v for k, v in caller_extra_body.items()
|
||
if k not in {"reasoning", "response_format"} and not str(k).startswith("_")
|
||
}
|
||
if passthrough:
|
||
existing = anthropic_kwargs.get("extra_body") or {}
|
||
if not isinstance(existing, dict):
|
||
existing = {}
|
||
anthropic_kwargs["extra_body"] = {**existing, **passthrough}
|
||
response = create_anthropic_message(
|
||
self._client,
|
||
anthropic_kwargs,
|
||
# Record provider-response timing every event, but tick forward progress only for
|
||
# substantive payloads so keepalives can't hold a stalled summary open. None keeps
|
||
# the fast get_final_message path.
|
||
on_stream_event=(_anthropic_aux_stream_event_hook() if _aux_progress_active() else None),
|
||
)
|
||
_nr = get_transport("anthropic_messages").normalize_response(response, strip_tool_prefix=self._is_oauth)
|
||
usage = None
|
||
if hasattr(response, "usage") and response.usage:
|
||
prompt_tokens = getattr(response.usage, "input_tokens", 0) or 0
|
||
completion_tokens = getattr(response.usage, "output_tokens", 0) or 0
|
||
usage = SimpleNamespace(
|
||
prompt_tokens=prompt_tokens, completion_tokens=completion_tokens,
|
||
total_tokens=getattr(response.usage, "total_tokens", 0) or (prompt_tokens + completion_tokens),
|
||
)
|
||
# ToolCall already duck-types as OpenAI shape via properties.
|
||
choice = SimpleNamespace(
|
||
index=0,
|
||
message=SimpleNamespace(content=_nr.content, tool_calls=_nr.tool_calls, reasoning=_nr.reasoning),
|
||
finish_reason=_nr.finish_reason,
|
||
)
|
||
return SimpleNamespace(choices=[choice], model=model, usage=usage)
|
||
|
||
|
||
class AnthropicAuxiliaryClient:
|
||
"""OpenAI-client-compatible wrapper over a native Anthropic client."""
|
||
|
||
def __init__(self, real_client: Any, model: str, api_key: str, base_url: str, is_oauth: bool = False):
|
||
self._real_client = real_client
|
||
self.chat = _ChatShim(_AnthropicCompletionsAdapter(real_client, model, is_oauth=is_oauth, base_url=base_url))
|
||
self.api_key = api_key
|
||
self.base_url = base_url
|
||
|
||
def close(self):
|
||
close_fn = getattr(self._real_client, "close", None)
|
||
if callable(close_fn):
|
||
close_fn()
|
||
|
||
|
||
class AsyncAnthropicAuxiliaryClient(_AsyncAuxiliaryClientBase):
|
||
pass
|
||
|
||
|
||
class _BedrockCompletionsAdapter:
|
||
"""Translates ``chat.completions.create(**kwargs)`` into Bedrock Converse."""
|
||
|
||
def __init__(self, region: str, model: str):
|
||
self._region = region
|
||
self._model = model
|
||
|
||
def create(self, **kwargs) -> Any:
|
||
from agent.bedrock_adapter import call_converse
|
||
model = kwargs.get("model", self._model)
|
||
max_tokens = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
|
||
# OpenAI accepts ``stop`` as str or list; Converse requires a list.
|
||
stop = kwargs.get("stop")
|
||
if isinstance(stop, str):
|
||
stop = [stop]
|
||
if kwargs.get("tool_choice") is not None:
|
||
# Converse toolChoice isn't wired through call_converse(); surface the drop.
|
||
logger.debug(
|
||
"BedrockAuxiliaryClient: tool_choice=%r not supported by the "
|
||
"Converse shim — ignored.", kwargs.get("tool_choice"),
|
||
)
|
||
if kwargs.get("stream"):
|
||
# Converse streaming isn't wired here; call_llm's streaming consumer
|
||
# detects a final object and downgrades to non-live output.
|
||
logger.debug(
|
||
"BedrockAuxiliaryClient: stream=True requested for %s — returning a complete response "
|
||
"(Converse shim does not stream); caller downgrades to non-streaming.", model,
|
||
)
|
||
response = call_converse(
|
||
region=self._region, model=model, messages=kwargs.get("messages", []), tools=kwargs.get("tools"),
|
||
# Converse specifically defaults to the model maximum when omitted.
|
||
# Truthiness mirrors the Anthropic shim: explicit 0 means omit.
|
||
max_tokens=int(max_tokens) if max_tokens else None, temperature=kwargs.get("temperature"),
|
||
top_p=kwargs.get("top_p"), stop_sequences=stop,
|
||
)
|
||
# Converse is complete-response here: mark provider progress only after
|
||
# return so TTFP reflects real Bedrock latency, not dispatch/setup.
|
||
_notify_aux_provider_response()
|
||
return response
|
||
|
||
|
||
class BedrockAuxiliaryClient:
|
||
"""OpenAI-client-compatible wrapper over AWS Bedrock Converse API."""
|
||
|
||
def __init__(self, region: str, model: str):
|
||
self._region = region
|
||
self._model = model
|
||
self.chat = _ChatShim(_BedrockCompletionsAdapter(region, model))
|
||
self.api_key = "aws-sdk"
|
||
self.base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
|
||
|
||
def close(self):
|
||
pass
|
||
|
||
|
||
class AsyncBedrockAuxiliaryClient(_AsyncAuxiliaryClientBase):
|
||
pass
|
||
|
||
|
||
def _endpoint_speaks_anthropic_messages(base_url: str) -> bool:
|
||
"""True if ``base_url`` speaks Anthropic Messages, not OpenAI chat.completions.
|
||
|
||
Mirrors ``hermes_cli.runtime_provider._detect_api_mode_for_url`` so aux and main agree: any
|
||
``/anthropic`` URL (MiniMax, Zhipu, LiteLLM), ``api.kimi.com/coding`` (chat 404s), ``api.anthropic.com``.
|
||
"""
|
||
normalized = (base_url or "").strip().lower().rstrip("/")
|
||
if not normalized:
|
||
return False
|
||
if urlparse(normalized).path.rstrip("/").endswith(("/anthropic", "/anthropic/v1")):
|
||
return True
|
||
hostname = base_url_hostname(normalized)
|
||
return hostname == "api.anthropic.com" or bool(hostname == "api.kimi.com" and "/coding" in normalized)
|
||
|
||
|
||
def _maybe_wrap_anthropic(
|
||
client_obj: Any, model: str, api_key: str, base_url: str, api_mode: Optional[str] = None
|
||
) -> Any:
|
||
"""Rewrap a plain OpenAI client in ``AnthropicAuxiliaryClient`` when the endpoint speaks Anthropic Messages.
|
||
|
||
Single transport-correction chokepoint at the end of every ``resolve_provider_client`` branch; returns
|
||
``client_obj`` unchanged for probe stubs/specialized adapters, OpenAI-wire, explicit non-Anthropic
|
||
``api_mode``, or missing ``anthropic`` SDK.
|
||
"""
|
||
# Anthropic/Bedrock/Codex wrappers, plus any client declaring HERMES_SKIP_TRANSPORT_WRAP
|
||
# (native/ACP shims, in-tree or plugin), must never be re-dispatched through a wire adapter —
|
||
# a class-attribute declaration rather than isinstance so this hot path never imports them.
|
||
if (
|
||
isinstance(client_obj, _AuxProbeClientStub)
|
||
or _safe_isinstance(client_obj, (AnthropicAuxiliaryClient, BedrockAuxiliaryClient, CodexAuxiliaryClient))
|
||
or _client_declares(client_obj, "HERMES_SKIP_TRANSPORT_WRAP")
|
||
):
|
||
return client_obj
|
||
# Explicit non-anthropic api_mode wins over URL heuristics.
|
||
if api_mode != "anthropic_messages" and (api_mode or not _endpoint_speaks_anthropic_messages(base_url)):
|
||
return client_obj
|
||
try:
|
||
from agent.anthropic_adapter import build_anthropic_client
|
||
except ImportError:
|
||
logger.warning(
|
||
"Endpoint %s speaks Anthropic Messages but the anthropic SDK is "
|
||
"not installed — falling back to OpenAI-wire (will likely 404).",
|
||
base_url,
|
||
)
|
||
return client_obj
|
||
try:
|
||
real_client = build_anthropic_client(api_key, base_url)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"Failed to build Anthropic client for %s (%s) — falling back to "
|
||
"OpenAI-wire client.", base_url, exc,
|
||
)
|
||
return client_obj
|
||
logger.debug(
|
||
"Auxiliary transport: wrapping client in AnthropicAuxiliaryClient "
|
||
"(model=%s, base_url=%s, api_mode=%s)",
|
||
model, base_url[:60] if base_url else "", api_mode or "auto-detected",
|
||
)
|
||
return AnthropicAuxiliaryClient(real_client, model, api_key, base_url, is_oauth=False)
|
||
|
||
|
||
def _read_nous_auth() -> Optional[dict]:
|
||
"""Nous provider state dict from the credential pool or ~/.hermes/auth.json; None when not active with tokens."""
|
||
pool_present, entry = _select_pool_entry("nous")
|
||
if pool_present:
|
||
if entry is None:
|
||
return None
|
||
return {
|
||
"access_token": getattr(entry, "access_token", ""),
|
||
"refresh_token": getattr(entry, "refresh_token", None),
|
||
"agent_key": getattr(entry, "agent_key", None),
|
||
"inference_base_url": _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL),
|
||
"portal_base_url": getattr(entry, "portal_base_url", None),
|
||
"client_id": getattr(entry, "client_id", None),
|
||
"scope": getattr(entry, "scope", None),
|
||
"token_type": getattr(entry, "token_type", "Bearer"),
|
||
"source": "pool",
|
||
}
|
||
try:
|
||
auth_path = _auth_json_path()
|
||
if not auth_path.is_file():
|
||
return None
|
||
data = json.loads(auth_path.read_text(encoding="utf-8-sig"))
|
||
if data.get("active_provider") != "nous":
|
||
return None
|
||
provider = data.get("providers", {}).get("nous", {})
|
||
# Must have at least an access_token or agent_key.
|
||
if not provider.get("agent_key") and not provider.get("access_token"):
|
||
return None
|
||
return provider
|
||
except Exception as exc:
|
||
logger.debug("Could not read Nous auth: %s", exc)
|
||
return None
|
||
|
||
|
||
def _nous_api_key(provider: dict) -> str:
|
||
"""Extract a usable Nous inference JWT from stored auth state."""
|
||
from hermes_cli.auth import _nous_invoke_jwt_is_usable
|
||
for token_key, expiry_key in (("agent_key", "agent_key_expires_at"), ("access_token", "expires_at")):
|
||
token = provider.get(token_key)
|
||
if not isinstance(token, str) or not token.strip():
|
||
continue
|
||
if _nous_invoke_jwt_is_usable(token, scope=provider.get("scope"), expires_at=provider.get(expiry_key)):
|
||
return token
|
||
return ""
|
||
|
||
|
||
def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]:
|
||
"""Resolve Nous auxiliary credentials from the selected pool entry."""
|
||
try:
|
||
from hermes_cli.auth import _agent_key_is_usable
|
||
pool = load_pool("nous")
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary Nous pool credential resolution failed: %s", exc)
|
||
return None
|
||
if not pool or not pool.has_credentials():
|
||
return None
|
||
try:
|
||
entry = pool.select()
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary Nous pool selection failed: %s", exc)
|
||
return None
|
||
if entry is None:
|
||
return None
|
||
|
||
def _entry_state(e: Any) -> Dict[str, Any]:
|
||
return {k: getattr(e, k, None) for k in (
|
||
"agent_key", "agent_key_expires_at", "access_token", "expires_at", "scope")}
|
||
|
||
if force_refresh or not _agent_key_is_usable(_entry_state(entry), _nous_min_key_ttl_seconds()):
|
||
try:
|
||
refreshed = pool.try_refresh_current()
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary Nous pool refresh failed: %s", exc)
|
||
refreshed = None
|
||
if refreshed is None:
|
||
return None
|
||
entry = refreshed
|
||
api_key = _nous_api_key(_entry_state(entry))
|
||
base_url = _pool_runtime_base_url(entry, _NOUS_DEFAULT_BASE_URL)
|
||
if not api_key or not base_url:
|
||
return None
|
||
return api_key, base_url
|
||
|
||
|
||
def _resolve_nous_runtime_api(
|
||
*, force_refresh: bool = False, stale_access_token: Optional[str] = None
|
||
) -> Optional[tuple[str, str]]:
|
||
"""Fresh Nous runtime credentials (pool first, then auth store + JWT refresh) — mirrors the main
|
||
agent's 401 recovery. ``stale_access_token`` is the bearer that just 401'd; with ``force_refresh``
|
||
it lets the auth store adopt a sibling process's rotation instead of re-POSTing the shared grant."""
|
||
pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh)
|
||
if pooled is not None:
|
||
return pooled
|
||
try:
|
||
from hermes_cli.auth import resolve_nous_runtime_credentials
|
||
creds = resolve_nous_runtime_credentials(
|
||
timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15),
|
||
force_refresh=force_refresh,
|
||
stale_access_token=stale_access_token or None,
|
||
)
|
||
except Exception as exc:
|
||
# Kept at WARNING (once per message) and remembered: the ladder falls back silently, and
|
||
# without this the goal judge only ever saw "judge error: RuntimeError" (#42177).
|
||
record_nous_credential_failure(exc)
|
||
return None
|
||
clear_nous_credential_failure()
|
||
return _creds_pair(creds)
|
||
|
||
|
||
def _creds_pair(creds: Dict[str, Any]) -> Optional[Tuple[str, str]]:
|
||
"""``(api_key, base_url)`` from a runtime-credentials dict, or None when either is missing."""
|
||
api_key = str(creds.get("api_key") or "").strip()
|
||
base_url = str(creds.get("base_url") or "").strip().rstrip("/")
|
||
if not api_key or not base_url:
|
||
return None
|
||
return api_key, base_url
|
||
|
||
|
||
def _resolve_xai_oauth_for_aux() -> Optional[Tuple[str, str]]:
|
||
"""Fresh xAI OAuth (api_key, base_url) for aux clients, or None.
|
||
|
||
Pool first (some xAI OAuth logins exist only as pool entries), then the singleton auth-store resolver.
|
||
"""
|
||
try:
|
||
from hermes_cli.auth import DEFAULT_XAI_OAUTH_BASE_URL, _xai_validate_inference_base_url
|
||
pool = load_pool("xai-oauth")
|
||
if pool and pool.has_credentials():
|
||
entry = pool.select()
|
||
if entry is not None:
|
||
api_key = str(
|
||
getattr(entry, "runtime_api_key", None) or getattr(entry, "access_token", "") or ""
|
||
).strip()
|
||
_url = lambda v: str(v or "").strip().rstrip("/") # noqa: E731
|
||
base_url = _xai_validate_inference_base_url(
|
||
_url(_scoped_key_env("HERMES_XAI_BASE_URL"))
|
||
or _url(_scoped_key_env("XAI_BASE_URL"))
|
||
or _url(getattr(entry, "runtime_base_url", None))
|
||
or _url(getattr(entry, "base_url", None)),
|
||
fallback=DEFAULT_XAI_OAUTH_BASE_URL,
|
||
)
|
||
if api_key and base_url:
|
||
return api_key, base_url
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary xAI OAuth pool credential resolution failed: %s", exc)
|
||
try:
|
||
from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
|
||
creds = resolve_xai_oauth_runtime_credentials()
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary xAI OAuth runtime credential resolution failed: %s", exc)
|
||
return None
|
||
return _creds_pair(creds)
|
||
|
||
|
||
def _resolve_codex_credential_and_base() -> Tuple[Optional[str], str]:
|
||
"""``(token, base_url)`` taken from ONE authority, so a Codex key is only ever sent to the host
|
||
it belongs to (#121486): the profile-scoped ``HERMES_CODEX_BASE_URL`` wins; otherwise a pooled
|
||
key goes where that pool entry routes (row URL / ``model.base_url``) and the auth.json OAuth
|
||
token goes to the ChatGPT default. ``(None, <base>)`` without a usable token."""
|
||
override = _codex_base_url_override()
|
||
pool_present, entry = _select_pool_entry("openai-codex")
|
||
if pool_present:
|
||
token = _pool_runtime_api_key(entry)
|
||
if token:
|
||
if override:
|
||
return token, override
|
||
# Same route rule as the chat path (row URL, else ``model.base_url``); never empty.
|
||
from hermes_cli.auth_codex import _codex_pool_route_base_url
|
||
return token, _codex_pool_route_base_url(_pool_runtime_base_url(entry))
|
||
# No usable pool token: auth.json only (re-selecting could pair another row's key with the default).
|
||
return _read_codex_singleton_token(), override or _CODEX_AUX_BASE_URL
|
||
|
||
|
||
def _read_codex_singleton_token() -> Optional[str]:
|
||
"""The profile's auth.json Codex access token (expired JWTs skipped), else None."""
|
||
try:
|
||
from hermes_cli.auth import _read_codex_tokens
|
||
access_token = _read_codex_tokens().get("tokens", {}).get("access_token")
|
||
if not isinstance(access_token, str) or not access_token.strip():
|
||
return None
|
||
# Expired JWTs would block the auto chain and prevent fallback to working providers.
|
||
try:
|
||
import base64
|
||
payload = access_token.split(".")[1]
|
||
payload += "=" * (-len(payload) % 4)
|
||
exp = json.loads(base64.urlsafe_b64decode(payload)).get("exp", 0)
|
||
if exp and time.time() > exp:
|
||
logger.debug("Codex access token expired (exp=%s), skipping", exp)
|
||
return None
|
||
except Exception:
|
||
pass # Non-JWT token or decode error — use as-is
|
||
return access_token.strip()
|
||
except Exception as exc:
|
||
logger.debug("Could not read Codex auth for auxiliary client: %s", exc)
|
||
return None
|
||
|
||
|
||
def _resolve_api_key_provider() -> Tuple[Optional[OpenAI], Optional[str]]:
|
||
"""Try each API-key provider in PROVIDER_REGISTRY order; (client, model) or (None, None)."""
|
||
try:
|
||
from hermes_cli.auth import PROVIDER_REGISTRY, resolve_api_key_provider_credentials
|
||
except ImportError:
|
||
logger.debug("Could not import PROVIDER_REGISTRY for API-key fallback")
|
||
return None, None
|
||
for provider_id, pconfig in PROVIDER_REGISTRY.items():
|
||
if pconfig.auth_type != "api_key":
|
||
continue
|
||
if _is_provider_unhealthy(provider_id):
|
||
logger.debug("Auxiliary api-key chain: %s is unhealthy, skipping", provider_id)
|
||
continue
|
||
if provider_id == "anthropic":
|
||
# Explicit-config gate: Claude Code credentials must not silently become aux fallback.
|
||
with contextlib.suppress(ImportError):
|
||
from hermes_cli.auth import is_provider_explicitly_configured
|
||
if not is_provider_explicitly_configured("anthropic"):
|
||
continue
|
||
return _try_anthropic()
|
||
if provider_id == "copilot":
|
||
# Explicit-config gate: ambient gh-CLI credentials must not silently become aux fallback (#114740).
|
||
with contextlib.suppress(ImportError):
|
||
from hermes_cli.auth import is_provider_explicitly_configured
|
||
if not is_provider_explicitly_configured("copilot"):
|
||
continue
|
||
model = _get_aux_model_for_provider(provider_id) or None
|
||
if model is None:
|
||
continue # skip provider if we don't know a valid aux model
|
||
pool_present, entry = _select_pool_entry(provider_id)
|
||
if pool_present:
|
||
api_key = _pool_runtime_api_key(entry)
|
||
if not api_key:
|
||
continue
|
||
raw_base_url = _pool_runtime_base_url(entry, pconfig.inference_base_url) or pconfig.inference_base_url
|
||
via = " via pool"
|
||
else:
|
||
creds = resolve_api_key_provider_credentials(provider_id)
|
||
api_key = str(creds.get("api_key", "")).strip()
|
||
if not api_key:
|
||
continue
|
||
raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
|
||
via = ""
|
||
# The session's own endpoint wins for its provider: the key was issued for that gateway, and
|
||
# sending it to the registry default 401s, then quarantines the provider the main model is on.
|
||
runtime = _normalize_main_runtime(None)
|
||
if runtime.get("provider") == provider_id and runtime.get("base_url"):
|
||
raw_base_url = runtime["base_url"].rstrip("/")
|
||
if isinstance(runtime.get("api_key"), str) and runtime["api_key"]:
|
||
api_key = runtime["api_key"]
|
||
via = " (session endpoint)"
|
||
logger.debug("Auxiliary text client: %s (%s)%s", pconfig.name, model, via)
|
||
# Native Gemini, else OpenAI-wire + Anthropic rewrap.
|
||
base_url = _to_openai_base_url(raw_base_url)
|
||
if provider_id == "gemini":
|
||
from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
|
||
if is_native_gemini_base_url(base_url):
|
||
return GeminiNativeClient(api_key=api_key, base_url=base_url), model
|
||
if base_url_host_matches(base_url, "api.kimi.com"):
|
||
headers = {"User-Agent": "claude-code/0.1.0"}
|
||
elif base_url_host_matches(base_url, "githubcopilot.com"):
|
||
from hermes_cli.models import copilot_default_headers
|
||
headers = copilot_default_headers()
|
||
elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
|
||
headers = build_nvidia_nim_headers(base_url)
|
||
else:
|
||
headers = _profile_default_headers(provider_id)
|
||
extra = {"default_headers": headers} if headers else {}
|
||
merged = _apply_user_default_headers(extra.get("default_headers"))
|
||
if merged:
|
||
extra["default_headers"] = merged
|
||
client = _create_openai_client(api_key=api_key, base_url=base_url, **extra)
|
||
return _maybe_wrap_anthropic(client, model, api_key, raw_base_url), model
|
||
return None, None
|
||
|
||
|
||
def _endpoint_default_headers(
|
||
base_url: str, provider: str, *, is_vision: bool = False, xai: bool = False,
|
||
) -> Optional[dict]:
|
||
"""Provider-specific client headers by endpoint host, merged with user ``model.default_headers``.
|
||
|
||
Kimi Code needs the claude-code User-Agent; Copilot needs its request headers
|
||
(``is_vision`` adds Copilot-Vision-Request); NVIDIA NIM and (optionally) xAI have
|
||
their own fingerprints; anything else falls back to the provider profile.
|
||
"""
|
||
if base_url_host_matches(base_url, "api.kimi.com"):
|
||
headers: dict = {"User-Agent": "claude-code/0.1.0"}
|
||
elif base_url_host_matches(base_url, "githubcopilot.com"):
|
||
from hermes_cli.copilot_auth import copilot_request_headers
|
||
headers = dict(copilot_request_headers(is_agent_turn=True, is_vision=is_vision))
|
||
elif base_url_host_matches(base_url, "integrate.api.nvidia.com"):
|
||
headers = dict(build_nvidia_nim_headers(base_url))
|
||
elif xai and base_url_host_matches(base_url, "x.ai"):
|
||
from tools.xai_http import hermes_xai_default_headers
|
||
headers = dict(hermes_xai_default_headers())
|
||
else:
|
||
headers = _profile_default_headers(provider) or {}
|
||
return _apply_user_default_headers(headers or None) or None
|
||
|
||
|
||
def _profile_default_headers(provider: str) -> Optional[dict]:
|
||
"""Client-level attribution headers from the provider profile (e.g. GMI User-Agent), or None."""
|
||
if not provider:
|
||
return None
|
||
with contextlib.suppress(Exception):
|
||
from providers import get_provider_profile
|
||
profile = get_provider_profile(provider)
|
||
if profile and profile.default_headers:
|
||
return dict(profile.default_headers)
|
||
return None
|
||
|
||
|
||
# Provider resolution helpers
|
||
|
||
_paid_lane_warned: set = set()
|
||
|
||
|
||
def _is_free_model(model: Optional[str]) -> bool:
|
||
"""True when ``model`` is a free SKU (``:free`` suffix or ``stealth/`` prefix) — naming-convention trust."""
|
||
if not model:
|
||
return False
|
||
normalized = str(model).strip()
|
||
return normalized.endswith(":free") or normalized.startswith("stealth/")
|
||
|
||
|
||
def _aux_openrouter_settings() -> Tuple[bool, str]:
|
||
"""Read (free_only, openrouter_model) from config; (False, _OPENROUTER_MODEL) on failure."""
|
||
try:
|
||
from hermes_cli.config import cfg_get, load_config_readonly
|
||
cfg = load_config_readonly()
|
||
free_only = bool(cfg_get(cfg, "auxiliary", "free_only", default=False))
|
||
val = cfg_get(cfg, "auxiliary", "openrouter_model")
|
||
model = val.strip() if isinstance(val, str) and val.strip() else _OPENROUTER_MODEL
|
||
return free_only, model
|
||
except Exception:
|
||
return False, _OPENROUTER_MODEL
|
||
|
||
|
||
def _warn_paid_lane_once(model: str) -> None:
|
||
"""Log a WARNING the first time a non-free OpenRouter model is engaged."""
|
||
if model in _paid_lane_warned:
|
||
return
|
||
_paid_lane_warned.add(model)
|
||
logger.warning(
|
||
"Auxiliary client: PAID lane engaged for auxiliary task — OpenRouter fallback model %r is not "
|
||
"a :free SKU and may incur real spend. Set auxiliary.free_only: true to restrict auxiliary "
|
||
"fallbacks to free models, or auxiliary.openrouter_model to a :free model.", model,
|
||
)
|
||
|
||
|
||
def _try_openrouter(explicit_api_key: Optional[Union[str, Callable[[], str]]] = None, model: str = None,
|
||
explicit_base_url: Optional[str] = None) -> Tuple[Optional[OpenAI], Optional[str]]:
|
||
free_only, cfg_model = _aux_openrouter_settings()
|
||
or_model = model or cfg_model
|
||
if free_only and not _is_free_model(or_model):
|
||
logger.warning(
|
||
"Auxiliary client: auxiliary.free_only is enabled but the OpenRouter fallback model %r is "
|
||
"not a :free SKU — skipping the OpenRouter fallback. Set auxiliary.openrouter_model to a "
|
||
":free model (e.g. nvidia/nemotron-3-ultra-550b-a55b:free) or disable auxiliary.free_only.",
|
||
or_model,
|
||
)
|
||
return None, None
|
||
if not _is_free_model(or_model):
|
||
_warn_paid_lane_once(or_model)
|
||
# A caller-supplied endpoint (fallback_providers entry, custom_providers entry) is
|
||
# authoritative over both the pool row and the canonical host (#121359).
|
||
override_url = (explicit_base_url or "").strip().rstrip("/")
|
||
pool_present, entry = _select_pool_entry("openrouter")
|
||
if pool_present:
|
||
or_key = explicit_api_key or _pool_runtime_api_key(entry)
|
||
if or_key:
|
||
base_url = override_url or _pool_runtime_base_url(entry, OPENROUTER_BASE_URL) or OPENROUTER_BASE_URL
|
||
logger.debug("Auxiliary client: OpenRouter via pool")
|
||
return _create_openai_client(
|
||
api_key=or_key, base_url=base_url, default_headers=build_or_headers()
|
||
), or_model
|
||
# Exhausted pool: fall through to OPENROUTER_API_KEY rather than fail.
|
||
logger.debug("Auxiliary client: OpenRouter pool exhausted, trying OPENROUTER_API_KEY")
|
||
or_key = explicit_api_key or _scoped_key_env("OPENROUTER_API_KEY")
|
||
if not or_key:
|
||
_mark_provider_unhealthy(
|
||
"openrouter", ttl=60, reason=_describe_openrouter_unavailable(or_model), level=logging.DEBUG)
|
||
return None, None
|
||
logger.debug("Auxiliary client: OpenRouter")
|
||
return _create_openai_client(
|
||
api_key=or_key, base_url=override_url or OPENROUTER_BASE_URL, default_headers=build_or_headers()
|
||
), or_model
|
||
|
||
|
||
def _describe_openrouter_unavailable(model: str = None) -> str:
|
||
"""Return the policy or credential reason OpenRouter was unavailable."""
|
||
free_only, cfg_model = _aux_openrouter_settings()
|
||
or_model = model or cfg_model
|
||
if free_only and not _is_free_model(or_model):
|
||
return (
|
||
f"auxiliary.free_only rejected non-free model {or_model!r}; "
|
||
"the request was skipped before provider availability checks"
|
||
)
|
||
pool_present, entry = _select_pool_entry("openrouter")
|
||
if pool_present:
|
||
if entry is None:
|
||
return "OpenRouter credential pool has no usable entries (credentials may be exhausted)"
|
||
if not _pool_runtime_api_key(entry):
|
||
return "OpenRouter credential pool entry is missing a runtime API key"
|
||
if not _scoped_key_env("OPENROUTER_API_KEY"):
|
||
return "OPENROUTER_API_KEY not set"
|
||
return "no usable OpenRouter credentials found"
|
||
|
||
|
||
def _try_nous(vision: bool = False) -> Tuple[Optional[OpenAI], Optional[str]]:
|
||
nous = _read_nous_auth()
|
||
runtime = _resolve_nous_runtime_api(force_refresh=False)
|
||
if runtime is None and not nous:
|
||
logger.warning("Auxiliary Nous client unavailable: no Nous authentication found (run: hermes auth).")
|
||
_mark_provider_unhealthy("nous", ttl=60, reason="no Nous authentication found", level=logging.DEBUG)
|
||
return None, None
|
||
if runtime is None and nous:
|
||
logger.debug("Auxiliary Nous: runtime JWT refresh failed; checking stored auth.json token.")
|
||
if runtime is not None:
|
||
api_key, base_url = runtime
|
||
else:
|
||
api_key = _nous_api_key(nous or {})
|
||
if not api_key:
|
||
logger.warning(
|
||
"Auxiliary Nous client unavailable: no usable inference JWT found "
|
||
"(run: hermes auth add nous)."
|
||
)
|
||
_mark_provider_unhealthy("nous", ttl=60, reason="no usable Nous inference JWT", level=logging.DEBUG)
|
||
return None, None
|
||
base_url = str(
|
||
(nous or {}).get("inference_base_url") or _scoped_key_env("NOUS_INFERENCE_BASE_URL") or _NOUS_DEFAULT_BASE_URL
|
||
).rstrip("/")
|
||
with contextlib.suppress(Exception):
|
||
from agent.nous_rate_guard import nous_rate_limit_remaining
|
||
from hermes_cli.anon_auth import is_anonymous_request
|
||
anonymous = is_anonymous_request("nous", api_key)
|
||
remaining = nous_rate_limit_remaining(anonymous=anonymous)
|
||
if remaining is not None and remaining > 0:
|
||
logger.debug("Auxiliary: skipping Nous Portal (rate-limited, resets in %.0fs)", remaining)
|
||
# The health marker is provider-wide, so a full-length anonymous cooldown would
|
||
# outlive signing in mid-cooldown; bound it instead of re-resolving credentials
|
||
# (auth store lock, pool read) on every auxiliary call for the cooldown's duration.
|
||
_mark_provider_unhealthy(
|
||
"nous", ttl=min(remaining, 60.0) if anonymous else remaining,
|
||
reason="Nous Portal rate-limited", level=logging.INFO)
|
||
return None, None
|
||
lane = "vision" if vision else "text"
|
||
# The free tier's host serves exactly one model, for every lane: asking it for the Portal's
|
||
# recommended aux model is a guaranteed 429 ``model_not_free``. Pin the route's model instead.
|
||
# Vision rides the same id (the backing model is multimodal; a backing that is not answers
|
||
# the request with the upstream's own error, which the ladder handles like any other).
|
||
from hermes_cli.anon_auth import GUEST_MODEL, route_is_welcome_host
|
||
global auxiliary_is_nous
|
||
if route_is_welcome_host(base_url):
|
||
auxiliary_is_nous = True
|
||
logger.debug("Auxiliary/%s: Nous free tier; using %s", lane, GUEST_MODEL)
|
||
return _create_openai_client(api_key=api_key, base_url=base_url), GUEST_MODEL
|
||
auxiliary_is_nous = True
|
||
logger.debug("Auxiliary client: Nous Portal")
|
||
# Portal recommended-models is authoritative (tier-aware); _NOUS_MODEL when unreachable/null.
|
||
# Probes skip the lookup: exact model is irrelevant and it hits the network.
|
||
model = _NOUS_MODEL
|
||
if not _aux_probe_active():
|
||
try:
|
||
from hermes_cli.models import get_nous_recommended_aux_model
|
||
recommended = get_nous_recommended_aux_model(vision=vision)
|
||
if recommended:
|
||
model = recommended
|
||
logger.debug("Auxiliary/%s: using Portal-recommended model %s", lane, model)
|
||
else:
|
||
logger.debug("Auxiliary/%s: no Portal recommendation, falling back to %s", lane, model)
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"Auxiliary/%s: recommended-models lookup failed (%s); "
|
||
"falling back to %s",
|
||
lane, exc, model,
|
||
)
|
||
return _create_openai_client(api_key=api_key, base_url=base_url), model
|
||
|
||
|
||
def _refresh_nous_recommended_model(*, vision: bool, stale_model: Optional[str]) -> Optional[str]:
|
||
"""Fresh Portal recommended model after a stale-model 404 (long-lived processes pin dropped models).
|
||
|
||
Returns the fresh recommendation, else ``_NOUS_MODEL``, whichever differs from ``stale_model``; None if neither.
|
||
"""
|
||
stale = (stale_model or "").strip().lower()
|
||
fresh: Optional[str] = None
|
||
try:
|
||
from hermes_cli.models import get_nous_recommended_aux_model
|
||
fresh = get_nous_recommended_aux_model(vision=vision, force_refresh=True)
|
||
except Exception as exc:
|
||
logger.debug("Nous recommended-model refresh failed (%s); using default %s", exc, _NOUS_MODEL)
|
||
if fresh and fresh.strip().lower() != stale:
|
||
return fresh
|
||
return _NOUS_MODEL if _NOUS_MODEL.strip().lower() != stale else None
|
||
|
||
|
||
def _read_main_field(field: str, *, readonly: bool, lower: bool = False) -> str:
|
||
"""Main ``model.<field>``: runtime override (``set_runtime_main``) first, then config.yaml.
|
||
|
||
The override wins so "active main model" gates see the live CLI/gateway runtime, not the persisted
|
||
default. ``readonly`` picks ``load_config_readonly`` (model/provider) vs ``load_config`` (api_key/base_url).
|
||
"""
|
||
override = _runtime_main_value(field)
|
||
if isinstance(override, str) and override.strip():
|
||
value = override.strip()
|
||
return value.lower() if lower else value
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli import config as _cfg_mod
|
||
cfg = (_cfg_mod.load_config_readonly if readonly else _cfg_mod.load_config)()
|
||
model_cfg = cfg.get("model", {})
|
||
if field == "model" and isinstance(model_cfg, str) and model_cfg.strip():
|
||
return model_cfg.strip()
|
||
if isinstance(model_cfg, dict):
|
||
value = model_cfg.get("default" if field == "model" else field, "")
|
||
if isinstance(value, str) and value.strip():
|
||
value = value.strip()
|
||
return value.lower() if lower else value
|
||
return ""
|
||
|
||
|
||
# Module-level callables (tests patch them): model/provider (lowercased) read the readonly config;
|
||
# api_key/base_url read the full config so ``custom`` aux tasks can inherit main creds.
|
||
_read_main_model = functools.partial(_read_main_field, "model", readonly=True)
|
||
_read_main_provider = functools.partial(_read_main_field, "provider", readonly=True, lower=True)
|
||
_read_main_api_key = functools.partial(_read_main_field, "api_key", readonly=False)
|
||
_read_main_base_url = functools.partial(_read_main_field, "base_url", readonly=False)
|
||
|
||
|
||
def _resolve_moa_aggregator(preset_name: Optional[str]) -> Tuple[Optional[str], Optional[str]]:
|
||
"""MoA preset → aggregator (provider, model); (None, None) if unresolvable. None/"" = default preset.
|
||
|
||
"moa" is virtual — aux tasks skip the fan-out and use the aggregator slot; shared so lookup can't drift.
|
||
"""
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
from hermes_cli.moa_config import resolve_moa_preset
|
||
preset = resolve_moa_preset(load_config().get("moa") or {}, preset_name or None)
|
||
agg = preset.get("aggregator") or {}
|
||
agg_provider = str(agg.get("provider") or "").strip()
|
||
agg_model = str(agg.get("model") or "").strip()
|
||
if agg_provider and agg_model and agg_provider.lower() != "moa":
|
||
return agg_provider, agg_model
|
||
except Exception:
|
||
logger.debug("MoA aggregator resolution failed for preset %r", preset_name, exc_info=True)
|
||
return None, None
|
||
|
||
|
||
def _read_main_model_for_aux() -> str:
|
||
"""Main model with MoA presets unwrapped to the aggregator's model; "" when unresolvable (a preset name would 400)."""
|
||
model = _read_main_model()
|
||
if (_read_main_provider() or "").strip().lower() == "moa":
|
||
_, agg_model = _resolve_moa_aggregator(model)
|
||
return agg_model or ""
|
||
return model
|
||
|
||
|
||
def _read_main_api_key_if_same_host(aux_base_url: str) -> str:
|
||
"""Main api_key only when *aux_base_url* shares the main base_url's host.
|
||
|
||
Unconditional inheritance would leak the credential to any misconfigured host; mismatch keeps ``no-key-required`` → 401.
|
||
"""
|
||
aux_host = base_url_hostname(aux_base_url)
|
||
if not aux_host or aux_host != base_url_hostname(_read_main_base_url()):
|
||
return ""
|
||
return _read_main_api_key()
|
||
|
||
|
||
# Compatibility mirrors for older readers/tests; the ContextVar below is
|
||
# authoritative (overlapping gateway sessions make a process-global unsafe).
|
||
_RUNTIME_MAIN_PROVIDER: str = ""
|
||
_RUNTIME_MAIN_MODEL: str = ""
|
||
_RUNTIME_MAIN_BASE_URL: str = ""
|
||
_RUNTIME_MAIN_API_KEY: Any = ""
|
||
_RUNTIME_MAIN_API_MODE: str = ""
|
||
_RUNTIME_MAIN_AUTH_MODE: str = ""
|
||
_RUNTIME_MAIN_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = (
|
||
contextvars.ContextVar("auxiliary_runtime_main", default=None)
|
||
)
|
||
|
||
_RELAY_AUX_CALL_CONTEXT: contextvars.ContextVar[Optional[Dict[str, Any]]] = (
|
||
contextvars.ContextVar("auxiliary_relay_call", default=None)
|
||
)
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def _relay_aux_call_scope(args: tuple, kwargs: dict):
|
||
"""Bind a fresh relay call context for one auxiliary call; mark it failed on any exception."""
|
||
task = args[0] if args else kwargs.get("task")
|
||
token = _RELAY_AUX_CALL_CONTEXT.set({
|
||
"task": str(task or "unknown"),
|
||
"request_id": f"aux-{uuid.uuid4().hex}",
|
||
"attempt_count": 0,
|
||
"provider": "",
|
||
"model": "",
|
||
"response_model": None,
|
||
"api_mode": "chat_completions",
|
||
})
|
||
try:
|
||
yield
|
||
except BaseException:
|
||
_fail_relay_auxiliary_call()
|
||
raise
|
||
finally:
|
||
_RELAY_AUX_CALL_CONTEXT.reset(token)
|
||
|
||
|
||
def _relay_auxiliary_call(callback):
|
||
"""Give every physical retry in one auxiliary call a shared Relay identity."""
|
||
@functools.wraps(callback)
|
||
def wrapped(*args, **kwargs):
|
||
with _relay_aux_call_scope(args, kwargs):
|
||
return callback(*args, **kwargs)
|
||
return wrapped
|
||
|
||
|
||
def _relay_auxiliary_call_async(callback):
|
||
"""Async counterpart to :func:`_relay_auxiliary_call`."""
|
||
@functools.wraps(callback)
|
||
async def wrapped(*args, **kwargs):
|
||
with _relay_aux_call_scope(args, kwargs):
|
||
return await callback(*args, **kwargs)
|
||
return wrapped
|
||
|
||
|
||
def _set_relay_auxiliary_route(provider: str | None, model: str | None, api_mode: str | None) -> None:
|
||
context = _RELAY_AUX_CALL_CONTEXT.get()
|
||
if context is None:
|
||
return
|
||
context["provider"] = str(provider or "auxiliary")
|
||
context["model"] = str(model or "unknown")
|
||
context["response_model"] = None
|
||
context["api_mode"] = str(api_mode or "chat_completions")
|
||
|
||
|
||
def _record_route_info(
|
||
route_info: Optional[Dict[str, str]], provider: Optional[str], model: Optional[str]
|
||
) -> None:
|
||
"""Expose the concrete route selected for one auxiliary call."""
|
||
if route_info is not None:
|
||
route_info["provider"] = provider or "auto"
|
||
route_info["model"] = model or "default"
|
||
|
||
|
||
def _relay_auxiliary_metadata(
|
||
*, provider: str | None = None, api_mode: str | None = None
|
||
) -> tuple[str, str, dict[str, Any]] | None:
|
||
context = _RELAY_AUX_CALL_CONTEXT.get()
|
||
if context is None:
|
||
return None
|
||
attempt_count = int(context.get("attempt_count") or 0)
|
||
context["attempt_count"] = attempt_count + 1
|
||
provider_name = str(provider or context.get("provider") or "auxiliary")
|
||
model_name = str(context.get("model") or "unknown")
|
||
return provider_name, model_name, {
|
||
"api_mode": str(api_mode or context.get("api_mode") or "chat_completions"),
|
||
"api_request_id": str(context["request_id"]),
|
||
"call_role": f"auxiliary:{context['task']}",
|
||
"retry_count": attempt_count,
|
||
"auxiliary_task": str(context["task"]),
|
||
}
|
||
|
||
|
||
def _relay_sync_completion(
|
||
client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
|
||
api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
|
||
) -> Any:
|
||
from agent.auxiliary_wire import prepare_chat_messages
|
||
|
||
kwargs = prepare_chat_messages(client, kwargs)
|
||
# The progress hook is installed per TASK, so every attempt (retries, recovery rungs, fallbacks)
|
||
# must stream through _create_with_progress or the compression watchdog sees silence (#98466).
|
||
callback = create or (lambda request: _create_with_progress(client, request))
|
||
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
|
||
# Isolate only the provider callback so the owning thread can unwind its lease/DB
|
||
# transaction on hard cancel without touching the shared client.
|
||
if route is None:
|
||
return _run_protected_sync_provider_call(callback, kwargs)
|
||
provider_name, fallback_model, metadata = route
|
||
from agent import relay_llm
|
||
from agent.auxiliary_hooks import run_with_aux_hooks
|
||
model_name = str(kwargs.get("model") or fallback_model)
|
||
return run_with_aux_hooks(
|
||
lambda: relay_llm.execute_current(
|
||
kwargs, lambda request: _run_protected_sync_provider_call(callback, request),
|
||
name=provider_name, model_name=model_name, metadata=metadata,
|
||
defer_logical_completion=True,
|
||
),
|
||
aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
|
||
provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
|
||
)
|
||
|
||
|
||
async def _relay_async_completion(
|
||
client: Any, kwargs: dict[str, Any], *, provider: str | None = None,
|
||
api_mode: str | None = None, create: Callable[[dict[str, Any]], Any] | None = None,
|
||
) -> Any:
|
||
from agent.auxiliary_wire import prepare_chat_messages
|
||
|
||
kwargs = prepare_chat_messages(client, kwargs)
|
||
# Async twin of the seam default above (#98466).
|
||
callback = create or (lambda request: _acreate_with_progress(client, request))
|
||
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
|
||
if route is None:
|
||
return await callback(kwargs)
|
||
provider_name, fallback_model, metadata = route
|
||
from agent import relay_llm
|
||
from agent.auxiliary_hooks import arun_with_aux_hooks
|
||
model_name = str(kwargs.get("model") or fallback_model)
|
||
return await arun_with_aux_hooks(
|
||
lambda: relay_llm.execute_current_async(
|
||
kwargs, callback, name=provider_name, model_name=model_name,
|
||
metadata=metadata, defer_logical_completion=True,
|
||
),
|
||
aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
|
||
provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""),
|
||
)
|
||
|
||
|
||
def _relay_sync_stream(
|
||
client: Any, kwargs: dict[str, Any], *, provider: str | None = None, api_mode: str | None = None
|
||
) -> Any:
|
||
from agent.auxiliary_wire import prepare_chat_messages
|
||
|
||
kwargs = prepare_chat_messages(client, kwargs)
|
||
# The bypass runs inside the provider callback, AFTER Relay has seen (and possibly
|
||
# rewritten) the real conversation; applying it to `kwargs` would hand Relay an empty one.
|
||
create = lambda request: client.chat.completions.create(**bypass_chat_sdk_request_transform(request, client)) # noqa: E731
|
||
route = _relay_auxiliary_metadata(provider=provider, api_mode=api_mode)
|
||
if route is None:
|
||
return create(kwargs)
|
||
provider_name, fallback_model, metadata = route
|
||
from agent import relay_llm
|
||
from agent.auxiliary_hooks import run_with_aux_hooks
|
||
model_name = str(kwargs.get("model") or fallback_model)
|
||
return run_with_aux_hooks(
|
||
lambda: relay_llm.stream_current(
|
||
kwargs, create, name=provider_name, model_name=model_name, finalizer=dict,
|
||
metadata=metadata, completed_response_predicate=lambda value: hasattr(value, "choices"),
|
||
),
|
||
aux_task=str(metadata.get("auxiliary_task") or ""), metadata=metadata, client=client, kwargs=kwargs,
|
||
provider=provider_name, model=model_name, api_mode=str(metadata.get("api_mode") or ""), streaming=True,
|
||
)
|
||
|
||
|
||
_RUNTIME_MAIN_COMPAT_SNAPSHOT: Tuple[Any, ...] = ("", "", "", "", "", "")
|
||
_RUNTIME_MAIN_COMPAT_LOCK = threading.Lock()
|
||
|
||
|
||
def _publish_runtime_main_mirrors(values: Tuple[Any, ...]) -> None:
|
||
"""Write the legacy globals + compat snapshot (``_MAIN_RUNTIME_FIELDS`` order) under the lock."""
|
||
global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL, _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY
|
||
global _RUNTIME_MAIN_API_MODE, _RUNTIME_MAIN_AUTH_MODE, _RUNTIME_MAIN_COMPAT_SNAPSHOT
|
||
with _RUNTIME_MAIN_COMPAT_LOCK:
|
||
(_RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL, _RUNTIME_MAIN_BASE_URL,
|
||
_RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE, _RUNTIME_MAIN_AUTH_MODE) = values
|
||
_RUNTIME_MAIN_COMPAT_SNAPSHOT = tuple(values)
|
||
|
||
|
||
def _compat_runtime_main() -> Optional[Dict[str, Any]]:
|
||
"""Expose deliberately patched legacy globals as a main context.
|
||
|
||
Mirrors must never become runtime inputs: a direct patch counts only when it differs from
|
||
the mirrored snapshot and only on the main thread.
|
||
"""
|
||
if threading.current_thread() is not threading.main_thread():
|
||
return None
|
||
values = (_RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL, _RUNTIME_MAIN_BASE_URL,
|
||
_RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE, _RUNTIME_MAIN_AUTH_MODE)
|
||
if values == _RUNTIME_MAIN_COMPAT_SNAPSHOT:
|
||
return None
|
||
return dict(zip(_MAIN_RUNTIME_FIELDS, values))
|
||
|
||
|
||
def _runtime_main_value(field: str) -> Any:
|
||
"""Read one runtime field through context-local/controlled legacy state."""
|
||
runtime = _RUNTIME_MAIN_CONTEXT.get()
|
||
if runtime is None:
|
||
runtime = _compat_runtime_main()
|
||
return (runtime.get(field) or "") if isinstance(runtime, dict) else ""
|
||
|
||
|
||
def set_runtime_main(
|
||
provider: str, model: str, *, requested_provider: str = "", base_url: str = "",
|
||
api_key: Any = "", api_mode: str = "", auth_mode: str = "", session_id: str = "",
|
||
cache_scope: str = "",
|
||
) -> contextvars.Token:
|
||
"""Record the current context's live main runtime for auxiliary routing.
|
||
|
||
Context-local so concurrent gateway sessions don't clobber each other; legacy mirrors are
|
||
updated for old readers. ``cache_scope`` is the rotation-stable logical cache scope,
|
||
preferred over ``session_id`` for prompt_cache_key derivation.
|
||
|
||
``cache_scope`` is the rotation-stable logical cache scope (compression- lineage root —
|
||
agent/prompt_cache_scope.py) resolved once per turn by turn_context; auxiliary Responses calls prefer it
|
||
over ``session_id`` for prompt_cache_key derivation (#79017).
|
||
"""
|
||
runtime = {
|
||
"provider": (provider or "").strip().lower(),
|
||
"requested_provider": (requested_provider or "").strip().lower(),
|
||
"model": (model or "").strip(),
|
||
"base_url": (base_url or "").strip(),
|
||
"api_key": _normalize_api_key(api_key),
|
||
"api_mode": (api_mode or "").strip(),
|
||
"auth_mode": (auth_mode or "").strip().lower(),
|
||
"session_id": (session_id or "").strip(),
|
||
"cache_scope": (cache_scope or "").strip(),
|
||
}
|
||
# Publish authoritative context before updating the locked mirrors.
|
||
token = _RUNTIME_MAIN_CONTEXT.set(runtime)
|
||
_publish_runtime_main_mirrors(tuple(runtime[field] for field in _MAIN_RUNTIME_FIELDS))
|
||
return token
|
||
|
||
|
||
def reset_runtime_main(token: contextvars.Token) -> None:
|
||
"""Restore the runtime binding that preceded one scoped turn."""
|
||
if token is None:
|
||
return
|
||
try:
|
||
_RUNTIME_MAIN_CONTEXT.reset(token)
|
||
except (RuntimeError, ValueError):
|
||
pass # Tokens can't be reset from a copied Context (workers inherit values, not token ownership).
|
||
|
||
|
||
@contextlib.contextmanager
|
||
def scoped_runtime_main(main_runtime: Optional[Dict[str, Any]]):
|
||
"""Temporarily bind an explicit runtime without touching legacy mirrors."""
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
token = _RUNTIME_MAIN_CONTEXT.set(runtime or None)
|
||
try:
|
||
yield runtime
|
||
finally:
|
||
_RUNTIME_MAIN_CONTEXT.reset(token)
|
||
|
||
|
||
def clear_runtime_main() -> None:
|
||
"""Clear the runtime override in the current context."""
|
||
_RUNTIME_MAIN_CONTEXT.set(None)
|
||
_publish_runtime_main_mirrors(("", "", "", "", "", ""))
|
||
|
||
|
||
def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[str]]:
|
||
"""Resolve the active custom/main endpoint like the main CLI (env OPENAI_BASE_URL or config-saved)."""
|
||
try:
|
||
from hermes_cli.auth import AuthError
|
||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||
runtime = resolve_runtime_provider(requested="custom")
|
||
except AuthError as exc:
|
||
# Bare 'custom' with nothing configured fails fast in the main resolver: there is no
|
||
# custom endpoint, so do NOT fall through to a stale env OPENAI_BASE_URL that the main
|
||
# resolver deliberately never consults.
|
||
logger.debug("Auxiliary client: no custom endpoint configured: %s", exc)
|
||
return None, None, None
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary client: custom runtime resolution failed: %s", exc)
|
||
runtime = None
|
||
if not isinstance(runtime, dict):
|
||
# Base URL is per-profile like the key one line below (a scoped key must not hit the default's proxy).
|
||
openai_base = _scoped_key_env("OPENAI_BASE_URL").rstrip("/")
|
||
if not openai_base:
|
||
return None, None, None
|
||
runtime = {"base_url": openai_base, "api_key": _scoped_key_env("OPENAI_API_KEY")}
|
||
custom_base = runtime.get("base_url")
|
||
custom_key = runtime.get("api_key")
|
||
custom_mode = runtime.get("api_mode")
|
||
if not isinstance(custom_base, str) or not custom_base.strip():
|
||
return None, None, None
|
||
custom_base = custom_base.strip().rstrip("/")
|
||
if base_url_host_matches(custom_base, "openrouter.ai"):
|
||
return None, None, None # requested='custom' falls back to OpenRouter when unconfigured.
|
||
# Local servers (Ollama, vLLM, ...) ignore auth but the SDK needs a non-empty key.
|
||
# Use a placeholder key — the OpenAI SDK requires a non-empty string but local servers ignore the
|
||
# Authorization header. Same fix as cli.py _ensure_runtime_credentials() (PR #2556).
|
||
if not isinstance(custom_key, str) or not custom_key.strip():
|
||
custom_key = "no-key-required"
|
||
if not isinstance(custom_mode, str) or not custom_mode.strip():
|
||
custom_mode = None
|
||
return custom_base, custom_key.strip(), custom_mode
|
||
|
||
|
||
def _current_custom_base_url() -> str:
|
||
custom_base, _, _ = _resolve_custom_runtime()
|
||
return custom_base or ""
|
||
|
||
|
||
def _validate_proxy_env_urls() -> None:
|
||
"""Fail fast on malformed proxy env URLs (a shell typo like ``:6153export`` otherwise surfaces as a cryptic httpx ``Invalid port``)."""
|
||
from urllib.parse import urlparse
|
||
normalize_proxy_env_vars()
|
||
for key in ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "https_proxy", "http_proxy", "all_proxy"):
|
||
value = str(os.environ.get(key) or "").strip()
|
||
if not value:
|
||
continue
|
||
try:
|
||
parsed = urlparse(value)
|
||
if parsed.scheme:
|
||
_ = parsed.port # raises ValueError for e.g. '6153export'
|
||
except ValueError as exc:
|
||
raise RuntimeError(
|
||
f"Malformed proxy environment variable {key}={value!r}. "
|
||
"Fix or unset your proxy settings and try again."
|
||
) from exc
|
||
|
||
|
||
def _validate_base_url(base_url: str) -> None:
|
||
"""Reject obviously broken custom endpoint URLs before they reach httpx."""
|
||
from urllib.parse import urlparse
|
||
candidate = str(base_url or "").strip()
|
||
if not candidate or candidate.startswith("acp://"):
|
||
return
|
||
try:
|
||
parsed = urlparse(candidate)
|
||
if parsed.scheme in {"http", "https"}:
|
||
_ = parsed.port # raises ValueError for malformed ports
|
||
except ValueError as exc:
|
||
raise RuntimeError(
|
||
f"Malformed custom endpoint URL: {candidate!r}. "
|
||
"Run `hermes setup` or `hermes model` and enter a valid http(s) base URL."
|
||
) from exc
|
||
|
||
|
||
def _try_custom_endpoint() -> Tuple[Optional[Any], Optional[str]]:
|
||
runtime = _resolve_custom_runtime()
|
||
custom_base, custom_key, custom_mode = (*runtime, None) if len(runtime) == 2 else runtime
|
||
if not custom_base or not custom_key:
|
||
return None, None
|
||
if custom_base.lower().startswith(_CODEX_AUX_BASE_URL.lower()):
|
||
return None, None
|
||
model = _read_main_model_for_aux() or "gpt-4o-mini"
|
||
logger.debug("Auxiliary client: custom endpoint (%s, api_mode=%s)", model, custom_mode or "chat_completions")
|
||
_clean_base, _dq = _extract_url_query_params(custom_base)
|
||
_extra = {"default_query": _dq} if _dq else {}
|
||
# User model.default_headers override SDK fingerprint headers (as on the main client) for strict gateways/WAFs.
|
||
_custom_headers = _apply_user_default_headers(None)
|
||
if _custom_headers:
|
||
_extra["default_headers"] = _custom_headers
|
||
if custom_mode == "codex_responses":
|
||
real_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
|
||
return CodexAuxiliaryClient(real_client, model), model
|
||
if custom_mode == "anthropic_messages":
|
||
# OAuth identity only when the host is exactly api.anthropic.com (key_cmd callable included,
|
||
# #114967); third-party Anthropic-compatible gateways never get it.
|
||
try:
|
||
from agent.anthropic_adapter import build_anthropic_client
|
||
from agent.anthropic_credentials import anthropic_route_is_oauth
|
||
real_client = build_anthropic_client(custom_key, custom_base)
|
||
except ImportError:
|
||
logger.warning(
|
||
"Custom endpoint declares api_mode=anthropic_messages but the "
|
||
"anthropic SDK is not installed — falling back to OpenAI-wire."
|
||
)
|
||
return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra), model
|
||
return AnthropicAuxiliaryClient(real_client, model, custom_key, custom_base,
|
||
is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), model
|
||
# URL-based anthropic detection for custom endpoints without explicit api_mode.
|
||
_fallback_client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
|
||
return _maybe_wrap_anthropic(_fallback_client, model, custom_key, custom_base, custom_mode), model
|
||
|
||
|
||
def _build_xai_oauth_aux_client(model: str) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""CodexAuxiliaryClient for xAI Grok OAuth (Responses API); (None, None) if not authed.
|
||
|
||
Caller must pass an explicit model — a pinned Grok default would rot as xAI's allowlist drifts.
|
||
"""
|
||
if not model:
|
||
logger.warning(
|
||
"Auxiliary client: xai-oauth requested without a model; "
|
||
"pass model explicitly (auxiliary.<task>.model in config.yaml)."
|
||
)
|
||
return None, None
|
||
resolved = _resolve_xai_oauth_for_aux()
|
||
if resolved is None:
|
||
return None, None
|
||
api_key, base_url = resolved
|
||
logger.debug("Auxiliary client: xAI OAuth (%s via Responses API)", model)
|
||
from tools.xai_http import hermes_xai_default_headers
|
||
real_client = _create_openai_client(
|
||
api_key=api_key, base_url=base_url, default_headers=hermes_xai_default_headers()
|
||
)
|
||
return CodexAuxiliaryClient(real_client, model), model
|
||
|
||
|
||
def _codex_base_url_override() -> str:
|
||
"""Profile-scoped ``HERMES_CODEX_BASE_URL`` (same read as the API-key env vars: under a
|
||
multiplexer the routed profile's .env decides the endpoint, never a sibling's process env)."""
|
||
return _scoped_key_env("HERMES_CODEX_BASE_URL").rstrip("/")
|
||
|
||
|
||
def _build_codex_client(model: str) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""CodexAuxiliaryClient for an explicit model; (None, None) without a Codex OAuth token.
|
||
|
||
No auto-selected default: the Codex model allow-list is undocumented and drifts.
|
||
"""
|
||
if not model:
|
||
logger.warning(
|
||
"Auxiliary client: openai-codex requested without a model; "
|
||
"pass model explicitly (auxiliary.<task>.model in config.yaml)."
|
||
)
|
||
return None, None
|
||
codex_token, base_url = _resolve_codex_credential_and_base()
|
||
if not codex_token:
|
||
return None, None
|
||
logger.debug("Auxiliary client: Codex OAuth (%s via Responses API)", model)
|
||
real_client = _create_openai_client(
|
||
api_key=codex_token, base_url=base_url,
|
||
default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url),
|
||
)
|
||
return CodexAuxiliaryClient(real_client, model), model
|
||
|
||
|
||
def _try_azure_foundry(
|
||
*, model: Optional[str] = None, explicit_api_key: Optional[str] = None,
|
||
explicit_base_url: Optional[str] = None, api_mode: Optional[str] = None,
|
||
) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Azure Foundry aux client via the main agent's ``_resolve_azure_foundry_runtime`` (api_key vs Entra
|
||
callable bearer, per-model api_mode, base_url overrides). Returns ``(client, model)`` or ``(None, None)``."""
|
||
try:
|
||
from hermes_cli.runtime_provider import _resolve_azure_foundry_runtime
|
||
from hermes_cli.auth import AuthError
|
||
from hermes_cli.config import load_config_readonly
|
||
except ImportError:
|
||
return None, None
|
||
try:
|
||
cfg = load_config_readonly()
|
||
model_cfg = cfg.get("model") if isinstance(cfg, dict) else {}
|
||
if not isinstance(model_cfg, dict):
|
||
model_cfg = {}
|
||
except Exception:
|
||
model_cfg = {}
|
||
try:
|
||
runtime = _resolve_azure_foundry_runtime(
|
||
requested_provider="azure-foundry", model_cfg=model_cfg,
|
||
explicit_api_key=explicit_api_key, explicit_base_url=explicit_base_url,
|
||
target_model=model,
|
||
)
|
||
except AuthError as exc:
|
||
logger.debug("Auxiliary azure-foundry: %s", exc)
|
||
return None, None
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary azure-foundry runtime error: %s", exc)
|
||
return None, None
|
||
api_key = runtime.get("api_key")
|
||
base_url = str(runtime.get("base_url", "") or "")
|
||
runtime_api_mode = api_mode or runtime.get("api_mode") or "chat_completions"
|
||
# api_key may be a callable token provider; bail only on None/"".
|
||
if not (callable(api_key) or api_key) or not base_url:
|
||
return None, None
|
||
final_model = _normalize_resolved_model(model or str(model_cfg.get("default") or ""), "azure-foundry")
|
||
if not final_model:
|
||
# No fallback aux model for Azure (needs a deployment name): let the auto chain fall through instead of 404ing.
|
||
logger.debug(
|
||
"Auxiliary azure-foundry: no model resolved (model=%r, default=%r)",
|
||
model, model_cfg.get("default"),
|
||
)
|
||
return None, None
|
||
# The SDK drops api-version query params from the base URL; pass via default_query.
|
||
_clean_base, _dq = _extract_url_query_params(base_url)
|
||
extra: Dict[str, Any] = {"default_query": _dq} if _dq else {}
|
||
client = _create_openai_client(api_key=api_key, base_url=_clean_base, **extra)
|
||
if runtime_api_mode == "codex_responses":
|
||
return CodexAuxiliaryClient(client, final_model), final_model
|
||
if runtime_api_mode == "anthropic_messages":
|
||
# api_key forwarded verbatim (string or Entra callable; build_anthropic_client installs the bearer hook).
|
||
return _maybe_wrap_anthropic(client, final_model, api_key, base_url, runtime_api_mode), final_model
|
||
return client, final_model
|
||
|
||
|
||
def _try_anthropic(explicit_api_key: Optional[Union[str, Callable[[], str]]] = None,
|
||
explicit_base_url: Optional[str] = None) -> Tuple[Optional[Any], Optional[str]]:
|
||
try:
|
||
from agent.anthropic_adapter import build_anthropic_client
|
||
from agent.anthropic_credentials import resolve_anthropic_token
|
||
except ImportError:
|
||
return None, None
|
||
pool_present, entry = _select_pool_entry("anthropic")
|
||
if pool_present and entry is not None:
|
||
token = explicit_api_key or _pool_runtime_api_key(entry)
|
||
else:
|
||
# Pool absent/empty: legacy resolver so a dead pool entry can't wedge aux tasks when a standalone credential exists.
|
||
entry = None
|
||
token = explicit_api_key or resolve_anthropic_token()
|
||
if not token:
|
||
return None, None
|
||
# Honor config.yaml model.base_url only when provider is anthropic AND the URL is
|
||
# Anthropic-compatible; a foreign host (Codex, OpenRouter) would 401 every aux call.
|
||
base_url = _pool_runtime_base_url(entry, _ANTHROPIC_DEFAULT_BASE_URL) if pool_present else _ANTHROPIC_DEFAULT_BASE_URL
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli.config import load_config_readonly
|
||
cfg = load_config_readonly()
|
||
model_cfg = cfg.get("model")
|
||
if isinstance(model_cfg, dict):
|
||
cfg_provider = str(model_cfg.get("provider") or "").strip().lower()
|
||
if cfg_provider == "anthropic":
|
||
cfg_base_url = (model_cfg.get("base_url") or "").strip().rstrip("/")
|
||
if cfg_base_url and _is_anthropic_compatible_host(cfg_base_url):
|
||
base_url = cfg_base_url
|
||
# A caller-supplied endpoint (fallback_providers entry, custom_providers entry) wins over
|
||
# both the pool row and config.yaml, under the same Anthropic-compatible host rule the
|
||
# primary path applies. A foreign host is REFUSED outright rather than silently demoted to
|
||
# the canonical host: continuing would send the explicit credential to a target the caller
|
||
# did not ask for (#121359).
|
||
override_url = (explicit_base_url or "").strip().rstrip("/")
|
||
if override_url:
|
||
if not _is_anthropic_compatible_host(override_url):
|
||
logger.warning(
|
||
"Auxiliary client: refusing anthropic explicit base_url %r — not an "
|
||
"Anthropic-compatible host; no client built and no request sent.",
|
||
override_url,
|
||
)
|
||
return None, None
|
||
base_url = override_url
|
||
from agent.anthropic_credentials import _is_oauth_token
|
||
is_oauth = _is_oauth_token(token)
|
||
model = _get_aux_model_for_provider("anthropic") or "claude-haiku-4-5-20251001"
|
||
if _aux_probe_active():
|
||
# Probe: token + adapter import resolved; skip real client construction.
|
||
return _AuxProbeClientStub(api_key="", base_url=base_url), model
|
||
logger.debug("Auxiliary client: Anthropic native (%s) at %s (oauth=%s)", model, base_url, is_oauth)
|
||
try:
|
||
real_client = build_anthropic_client(token, base_url)
|
||
except ImportError:
|
||
return None, None # Adapter imports fine but the anthropic SDK itself is missing.
|
||
return AnthropicAuxiliaryClient(real_client, model, token, base_url, is_oauth=is_oauth), model
|
||
|
||
|
||
_MAIN_RUNTIME_FIELDS = ("provider", "model", "base_url", "api_key", "api_mode", "auth_mode")
|
||
_MAIN_RUNTIME_CONTEXT_FIELDS = _MAIN_RUNTIME_FIELDS + (
|
||
"requested_provider", "session_id", "cache_scope",
|
||
)
|
||
|
||
|
||
def _normalize_main_runtime(main_runtime: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||
"""Return a sanitized copy of a live main-runtime override.
|
||
|
||
``api_key`` may be a zero-arg callable (Entra ID token provider, accepted by the OpenAI SDK)
|
||
— preserved as-is so aux clients share main-agent auth.
|
||
"""
|
||
if main_runtime is None:
|
||
# Context-local state first; compat mirrors may hold another concurrent session's endpoint/key.
|
||
main_runtime = _RUNTIME_MAIN_CONTEXT.get()
|
||
if main_runtime is None:
|
||
main_runtime = _compat_runtime_main()
|
||
if not isinstance(main_runtime, dict):
|
||
return {}
|
||
normalized: Dict[str, Any] = {}
|
||
for field in _MAIN_RUNTIME_CONTEXT_FIELDS:
|
||
value = main_runtime.get(field)
|
||
if field == "api_key" and callable(value) and not isinstance(value, str):
|
||
normalized[field] = value
|
||
elif isinstance(value, str) and value.strip():
|
||
normalized[field] = value.strip()
|
||
for identity_field in ("provider", "requested_provider"):
|
||
identity = normalized.get(identity_field)
|
||
if isinstance(identity, str):
|
||
normalized[identity_field] = identity.lower()
|
||
return normalized
|
||
|
||
|
||
def _get_provider_chain() -> List[tuple]:
|
||
"""Ordered provider detection chain, built at call time so ``_try_*`` patches are picked up.
|
||
|
||
``openai-codex`` is deliberately absent (shifting allow-list breaks guessed-model fallback).
|
||
"""
|
||
return [
|
||
("openrouter", _try_openrouter), ("nous", _try_nous),
|
||
("local/custom", _try_custom_endpoint), ("api-key", _resolve_api_key_provider),
|
||
]
|
||
|
||
|
||
# "Recently 402'd" unhealthy-provider cache: a depleted provider stays so for hours, so hiding it
|
||
# for a TTL saves an RTT per aux call. In-process only (profiles may use different keys).
|
||
_AUX_UNHEALTHY_TTL_SECONDS = 600 # 10 minutes
|
||
_aux_unhealthy_until: Dict[Any, float] = {}
|
||
_aux_unhealthy_logged_at: Dict[Any, float] = {}
|
||
_aux_unhealthy_reason: Dict[Any, str] = {}
|
||
# resolved_provider / explicit-config names → chain labels.
|
||
_AUX_UNHEALTHY_LABEL_ALIASES = {
|
||
"openrouter": "openrouter", "nous": "nous", "custom": "local/custom",
|
||
"local/custom": "local/custom", "openai-codex": "openai-codex", "codex": "openai-codex",
|
||
}
|
||
|
||
|
||
def _normalize_chain_label(provider: str) -> str:
|
||
"""resolved_provider → chain label; unknown API-key providers fall back to the lowercased input."""
|
||
if not provider:
|
||
return ""
|
||
p = str(provider).strip().lower()
|
||
return _AUX_UNHEALTHY_LABEL_ALIASES.get(p, p)
|
||
|
||
|
||
_AUX_UNHEALTHY_PAYMENT_REASON = "payment / credit error"
|
||
|
||
|
||
def _mark_provider_unhealthy(
|
||
provider: str, ttl: Optional[float] = None, *, base_url: Optional[str] = None,
|
||
reason: str = _AUX_UNHEALTHY_PAYMENT_REASON, level: int = logging.WARNING,
|
||
) -> None:
|
||
"""Hide one provider endpoint until the TTL expires. ``reason`` is what the log says and what the
|
||
skip line echoes: absent credentials are an expected state (DEBUG), a confirmed 402 is a fault
|
||
(WARNING) — the old fixed payment wording sent local-only users chasing billing (#64144)."""
|
||
label = _normalize_chain_label(provider)
|
||
if not label:
|
||
return
|
||
key = _unhealthy_cache_key(label, base_url)
|
||
ttl = _AUX_UNHEALTHY_TTL_SECONDS if ttl is None else ttl
|
||
expires_at = time.time() + ttl
|
||
_aux_unhealthy_until[key] = expires_at
|
||
_aux_unhealthy_reason[key] = reason
|
||
logger.log(
|
||
level,
|
||
"Auxiliary: marking %s unhealthy for %ds (%s). "
|
||
"Subsequent auxiliary calls will skip it until %s.",
|
||
label, int(ttl), reason, time.strftime("%H:%M:%S", time.localtime(expires_at)),
|
||
)
|
||
|
||
|
||
def _is_provider_unhealthy(label: str, base_url: Optional[str] = None) -> bool:
|
||
"""True iff this provider endpoint is unhealthy and unexpired; lazily evicts expired entries."""
|
||
if not label:
|
||
return False
|
||
key = _unhealthy_cache_key(label, base_url)
|
||
expires_at = _aux_unhealthy_until.get(key)
|
||
if expires_at is None:
|
||
return False
|
||
if time.time() >= expires_at:
|
||
_aux_unhealthy_until.pop(key, None)
|
||
_aux_unhealthy_logged_at.pop(key, None)
|
||
_aux_unhealthy_reason.pop(key, None)
|
||
return False
|
||
return True
|
||
|
||
|
||
def _log_skip_unhealthy(
|
||
label: str, task: Optional[str] = None, *, base_url: Optional[str] = None,
|
||
) -> None:
|
||
"""Log a skipped unhealthy provider at most once per minute per label."""
|
||
now = time.time()
|
||
key = _unhealthy_cache_key(label, base_url)
|
||
if now - _aux_unhealthy_logged_at.get(key, 0.0) >= 60:
|
||
_aux_unhealthy_logged_at[key] = now
|
||
expires_at = _aux_unhealthy_until.get(key, now)
|
||
logger.info(
|
||
"Auxiliary %s: skipping %s (%s, retry in %ds)",
|
||
task or "call", label, _aux_unhealthy_reason.get(key, _AUX_UNHEALTHY_PAYMENT_REASON),
|
||
max(0, int(expires_at - now)),
|
||
)
|
||
|
||
|
||
def _reset_aux_unhealthy_cache() -> None:
|
||
"""Clear the unhealthy cache (tests / explicit user reset)."""
|
||
_aux_unhealthy_until.clear()
|
||
_aux_unhealthy_logged_at.clear()
|
||
_aux_unhealthy_reason.clear()
|
||
|
||
|
||
def _contains_any(text: str, needles: Tuple[str, ...]) -> bool:
|
||
"""True when any needle is a substring of ``text``."""
|
||
return any(kw in text for kw in needles)
|
||
|
||
|
||
# Billing-body markers (credit exhaustion wrapped in 402/403/404/429 bodies). The main classifier's
|
||
# ``_BILLING_PATTERNS`` is the single source (#107166: the two lists had drifted, so OpenRouter's org
|
||
# "Budget limit exceeded" / "Key limit exceeded" 403s were billing for the main loop but not for the aux
|
||
# ladder, and aux fallback never fired); the aux list only adds broader phrasings and daily/weekly quota
|
||
# exhaustion (functionally credit exhaustion; "resource exhausted" is the Vertex/gRPC quota phrasing —
|
||
# also serialized by SDK wrappers and NIM as RESOURCE_EXHAUSTED / ResourceExhausted / resource-exhausted).
|
||
_PAYMENT_KEYWORDS = _BILLING_PATTERNS + (
|
||
"credits", "insufficient funds", "can only afford", "billing",
|
||
"isn't available on the free tier", "key limit exceeded", "budget limit",
|
||
"requires a subscription", "upgrade for access", "upgrade for higher limits",
|
||
"reached your session usage limit", "quota exceeded", "quota_exceeded",
|
||
"too many tokens per day", "daily limit", "tokens per day", "daily quota", "resource exhausted",
|
||
"resource_exhausted", "resource-exhausted", "resourceexhausted",
|
||
"weekly usage limit", "weekly limit",
|
||
)
|
||
|
||
|
||
def _is_payment_error(exc: Exception) -> bool:
|
||
"""Payment/credit/quota exhaustion: HTTP 402, or a billing/quota body on 403/404/429/no-status."""
|
||
status = getattr(exc, "status_code", None)
|
||
return status == 402 or (
|
||
status in {403, 404, 429, None} and _contains_any(str(exc).lower(), _PAYMENT_KEYWORDS)
|
||
)
|
||
|
||
|
||
def _nous_portal_account_has_fresh_paid_access() -> bool:
|
||
"""Return True only when the fresh Nous account API says paid access is allowed."""
|
||
try:
|
||
from hermes_cli.nous_account import get_nous_portal_account_info
|
||
return get_nous_portal_account_info(force_fresh=True).paid_service_access is True
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary Nous paid-entitlement refresh check failed: %s", exc)
|
||
return False
|
||
|
||
|
||
_RATE_LIMIT_KEYWORDS = (
|
||
"rate limit", "rate_limit", "too many requests", "try again", "retry after", "resets in"
|
||
)
|
||
_RATE_LIMIT_BILLING_KEYWORDS = (
|
||
"credits", "insufficient funds", "billing", "payment required", "can only afford",
|
||
"out of funds", "run out of funds", "balance_depleted", "no usable credits",
|
||
"model_not_supported_on_free_tier", "not available on the free tier", "isn't available on the free tier",
|
||
)
|
||
|
||
|
||
def _is_overloaded_error(exc: Exception) -> bool:
|
||
"""Server busy, credential fine ("at capacity upstream … not your API key's rate limit"): back off,
|
||
never bench the credential (#108349). Same phrase table as the main classifier."""
|
||
return _contains_any(str(exc).lower(), _OVERLOADED_PATTERNS)
|
||
|
||
|
||
def _is_rate_limit_error(exc: Exception) -> bool:
|
||
"""429 rate limit (not billing/quota, which _is_payment_error owns).
|
||
|
||
OpenAI's RateLimitError may omit .status_code — matched by class name. A generic 429 without
|
||
billing keywords counts as a rate limit.
|
||
"""
|
||
# (PR #8023 pattern)
|
||
if type(exc).__name__ == "RateLimitError":
|
||
return True
|
||
if getattr(exc, "status_code", None) != 429:
|
||
return False
|
||
err_lower = str(exc).lower()
|
||
return _contains_any(err_lower, _RATE_LIMIT_KEYWORDS) or not _contains_any(err_lower, _RATE_LIMIT_BILLING_KEYWORDS)
|
||
|
||
|
||
def _is_timeout_error(exc: Exception) -> bool:
|
||
"""Full-budget request timeout, distinct from a fast connection drop.
|
||
|
||
A timeout burns the whole ``timeout`` budget, so a same-provider retry on the compression
|
||
path doubles wall time; fast drops stay on the retry path.
|
||
"""
|
||
with contextlib.suppress(ImportError):
|
||
from openai import APITimeoutError
|
||
if isinstance(exc, APITimeoutError):
|
||
return True
|
||
return "Timeout" in type(exc).__name__ or "timed out" in str(exc).lower()
|
||
|
||
|
||
def _is_connection_error(exc: Exception) -> bool:
|
||
"""Connection/network errors (endpoint unreachable), as opposed to 4xx/5xx API errors."""
|
||
with contextlib.suppress(ImportError):
|
||
from openai import APIConnectionError, APITimeoutError
|
||
if isinstance(exc, (APIConnectionError, APITimeoutError)):
|
||
return True
|
||
if _contains_any(type(exc).__name__, ("Connection", "Timeout", "DNS", "SSL")):
|
||
return True
|
||
return _contains_any(str(exc).lower(), (
|
||
"connection refused", "name or service not known", "no route to host",
|
||
"network is unreachable", "timed out", "connection reset",
|
||
# httpcore/httpx premature stream close — transient, retry/reroute.
|
||
"incomplete chunked read", "peer closed connection", "response ended prematurely",
|
||
"unexpected eof", "remoteprotocolerror", "localprotocolerror",
|
||
))
|
||
|
||
|
||
def _exc_http_status(exc: Exception) -> Any:
|
||
"""HTTP status on the exception itself or on its ``response`` (None when neither carries one)."""
|
||
return getattr(exc, "status_code", None) or getattr(getattr(exc, "response", None), "status_code", None)
|
||
|
||
|
||
def _is_transient_transport_error(exc: Exception) -> bool:
|
||
"""One-off transport blip worth retrying on the SAME provider: connection/stream-close errors plus pure 5xx/408.
|
||
|
||
Deliberately narrow: payment/auth/rate-limit errors switch provider, refresh creds, or rotate the pool.
|
||
"""
|
||
if _is_connection_error(exc):
|
||
return True
|
||
status = _exc_http_status(exc)
|
||
return isinstance(status, int) and (status == 408 or 500 <= status < 600)
|
||
|
||
|
||
_DEFAULT_TRANSIENT_RETRIES = 2
|
||
_TRANSIENT_RETRY_BACKOFF_BASE = 1.0 # Backoff base (seconds); overridable so tests can zero it out.
|
||
|
||
|
||
def _transient_retry_count() -> int:
|
||
"""Same-provider retries for a transient blip: ``auxiliary.transient_retries``
|
||
(default 2), clamped to [0, 6]; config-read failures fall back to default."""
|
||
try:
|
||
from hermes_cli.config import cfg_get, load_config
|
||
val = cfg_get(load_config(), "auxiliary", "transient_retries")
|
||
return _DEFAULT_TRANSIENT_RETRIES if val is None else max(0, min(int(val), 6))
|
||
except Exception:
|
||
return _DEFAULT_TRANSIENT_RETRIES
|
||
|
||
|
||
def _is_auth_error(exc: Exception) -> bool:
|
||
"""Auth failures that should trigger provider-specific refresh."""
|
||
status = getattr(exc, "status_code", None)
|
||
if status == 401:
|
||
return True
|
||
err_lower = str(exc).lower()
|
||
if "error code: 401" in err_lower or "authenticationerror" in type(exc).__name__.lower():
|
||
return True
|
||
# xAI returns 403 "unauthenticated:bad-credentials" for expired OAuth tokens — semantically a 401.
|
||
return "bad-credentials" in err_lower and (status == 403 or "unauthenticated" in err_lower)
|
||
|
||
|
||
def _is_unsupported_parameter_error(exc: Exception, param: str) -> bool:
|
||
"""Provider 400 for an unsupported request parameter: the parameter name plus a generic
|
||
unsupported/unknown/unrecognized marker, so call sites can retry without the key."""
|
||
param_lower = (param or "").lower()
|
||
if not param_lower:
|
||
return False
|
||
err_lower = str(exc).lower()
|
||
return param_lower in err_lower and _contains_any(err_lower, UNSUPPORTED_PARAM_MARKERS)
|
||
|
||
|
||
def _is_structured_output_rejection(exc: Exception) -> bool:
|
||
"""Provider 400/422 rejecting the structured-output field, on either wire: OpenAI ``response_format``
|
||
(incl. vLLM's ``guided_grammar``/xgrammar failures) or Anthropic ``output_config.format`` ("Extra inputs
|
||
are not permitted"). Callers tolerate an unconstrained reply, so the reaction is one retry without it."""
|
||
status = getattr(exc, "status_code", None)
|
||
if status is not None and status not in {400, 422}:
|
||
return False
|
||
err_lower = str(exc).lower()
|
||
# vLLM grammar-backend failures name the translated parameter, not ours.
|
||
if _contains_any(err_lower, ("guided_grammar", "xgrammar", "compile_grammar_error")):
|
||
return True
|
||
if "extra inputs are not permitted" in err_lower and (
|
||
"response_format" in err_lower or "output_config" in err_lower
|
||
):
|
||
return True
|
||
if "response_format" in err_lower and "unavailable" in err_lower:
|
||
return True
|
||
# Gateways that validate the request body with a strict pydantic model reject the
|
||
# OBJECT-form json_schema by shape ("str type expected" on response_format.json_schema,
|
||
# 422) rather than by naming the feature. The field is what they refuse; the retry
|
||
# without it is the same remedy, so treat the shape error as a rejection too.
|
||
if "response_format" in err_lower and "json_schema" in err_lower:
|
||
return True
|
||
# Gemini native names its own generationConfig keys, never ours: "Function calling with a response
|
||
# mime type: 'application/json' is unsupported" (pre-Gemini-3 + tools via a proxy), or an
|
||
# "Unknown name"/"Invalid value" 400 on response_schema / response_json_schema for a schema the
|
||
# surface cannot express. Same remedy: one retry without the format.
|
||
if _contains_any(err_lower, ("response mime type", "response_schema", "response_json_schema")):
|
||
return True
|
||
return _is_unsupported_parameter_error(exc, "response_format") or _is_unsupported_parameter_error(exc, "output_config")
|
||
|
||
|
||
def _without_structured_output_format(kwargs: dict) -> Optional[dict]:
|
||
"""Copy *kwargs* without ``response_format`` (top-level and ``extra_body``); None when nothing was
|
||
removed, so call sites don't retry an unchanged request."""
|
||
retry_kwargs = dict(kwargs)
|
||
changed = retry_kwargs.pop("response_format", None) is not None
|
||
extra_body = retry_kwargs.get("extra_body")
|
||
if isinstance(extra_body, dict) and "response_format" in extra_body:
|
||
remaining = {k: v for k, v in extra_body.items() if k != "response_format"}
|
||
if remaining:
|
||
retry_kwargs["extra_body"] = remaining
|
||
else:
|
||
retry_kwargs.pop("extra_body", None)
|
||
changed = True
|
||
return retry_kwargs if changed else None
|
||
|
||
|
||
def _is_reasoning_field_rejection(exc: Exception) -> bool:
|
||
"""Provider 400 rejecting a reasoning wire control by name (``reasoning_effort``, ``reasoning``,
|
||
``thinking``/``think``). Chat-only models behind OpenAI-compatible relays reject the top-level
|
||
``reasoning_effort: none`` a disabled ``reasoning_config`` projects on the custom profile
|
||
("Unrecognized request argument supplied: reasoning_effort", #112781); the route default is
|
||
the right answer for such a model, so the reaction is one retry without any reasoning field."""
|
||
status = getattr(exc, "status_code", None)
|
||
if status is not None and status not in {400, 422}:
|
||
return False
|
||
return is_reasoning_field_rejection(str(exc))
|
||
|
||
|
||
def _is_reasoning_required_rejection(exc: Exception) -> bool:
|
||
"""Provider 400 refusing to switch reasoning OFF ("Reasoning is mandatory for this endpoint and cannot
|
||
be disabled"): the field is understood, only the disable is refused, so the rung steps the effort up to
|
||
the floor instead of dropping the field (agent/auxiliary_reasoning_floor.py)."""
|
||
status = getattr(exc, "status_code", None)
|
||
if status is not None and status not in {400, 422}:
|
||
return False
|
||
return is_reasoning_required_rejection(str(exc))
|
||
|
||
|
||
def _without_reasoning_fields(kwargs: dict) -> Optional[dict]:
|
||
"""Copy *kwargs* without reasoning wire controls (top-level ``reasoning_effort``, the adapter's
|
||
private ``_reasoning_config`` and every ``extra_body`` reasoning key); None when nothing was
|
||
removed, so call sites don't retry an unchanged request."""
|
||
retry_kwargs = dict(kwargs)
|
||
changed = retry_kwargs.pop("reasoning_effort", None) is not None
|
||
changed = retry_kwargs.pop("_reasoning_config", None) is not None or changed
|
||
extra_body = retry_kwargs.get("extra_body")
|
||
if isinstance(extra_body, dict):
|
||
remaining = {k: v for k, v in extra_body.items() if str(k).strip().lower() not in _PROFILE_REASONING_KEYS}
|
||
if len(remaining) != len(extra_body):
|
||
if remaining:
|
||
retry_kwargs["extra_body"] = remaining
|
||
else:
|
||
retry_kwargs.pop("extra_body", None)
|
||
changed = True
|
||
return retry_kwargs if changed else None
|
||
|
||
|
||
def _is_model_not_found_error(exc: Exception) -> bool:
|
||
""""Requested model doesn't exist" (404 / invalid model) — typically a long-lived process pinned a
|
||
since-dropped model. Excludes billing keywords, which :func:`_is_payment_error` owns."""
|
||
status = getattr(exc, "status_code", None)
|
||
err_lower = str(exc).lower()
|
||
if _contains_any(err_lower, (
|
||
"credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
|
||
"no usable credits", "free tier", "free-tier", "not available on the free tier",
|
||
)):
|
||
return False
|
||
if status not in {404, 400, None}:
|
||
return False
|
||
return _contains_any(err_lower, (
|
||
"model does not exist", "does not exist in our configuration", "openrouter catalog",
|
||
"is not a valid model", "no such model", "model not found",
|
||
"the model `", # OpenAI-style: "The model `X` does not exist"
|
||
"model_not_found", "unknown model",
|
||
))
|
||
|
||
|
||
def _is_model_incompatible_error(exc: Exception) -> bool:
|
||
""""This route cannot serve this model" 400 (capability mismatch, e.g. a Codex/ChatGPT-account
|
||
fallback asked to run a non-OpenAI model). Auth/payment predicates don't fire, so this keeps the
|
||
chain going instead of aborting. Excludes billing 400s and not-found 400s."""
|
||
status = getattr(exc, "status_code", None)
|
||
if status not in {400, None}:
|
||
return False
|
||
err_lower = str(exc).lower()
|
||
if _is_model_not_found_error(exc):
|
||
return False
|
||
# Billing keywords checked directly: _is_payment_error is status-gated and misses 400-coded billing bodies.
|
||
if _contains_any(err_lower, (
|
||
"credits", "insufficient funds", "billing", "out of funds", "balance_depleted",
|
||
"no usable credits", "payment required", "free tier", "free-tier",
|
||
"not available on the free tier", "model_not_supported_on_free_tier", "quota",
|
||
)):
|
||
return False
|
||
return _contains_any(err_lower, (
|
||
"is not supported when using", # codex/ChatGPT-account model gating
|
||
"model is not supported", "not supported with this", "not supported for this account",
|
||
"model_not_supported", "does not support this model", "unsupported model",
|
||
))
|
||
|
||
|
||
def _is_invalid_aux_response_error(exc: Exception) -> bool:
|
||
"""HTTP-200 empty/malformed ChatCompletions — a capability failure routed like model incompatibility."""
|
||
if not isinstance(exc, RuntimeError):
|
||
return False
|
||
msg = str(exc).lower()
|
||
return "auxiliary " in msg and "llm returned invalid response" in msg and "choices[0].message" in msg
|
||
|
||
|
||
def _is_statusless_structured_provider_error(exc: Exception) -> bool:
|
||
"""Detect a structured provider failure that has no HTTP status.
|
||
|
||
OpenAI-compatible relays may commit SSE with status 200, then send an
|
||
OpenAI-style ``error`` event. The SDK raises a status-less ``APIError`` with
|
||
``body=data["error"]`` — the INNER error object or a bare string (an
|
||
``{"error": ...}`` wrapper is accepted too). Any non-empty structured error in
|
||
that status-less shape is a route failure; ordinary HTTP errors keep their
|
||
existing status-based classifiers, and message text alone is insufficient.
|
||
"""
|
||
if _exc_http_status(exc) is not None:
|
||
return False
|
||
body = getattr(exc, "body", None)
|
||
err = body.get("error") if isinstance(body, dict) and "error" in body else body
|
||
if isinstance(err, str):
|
||
return bool(err.strip())
|
||
return isinstance(err, dict) and any(err.get(k) for k in ("type", "code", "message"))
|
||
|
||
|
||
# Tasks on a user-visible critical path (compression blocks resuming an oversized session; vision
|
||
# stalls the serialised turn queue). A same-provider retry after a full-budget timeout costs another
|
||
# whole ``timeout`` window, so they skip straight to fallback; fast blips still retry.
|
||
# Fast blips (a streaming-close or a 5xx) still retry, since those are cheap. See issue #54465 for the
|
||
# compression case. Title generation joins them because its retries multiplied the user's
|
||
# ``auxiliary.title_generation.timeout`` (~4x: three full windows plus backoff) on a slow local
|
||
# model, and the auto-title thread outlived the deadline the user thought they had set (#89445, #66251).
|
||
_TIMEOUT_NO_RETRY_TASKS = frozenset({"compression", "vision", "title_generation"})
|
||
|
||
|
||
def _should_skip_same_provider_retry(task: Optional[str], exc: Exception) -> bool:
|
||
"""True when a transient error on a critical-path task should go straight to fallback.
|
||
|
||
Carve-out: a fast first-token fail (dead stream within the no-progress window, zero output —
|
||
see ``_timeout_message``) is cheap and keeps the same-provider retry; mid-stream stalls and
|
||
hard-ceiling timeouts skip to fallback.
|
||
"""
|
||
return task in _TIMEOUT_NO_RETRY_TASKS and _is_timeout_error(exc) and "no-progress timeout" not in str(exc)
|
||
|
||
|
||
def _evict_cached_clients(provider: str) -> None:
|
||
"""Drop this profile's cached auxiliary clients for a provider so fresh creds are used.
|
||
|
||
Scoped to the calling profile (``hermes_home_key()`` is the first key slot): a rotation in
|
||
one profile must not drop another profile's client for the same provider in a multiplexing
|
||
gateway, since that profile's credentials did not change. Entries are popped, not closed:
|
||
a concurrent caller may be mid-request on the shared client (closing it raises ReadError /
|
||
"client has been closed" for them); the dropped client is retired by GC like the FIFO
|
||
overflow path in ``_get_cached_client``.
|
||
"""
|
||
normalized = _normalize_aux_provider(provider)
|
||
home = hermes_home_key()
|
||
with _client_cache_lock:
|
||
for key in [key for key in _client_cache
|
||
if key[0] == home and _normalize_aux_provider(str(key[1])) == normalized]:
|
||
_client_cache.pop(key, None)
|
||
|
||
|
||
def _evict_cached_client_instance(target: Any) -> bool:
|
||
"""Drop cache entries whose stored client (or its ``_real_client``) is *target*; True if any evicted.
|
||
|
||
Used when a cached client is poisoned (closed transport after a timeout). Async wrappers must
|
||
expose the same ``_real_client`` as their sync sibling or the async entry survives.
|
||
"""
|
||
if target is None:
|
||
return False
|
||
evicted = False
|
||
with _client_cache_lock:
|
||
for key, entry in list(_client_cache.items()):
|
||
cached = entry[0] if entry is not None else None
|
||
if cached is not None and (cached is target or getattr(cached, "_real_client", None) is target):
|
||
del _client_cache[key]
|
||
evicted = True
|
||
return evicted
|
||
|
||
|
||
def _pool_credential_digest(pool: Any, entry: Any = None) -> str:
|
||
"""Secret-safe digest over the runtime key(s) backing the cache hint.
|
||
|
||
When ``entry`` (the peeked selection) carries a key, only THAT key is digested: rotating a
|
||
sibling account must not churn this entry's cached client. Otherwise every pool entry
|
||
contributes, taken WITHOUT availability filters (``pool.entries()``, not ``peek()``): an entry
|
||
benched by a per-model cooldown still counts, so a token rotated while the cooldown is active
|
||
changes the digest too (#113022 point 5). Only the blake2b of the keys enters the cache key.
|
||
"""
|
||
keys = [k for k in (_pool_runtime_api_key(entry),) if k] if entry is not None else []
|
||
if not keys:
|
||
try:
|
||
entries_fn = getattr(pool, "entries", None)
|
||
entries = entries_fn() if callable(entries_fn) else []
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary client: could not list pool entries for digest: %s", exc)
|
||
return ""
|
||
keys = sorted(k for k in (_pool_runtime_api_key(e) for e in entries or ()) if k)
|
||
if not keys:
|
||
return ""
|
||
digest = hashlib.blake2b(digest_size=16)
|
||
for key in keys:
|
||
digest.update(key.encode("utf-8"))
|
||
digest.update(b"\0")
|
||
return digest.hexdigest()
|
||
|
||
|
||
def _pool_cache_hint(provider: str, *, main_runtime: Optional[Dict[str, Any]] = None) -> str:
|
||
"""Return a cache discriminator that follows the pooled credential, not just its entry id.
|
||
|
||
``<provider>:<entry id>:<digest of that entry's key>`` when ``peek()`` names an entry,
|
||
``<provider>::<digest of every entry's key>`` when every entry is filtered out (cooldown)
|
||
but keyed entries exist. A token rotated or
|
||
refreshed by another process yields a new client instead of a 401 round trip on the cached
|
||
one; the stale entry ages out through the non-closing FIFO cap (#113022).
|
||
"""
|
||
normalized = _normalize_aux_provider(provider)
|
||
if normalized == "auto":
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
normalized = _normalize_aux_provider(runtime.get("provider") or _read_main_provider())
|
||
if normalized in {"", "auto", "custom"}:
|
||
return ""
|
||
pool = _load_pool_with_credentials(normalized, " (cache hint)")
|
||
if pool is None:
|
||
return ""
|
||
entry = _peek_pool_entry(normalized, pool)
|
||
digest = _pool_credential_digest(pool, entry)
|
||
entry_id = str(getattr(entry, "id", "") or "").strip() if entry is not None else ""
|
||
if not entry_id and not digest:
|
||
return ""
|
||
return f"{normalized}:{entry_id}:{digest}"
|
||
|
||
|
||
# Ordered (host, provider) tables for inferring a backend from a client base URL.
|
||
_POOL_PROVIDER_BY_HOST = (
|
||
("chatgpt.com", "openai-codex"), ("openrouter.ai", "openrouter"),
|
||
("inference-api.nousresearch.com", "nous"), ("api.anthropic.com", "anthropic"),
|
||
("githubcopilot.com", "copilot"), ("api.kimi.com", "kimi-coding"), ("api.x.ai", "xai-oauth"),
|
||
)
|
||
_AUTH_REFRESH_PROVIDER_BY_HOST = (
|
||
("api.githubcopilot.com", "copilot"), ("chatgpt.com", "openai-codex"),
|
||
("api.anthropic.com", "anthropic"), ("inference-api.nousresearch.com", "nous"),
|
||
# An aux call that inherits the main xai-oauth route arrives as "auto"; without this row the
|
||
# 403 bad-credentials rung skipped the refresh and benched the only grant (#84845).
|
||
("api.x.ai", "xai-oauth"),
|
||
)
|
||
|
||
|
||
def _provider_for_host(base_url: str, table: Tuple[Tuple[str, str], ...]) -> Optional[str]:
|
||
"""First provider in ``table`` whose host matches ``base_url``, else None."""
|
||
for host, provider in table:
|
||
if base_url_host_matches(base_url, host):
|
||
return provider
|
||
return None
|
||
|
||
|
||
def _recoverable_pool_provider(
|
||
resolved_provider: str, client: Any, main_runtime: Optional[Dict[str, Any]] = None
|
||
) -> Optional[str]:
|
||
"""Infer which provider pool can recover the current auxiliary client.
|
||
None when the client targets a different host than the session's configured endpoint for that
|
||
provider: a rejection there says nothing about the key, so rotating/quarantining it would kill a
|
||
working credential (Miho report — proxy users)."""
|
||
normalized = _normalize_aux_provider(resolved_provider)
|
||
base = str(getattr(client, "base_url", "") or "")
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
rt_base = str(runtime.get("base_url") or "")
|
||
rt_key = runtime.get("api_key")
|
||
client_key = getattr(client, "api_key", None)
|
||
# Only the SESSION's own key is shielded, and only when it was sent somewhere other than the
|
||
# session's origin (scheme+host+port — a port or HTTPS→HTTP change is a different trust boundary).
|
||
# An independently owned auxiliary pool keeps rotating at its own origin.
|
||
if (base and rt_base and normalized == runtime.get("provider")
|
||
and isinstance(rt_key, str) and rt_key and client_key == rt_key
|
||
and base_url_origin(base) != base_url_origin(rt_base)):
|
||
logger.info("Auxiliary: %s rejected the session key at %s, but the session's endpoint is %s — "
|
||
"endpoint mismatch, not a dead key; skipping credential rotation",
|
||
normalized, base_url_hostname(base), base_url_hostname(rt_base))
|
||
return None
|
||
if normalized not in {"", "auto", "custom"}:
|
||
return normalized
|
||
known = _provider_for_host(base, _POOL_PROVIDER_BY_HOST)
|
||
if known is not None:
|
||
return known
|
||
# Providers outside the table (e.g. opencode-go): match base URL against registered
|
||
# api_key providers so pool rotation works for them too.
|
||
if main_runtime:
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
rt_provider = runtime.get("provider", "")
|
||
if rt_provider and rt_provider not in {"", "auto", "custom"}:
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli.auth import PROVIDER_REGISTRY
|
||
pconfig = PROVIDER_REGISTRY.get(rt_provider)
|
||
if pconfig and getattr(pconfig, "auth_type", None) == "api_key":
|
||
# The pool's key was issued for the endpoint the main runtime actually uses; a
|
||
# rejection at any other host (registry default vs configured proxy) says nothing
|
||
# about that key, so it must not be marked exhausted.
|
||
rt_base = str(runtime.get("base_url") or getattr(pconfig, "inference_base_url", "") or "").rstrip("/")
|
||
if rt_base and base_url_host_matches(base, base_url_hostname(rt_base)):
|
||
return rt_provider
|
||
return None
|
||
|
||
|
||
def _recover_provider_pool(provider: str, exc: Exception, *, failed_api_key: str = "") -> bool:
|
||
"""Try same-provider credential-pool recovery for auxiliary calls.
|
||
|
||
``failed_api_key`` lets mark_exhausted_and_rotate identify the right pool entry even if
|
||
another process already rotated (current() would be None).
|
||
"""
|
||
normalized = _normalize_aux_provider(provider)
|
||
try:
|
||
pool = load_pool(normalized)
|
||
except Exception as load_exc:
|
||
logger.debug("Auxiliary client: could not load pool for %s recovery: %s", normalized, load_exc)
|
||
return False
|
||
if not pool or not pool.has_credentials():
|
||
return False
|
||
status_code = getattr(exc, "status_code", None)
|
||
|
||
def _rotate(fallback_status: int) -> bool:
|
||
error_context: Dict[str, Any] = {"message": str(exc)}
|
||
if status_code is not None:
|
||
error_context["status_code"] = status_code
|
||
next_entry = pool.mark_exhausted_and_rotate(
|
||
status_code=status_code if status_code is not None else fallback_status,
|
||
error_context=error_context, api_key_hint=failed_api_key or None,
|
||
)
|
||
if next_entry is None:
|
||
return False
|
||
_evict_cached_clients(normalized)
|
||
return True
|
||
|
||
if _is_auth_error(exc):
|
||
if pool.try_refresh_current() is not None:
|
||
_evict_cached_clients(normalized)
|
||
return True
|
||
return _rotate(401)
|
||
if _is_payment_error(exc):
|
||
return _rotate(402)
|
||
if _is_rate_limit_error(exc) and not _is_overloaded_error(exc):
|
||
return _rotate(429)
|
||
return False
|
||
|
||
|
||
def _prepare_same_provider_retry(
|
||
*, task: Optional[str], resolved_provider: str, resolved_model: Optional[str],
|
||
resolved_base_url: Optional[str], resolved_api_key: Optional[str],
|
||
resolved_api_mode: Optional[str], main_runtime: Optional[Dict[str, Any]],
|
||
final_model: Optional[str], messages: list, temperature: Optional[float],
|
||
max_tokens: Optional[int], tools: Optional[list], effective_timeout: float,
|
||
effective_extra_body: dict, reasoning_config: Optional[dict], async_mode: bool,
|
||
extra_headers: Optional[Dict[str, str]] = None,
|
||
) -> Tuple[Any, Dict[str, Any]]:
|
||
"""Rebuild (client, request kwargs) for a same-provider retry after credential recovery."""
|
||
if task == "vision":
|
||
effective_provider, retry_client, retry_model = resolve_vision_provider_client(
|
||
provider=resolved_provider, model=final_model, base_url=resolved_base_url,
|
||
api_key=resolved_api_key, async_mode=async_mode,
|
||
)
|
||
else:
|
||
retry_client, retry_model = _get_cached_client(
|
||
resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
|
||
api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
|
||
)
|
||
effective_provider = _effective_provider_for_client(retry_client, resolved_provider)
|
||
if retry_client is None:
|
||
raise RuntimeError(
|
||
f"Auxiliary {task or 'call'}: provider {resolved_provider} could not be rebuilt after recovery"
|
||
)
|
||
retry_base = str(getattr(retry_client, "base_url", "") or "")
|
||
retry_kwargs = _build_call_kwargs(
|
||
effective_provider or resolved_provider, retry_model or final_model, messages,
|
||
temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=effective_timeout,
|
||
extra_body=effective_extra_body, reasoning_config=reasoning_config,
|
||
base_url=retry_base or resolved_base_url, task=task,
|
||
)
|
||
# Preserve per-request attribution headers (e.g. Copilot ``x-initiator``) so the retry keeps capability gating.
|
||
if extra_headers:
|
||
# Copilot's ``x-initiator: user``) across the rebuilt-client retry — dropping them here would let a
|
||
# recovery retry silently lose capability gating (#60293).
|
||
# Preserve per-request attribution headers across the rebuilt-client retry — see the sync variant
|
||
# above (#60293).
|
||
retry_kwargs["extra_headers"] = dict(extra_headers)
|
||
if _is_anthropic_compat_endpoint(resolved_provider, retry_base):
|
||
retry_kwargs["messages"] = _convert_openai_images_to_anthropic(retry_kwargs["messages"])
|
||
return retry_client, retry_kwargs
|
||
|
||
|
||
def _retry_same_provider_sync(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
|
||
retry_client, retry_kwargs = _prepare_same_provider_retry(
|
||
task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=False, **prep,
|
||
)
|
||
return _validate_llm_response(
|
||
_relay_sync_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode), task,
|
||
)
|
||
|
||
|
||
async def _retry_same_provider_async(*, resolved_provider: str, resolved_api_mode: Optional[str], task: Optional[str], **prep) -> Any:
|
||
retry_client, retry_kwargs = _prepare_same_provider_retry(
|
||
task=task, resolved_provider=resolved_provider, resolved_api_mode=resolved_api_mode, async_mode=True, **prep,
|
||
)
|
||
return _validate_llm_response(
|
||
await _relay_async_completion(retry_client, retry_kwargs, provider=resolved_provider, api_mode=resolved_api_mode),
|
||
task,
|
||
)
|
||
|
||
|
||
def _creds_have_api_key(creds: Dict[str, Any]) -> bool:
|
||
return bool(str(creds.get("api_key", "") or "").strip())
|
||
|
||
|
||
def _refresh_copilot_credentials() -> bool:
|
||
from hermes_cli.copilot_auth import _jwt_cache, _token_fingerprint, exchange_copilot_token, resolve_copilot_token
|
||
raw_token, _source = resolve_copilot_token()
|
||
if not str(raw_token or "").strip():
|
||
return False
|
||
_jwt_cache.pop(_token_fingerprint(raw_token), None)
|
||
exchange_copilot_token(raw_token)
|
||
return True
|
||
|
||
|
||
def _refresh_codex_credentials() -> bool:
|
||
from hermes_cli.auth import resolve_codex_runtime_credentials
|
||
return _creds_have_api_key(resolve_codex_runtime_credentials(force_refresh=True))
|
||
|
||
|
||
def _refresh_nous_credentials() -> bool:
|
||
from hermes_cli.auth import resolve_nous_runtime_credentials
|
||
return _creds_have_api_key(resolve_nous_runtime_credentials(
|
||
timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=True
|
||
))
|
||
|
||
|
||
def _refresh_anthropic_credentials(failed_api_key: str = "") -> bool:
|
||
from agent.anthropic_credentials import read_claude_code_credentials, _refresh_oauth_token
|
||
token = failed_api_key
|
||
if not token:
|
||
return False
|
||
pool = load_pool("anthropic")
|
||
if pool.entry_id_for_api_key(token):
|
||
return pool.try_refresh_matching(api_key_hint=token) is not None
|
||
creds = read_claude_code_credentials()
|
||
# Never spend an ambient login's refresh rotation for another request's key.
|
||
if isinstance(creds, dict) and creds.get("accessToken") == token and creds.get("refreshToken"):
|
||
return bool(_refresh_oauth_token(creds))
|
||
return False
|
||
|
||
|
||
def _refresh_xai_oauth_credentials() -> bool:
|
||
"""Pool-level refresh first, then the singleton auth-store resolver."""
|
||
pool = load_pool("xai-oauth")
|
||
if pool and pool.has_credentials():
|
||
pool.select()
|
||
refreshed = pool.try_refresh_current()
|
||
if refreshed is not None and str(getattr(refreshed, "runtime_api_key", "") or "").strip():
|
||
return True
|
||
from hermes_cli.auth import resolve_xai_oauth_runtime_credentials
|
||
return _creds_have_api_key(resolve_xai_oauth_runtime_credentials(force_refresh=True))
|
||
|
||
|
||
def _refresh_vertex_credentials() -> bool:
|
||
"""Mirrors run_agent's Vertex refresh; the cache key ignores the rotating bearer, so
|
||
without the eviction that follows, a ~1h-expired aux Vertex client 401s forever."""
|
||
from agent.vertex_adapter import get_vertex_config
|
||
token, base_url = get_vertex_config()
|
||
return bool(isinstance(token, str) and token.strip() and isinstance(base_url, str) and base_url.strip())
|
||
|
||
|
||
# Each refresher returns True when a usable credential exists; the caller then evicts cached clients.
|
||
_CREDENTIAL_REFRESHERS: Dict[str, Callable[..., bool]] = {
|
||
"copilot": _refresh_copilot_credentials, "openai-codex": _refresh_codex_credentials,
|
||
"nous": _refresh_nous_credentials, "anthropic": _refresh_anthropic_credentials,
|
||
"xai-oauth": _refresh_xai_oauth_credentials, "vertex": _refresh_vertex_credentials,
|
||
}
|
||
|
||
|
||
def _refresh_provider_credentials(provider: str, *, failed_api_key: str = "") -> bool:
|
||
"""Refresh short-lived credentials for OAuth-backed auxiliary providers."""
|
||
normalized = _normalize_aux_provider(provider)
|
||
refresher = _CREDENTIAL_REFRESHERS.get(normalized)
|
||
if refresher is None:
|
||
return False
|
||
try:
|
||
if not (refresher(failed_api_key) if normalized == "anthropic" else refresher()):
|
||
return False
|
||
_evict_cached_clients(normalized)
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary provider credential refresh failed for %s: %s", normalized, exc)
|
||
return False
|
||
|
||
|
||
def _auth_refresh_provider_for_route(
|
||
resolved_provider: Optional[str], client_base_url: str, effective_provider: str = "",
|
||
) -> str:
|
||
"""Provider whose short-lived credentials should be refreshed; auto-routed calls keep
|
||
``resolved_provider == "auto"``, so infer the backend from the client's base URL."""
|
||
normalized = _normalize_aux_provider(resolved_provider)
|
||
if normalized and normalized != "auto":
|
||
return normalized
|
||
host_provider = _provider_for_host(client_base_url, _AUTH_REFRESH_PROVIDER_BY_HOST)
|
||
# The auto client already knows it runs on the host's API-key sibling (``xai`` on api.x.ai):
|
||
# an XAI_API_KEY 401 must not spend a stale ``xai-oauth`` grant's refresh and switch routes.
|
||
if host_provider and host_provider == f"{_normalize_aux_provider(effective_provider)}-oauth":
|
||
return normalized
|
||
return host_provider or normalized
|
||
|
||
|
||
def _fallback_chain_entry(task: Optional[str], fb_label: str) -> Optional[Dict[str, Any]]:
|
||
"""Resolve the ``fallback_chain`` entry a ``fallback_chain[<i>](<provider>)`` label points at,
|
||
or None when the label is not a configured-chain candidate or the index no longer resolves."""
|
||
if not task or not fb_label:
|
||
return None
|
||
m = re.match(r"fallback_chain\[(\d+)\]", fb_label)
|
||
if not m:
|
||
return None
|
||
try:
|
||
chain = _get_auxiliary_task_config(task).get("fallback_chain")
|
||
entry = chain[int(m.group(1))] if isinstance(chain, list) else None
|
||
except Exception:
|
||
return None
|
||
return entry if isinstance(entry, dict) else None
|
||
|
||
|
||
def _coerce_positive_timeout(raw: Any) -> Optional[float]:
|
||
"""Coerce a config ``timeout`` to a positive float, or None (rejects bools, which are ints)."""
|
||
if isinstance(raw, (int, float)) and not isinstance(raw, bool) and raw > 0:
|
||
return float(raw)
|
||
return None
|
||
|
||
|
||
def _fallback_entry_timeout(task: Optional[str], fb_label: str) -> Optional[float]:
|
||
"""Per-entry ``timeout`` for a configured fallback candidate, or None (keep the task-level
|
||
timeout). Inheriting the primary's deadline used to kill healthy-but-slower fallbacks.
|
||
|
||
A fallback candidate previously inherited the exact timeout the primary provider was called with. When
|
||
that deadline was tuned for the primary (or the primary simply consumed its whole budget before failing
|
||
over), the fallback aborted on the same clock even when independently healthy — a 163k-token compression
|
||
that needs ~90s on the fallback died at the primary's 30s deadline every turn (#62452).
|
||
"""
|
||
entry = _fallback_chain_entry(task, fb_label)
|
||
return _coerce_positive_timeout(entry.get("timeout") if entry else None)
|
||
|
||
|
||
def _fallback_provider_from_label(label: str) -> str:
|
||
"""Recover the provider identifier from a fallback display label."""
|
||
match = re.match(r"(?:fallback_chain\[\d+\]|fallback_providers\[\d+\]|main-agent)\(([^)]+)\)$", label or "")
|
||
return match.group(1).strip() if match else str(label or "").strip()
|
||
|
||
|
||
class _FallbackDestination(NamedTuple):
|
||
provider: str
|
||
base_url: str
|
||
api_mode: Optional[str]
|
||
model: Optional[str]
|
||
|
||
|
||
def _complete_fallback_destination(
|
||
provider: str, base_url: str, api_mode: Optional[str], model: Optional[str]
|
||
) -> _FallbackDestination:
|
||
if not api_mode:
|
||
if _endpoint_speaks_anthropic_messages(base_url):
|
||
api_mode = "anthropic_messages"
|
||
else:
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli.runtime_provider import resolve_runtime_provider
|
||
runtime = resolve_runtime_provider(
|
||
requested=provider, explicit_base_url=base_url or None, target_model=model or ""
|
||
)
|
||
api_mode = str(runtime.get("api_mode") or "").strip() or None
|
||
return _FallbackDestination(provider, base_url, api_mode, model)
|
||
|
||
|
||
def _fallback_destination_from_entry(
|
||
entry: Dict[str, Any], fb_client: Any, fb_model: Optional[str]
|
||
) -> _FallbackDestination:
|
||
provider = str(entry.get("provider") or "").strip()
|
||
base_url = str(entry.get("base_url") or getattr(fb_client, "base_url", "") or "").strip()
|
||
api_mode = str(entry.get("api_mode") or entry.get("transport") or "").strip() or None
|
||
model = fb_model or str(entry.get("model") or "").strip() or None
|
||
return _complete_fallback_destination(provider, base_url, api_mode, model)
|
||
|
||
|
||
def _fallback_destination(
|
||
task: Optional[str], fb_client: Any, fb_model: Optional[str], fb_label: str
|
||
) -> _FallbackDestination:
|
||
"""Route identity of a fallback request: attached destination, else configured entry, else label."""
|
||
attached = getattr(fb_client, "_hermes_fallback_destination", None)
|
||
if isinstance(attached, _FallbackDestination):
|
||
return attached
|
||
entry = _fallback_chain_entry(task, fb_label)
|
||
if entry is not None:
|
||
return _fallback_destination_from_entry(entry, fb_client, fb_model)
|
||
return _complete_fallback_destination(
|
||
_fallback_provider_from_label(fb_label), str(getattr(fb_client, "base_url", "") or ""), None, fb_model,
|
||
)
|
||
|
||
|
||
def _replan_synchronous_cache_sections(
|
||
messages: list, tools: Optional[list], *, destination: _FallbackDestination
|
||
) -> tuple[list, list]:
|
||
"""Strip source decoration and plan one synchronous destination locally."""
|
||
from agent.agent_runtime_helpers import configured_cache_ttl, plan_cache_sections_for_destination
|
||
return plan_cache_sections_for_destination(
|
||
messages, tools, provider=destination.provider, base_url=destination.base_url,
|
||
api_mode=destination.api_mode or "", model=destination.model or "",
|
||
# Operator's configured TTL so fallbacks don't regress 1h → 5m default (no live agent here; read config).
|
||
cache_ttl=configured_cache_ttl(),
|
||
)
|
||
|
||
|
||
def _fallback_request_kwargs(
|
||
destination: _FallbackDestination, *, task: Optional[str], messages: list,
|
||
tools: Optional[list], temperature: Optional[float], max_tokens: Optional[int],
|
||
effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
|
||
fallback_entry: dict, task_config: dict, apply_fast_lane: bool,
|
||
) -> Dict[str, Any]:
|
||
"""Build request kwargs for one fallback destination (cache-section replan + fast-lane cap)."""
|
||
fallback_max_tokens, fallback_extra_body = max_tokens, effective_extra_body
|
||
if apply_fast_lane:
|
||
fallback_max_tokens, fallback_extra_body = _compression_fast_lane_controls(
|
||
task, actual_provider=destination.provider, actual_model=destination.model,
|
||
requested_provider=fallback_entry.get("provider"),
|
||
requested_model=fallback_entry.get("model"), route_config=fallback_entry,
|
||
leak_guard_config=task_config, max_tokens=max_tokens, extra_body=effective_extra_body,
|
||
)
|
||
fallback_messages, fallback_tools = _replan_synchronous_cache_sections(messages, tools, destination=destination)
|
||
fb_kwargs = _build_call_kwargs(
|
||
destination.provider, destination.model, fallback_messages,
|
||
temperature=temperature, max_tokens=fallback_max_tokens, tools=fallback_tools, timeout=effective_timeout,
|
||
extra_body=fallback_extra_body, reasoning_config=reasoning_config, base_url=destination.base_url, task=task)
|
||
return fb_kwargs
|
||
|
||
|
||
def _plan_fallback_candidate(
|
||
fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str],
|
||
effective_timeout: float, apply_fast_lane: bool, **request,
|
||
) -> Tuple[_FallbackDestination, Dict[str, Any], Callable[[str, Any, Optional[str]], Dict[str, Any]]]:
|
||
"""Resolve the destination + first-attempt kwargs for a fallback candidate.
|
||
|
||
Returns ``(destination, kwargs, rebuild)`` where ``rebuild(provider, client, model)`` produces
|
||
kwargs for the credential-refreshed retry destination. A configured-chain entry's own
|
||
``timeout`` overrides ``effective_timeout``.
|
||
"""
|
||
fb_timeout = _fallback_entry_timeout(task, fb_label)
|
||
if fb_timeout is not None and fb_timeout != effective_timeout:
|
||
logger.info(
|
||
"Auxiliary %s: %s using its configured timeout %.0fs "
|
||
"(task-level was %.0fs)",
|
||
task or "call", fb_label, fb_timeout, effective_timeout,
|
||
)
|
||
effective_timeout = fb_timeout
|
||
destination = _fallback_destination(task, fb_client, fb_model, fb_label)
|
||
task_config = _get_auxiliary_task_config(task) if task == "compression" else {}
|
||
fallback_entry = _fallback_chain_entry(task, fb_label) or {}
|
||
common = dict(
|
||
task=task, effective_timeout=effective_timeout, fallback_entry=fallback_entry,
|
||
task_config=task_config, apply_fast_lane=apply_fast_lane, **request,
|
||
)
|
||
|
||
def _rebuild(provider: str, client: Any, model: Optional[str]) -> Tuple[_FallbackDestination, Dict[str, Any]]:
|
||
retry_destination = _FallbackDestination(
|
||
provider, destination.base_url or str(getattr(client, "base_url", "") or ""),
|
||
destination.api_mode, model or destination.model,
|
||
)
|
||
return retry_destination, _fallback_request_kwargs(retry_destination, **common)
|
||
|
||
return destination, _fallback_request_kwargs(destination, **common), _rebuild
|
||
|
||
|
||
def _quarantine_fallback_candidate(
|
||
task: Optional[str], fb_label: str, fb_provider: str, fb_err: Exception, *,
|
||
base_url: str = "", tag: str = "", reason: Optional[str] = None,
|
||
) -> None:
|
||
"""The candidate cannot serve this walk (``reason`` = its ``_FALLBACK_REASONS`` capacity label,
|
||
None = dead token): mark it unhealthy so the ordered re-walk skips it and the caller moves on to
|
||
the next entry. Transient classes get a short hold, payment/quota and dead tokens the long one."""
|
||
_mark_provider_unhealthy(
|
||
fb_provider or fb_label, ttl=fallback_candidate_quarantine_ttl(reason),
|
||
base_url=base_url, reason=reason or "stale fallback credential")
|
||
why = f"is out of capacity ({reason})" if reason else "has a stale/unrefreshable credential"
|
||
logger.warning("Auxiliary %s%s: fallback candidate %s %s (%s) — skipping to next fallback",
|
||
task or "call", tag, fb_label, why, fb_err)
|
||
|
||
|
||
def _plan_fallback_auth_retry(
|
||
destination: _FallbackDestination,
|
||
rebuild: Callable[[str, Any, Optional[str]], Tuple[_FallbackDestination, Dict[str, Any]]], *,
|
||
async_mode: bool,
|
||
failed_api_key: str = "",
|
||
) -> Tuple[str, Optional[Tuple[Any, Dict[str, Any], _FallbackDestination]]]:
|
||
"""After an auth error on a fallback candidate: refresh credentials and rebuild the request.
|
||
Returns ``(refresh_provider, retry)``; ``retry`` = ``(client, kwargs, destination)`` or None."""
|
||
fb_provider = _auth_refresh_provider_for_route(destination.provider, destination.base_url)
|
||
refresh_kwargs = {"failed_api_key": failed_api_key} if fb_provider == "anthropic" else {}
|
||
if fb_provider not in {"auto", "", None} and _refresh_provider_credentials(fb_provider, **refresh_kwargs):
|
||
retry_client, retry_model = _get_cached_client(
|
||
fb_provider, destination.model, **({"async_mode": True} if async_mode else {}),
|
||
base_url=destination.base_url or None, api_mode=destination.api_mode,
|
||
)
|
||
if retry_client is not None:
|
||
retry_destination, retry_kwargs = rebuild(fb_provider, retry_client, retry_model)
|
||
return fb_provider, (retry_client, retry_kwargs, retry_destination)
|
||
return fb_provider, None
|
||
|
||
|
||
def _call_fallback_candidate_sync(
|
||
fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
|
||
temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
|
||
effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
|
||
) -> Optional[Any]:
|
||
"""Call one fallback candidate with stale-credential recovery: on an auth error refresh its
|
||
credentials and retry once with a rebuilt client; if that also auth-fails, quarantine the
|
||
provider and return None so the caller moves on. A capacity error (quota/rate-limit 429, 402,
|
||
connection, route-incompatible model, malformed response) also quarantines and returns None so
|
||
the ordered chain advances to the next configured entry (#106367); other errors raise.
|
||
|
||
``effective_timeout`` is the task-level deadline; a configured-chain candidate with its own ``timeout``
|
||
entry gets that instead, so a fallback tuned differently from the primary is allowed its own budget
|
||
(#62452).
|
||
"""
|
||
destination, fb_kwargs, rebuild = _plan_fallback_candidate(
|
||
fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
|
||
apply_fast_lane=True, messages=messages, tools=tools, temperature=temperature,
|
||
max_tokens=max_tokens, effective_extra_body=effective_extra_body,
|
||
reasoning_config=reasoning_config,
|
||
)
|
||
|
||
def _send(client: Any, request_kwargs: Dict[str, Any], dest: _FallbackDestination) -> Any:
|
||
return _validate_llm_response(
|
||
_relay_sync_completion(
|
||
client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode,
|
||
create=lambda request: _create_with_progress(
|
||
client, request, task,
|
||
force_stream=_provider_requires_stream(dest.provider, dest.base_url),
|
||
),
|
||
),
|
||
task,
|
||
)
|
||
from agent.auxiliary_fallback_recovery import send_with_parameter_rungs
|
||
|
||
def _send_recovering(client: Any, request_kwargs: Dict[str, Any], dest: _FallbackDestination) -> Any:
|
||
return send_with_parameter_rungs(
|
||
lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
|
||
try:
|
||
return _send_recovering(fb_client, fb_kwargs, destination)
|
||
except Exception as fb_err:
|
||
if not _is_auth_error(fb_err):
|
||
capacity = fallback_candidate_unavailable_reason(fb_err)
|
||
if capacity is None:
|
||
raise
|
||
_quarantine_fallback_candidate(
|
||
task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
|
||
reason=capacity)
|
||
return None
|
||
fb_provider, retry = _plan_fallback_auth_retry(
|
||
destination, rebuild, async_mode=False, failed_api_key=getattr(fb_client, "api_key", ""))
|
||
failed_destination = destination
|
||
if retry is not None:
|
||
failed_destination = retry[2]
|
||
try:
|
||
return _send_recovering(*retry)
|
||
except Exception as retry_err:
|
||
if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
|
||
raise
|
||
_quarantine_fallback_candidate(
|
||
task, fb_label, fb_provider, fb_err, base_url=failed_destination.base_url,
|
||
)
|
||
return None
|
||
|
||
|
||
async def _call_fallback_candidate_async(
|
||
fb_client: Any, fb_model: Optional[str], fb_label: str, *, task: Optional[str], messages: list,
|
||
temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
|
||
effective_timeout: float, effective_extra_body: dict, reasoning_config: Optional[dict],
|
||
) -> Optional[Any]:
|
||
"""Async mirror of :func:`_call_fallback_candidate_sync` (no fast-lane cap on this wire)."""
|
||
destination, fb_kwargs, rebuild = _plan_fallback_candidate(
|
||
fb_client, fb_model, fb_label, task=task, effective_timeout=effective_timeout,
|
||
apply_fast_lane=False, messages=messages, tools=tools, temperature=temperature,
|
||
max_tokens=max_tokens, effective_extra_body=effective_extra_body,
|
||
reasoning_config=reasoning_config,
|
||
)
|
||
|
||
async def _send(client: Any, request_kwargs: Dict[str, Any], dest: _FallbackDestination) -> Any:
|
||
return _validate_llm_response(
|
||
await _relay_async_completion(client, request_kwargs, provider=dest.provider, api_mode=dest.api_mode),
|
||
task,
|
||
)
|
||
from agent.auxiliary_fallback_recovery import send_with_parameter_rungs_async
|
||
|
||
async def _send_recovering(client: Any, request_kwargs: Dict[str, Any], dest: _FallbackDestination) -> Any:
|
||
return await send_with_parameter_rungs_async(
|
||
lambda c, kw: _send(c, kw, dest), client, request_kwargs, task=task)
|
||
try:
|
||
return await _send_recovering(fb_client, fb_kwargs, destination)
|
||
except Exception as fb_err:
|
||
if not _is_auth_error(fb_err):
|
||
capacity = fallback_candidate_unavailable_reason(fb_err)
|
||
if capacity is None:
|
||
raise
|
||
_quarantine_fallback_candidate(
|
||
task, fb_label, destination.provider, fb_err, base_url=destination.base_url,
|
||
tag=" (async)", reason=capacity)
|
||
return None
|
||
fb_provider, retry = _plan_fallback_auth_retry(
|
||
destination, rebuild, async_mode=True, failed_api_key=getattr(fb_client, "api_key", ""))
|
||
failed_destination = destination
|
||
if retry is not None:
|
||
failed_destination = retry[2]
|
||
try:
|
||
return await _send_recovering(*retry)
|
||
except Exception as retry_err:
|
||
if not _is_auth_error(retry_err) and fallback_candidate_unavailable_reason(retry_err) is None:
|
||
raise
|
||
_quarantine_fallback_candidate(
|
||
task, fb_label, fb_provider, fb_err,
|
||
base_url=failed_destination.base_url, tag=" (async)",
|
||
)
|
||
return None
|
||
|
||
|
||
def _try_payment_fallback(
|
||
failed_provider: str, task: str = None, reason: str = "payment error", *,
|
||
failed_base_url: str = "", failure_scope: Any = None, main_runtime: Optional[Dict[str, Any]] = None,
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Try the auto-detection chain after a payment/credit or connection error, skipping the failed
|
||
provider (and the main-provider path when it maps to the same backend). Returns (client, model, label) or (None, None, "")."""
|
||
skip = failed_provider.lower().strip()
|
||
# The SESSION's provider decides whether discovery is allowed: a live `/model xai-oauth` session
|
||
# over a persisted ``provider: auto`` is a selection, so the disk value alone is not the answer.
|
||
main_provider = _normalize_main_runtime(main_runtime).get("provider") or _read_main_provider()
|
||
if not _discovery_chain_allowed(main_provider, task):
|
||
return None, None, ""
|
||
skip_labels = {skip}
|
||
if main_provider and main_provider.lower() in skip:
|
||
skip_labels.add(main_provider.lower())
|
||
skip_chain_labels = {_normalize_chain_label(s) for s in skip_labels}
|
||
skip_backend = _failed_backend_skip(
|
||
failed_provider, None, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||
tried = []
|
||
for label, try_fn in _get_provider_chain():
|
||
candidate_base_url = _custom_health_base_url(label)
|
||
if (not failed_base_url and label in skip_chain_labels) or skip_backend(
|
||
label, None, candidate_base_url):
|
||
continue
|
||
if _is_provider_unhealthy(label, candidate_base_url):
|
||
_log_skip_unhealthy(label, task, base_url=candidate_base_url)
|
||
tried.append(f"{label} (unhealthy)")
|
||
continue
|
||
client, model = try_fn()
|
||
if client is not None:
|
||
logger.info("Auxiliary %s: %s on %s — falling back to %s (%s)",
|
||
task or "call", reason, failed_provider, label, model or "default")
|
||
return client, model, label
|
||
tried.append(label)
|
||
logger.warning("Auxiliary %s: %s on %s and no fallback available (tried: %s)",
|
||
task or "call", reason, failed_provider, ", ".join(tried))
|
||
return None, None, ""
|
||
|
||
|
||
def _failed_backend_skip(
|
||
failed_provider: str, failed_model: Optional[str], *, failed_base_url: str = "",
|
||
failure_scope: Any = None,
|
||
) -> Callable[..., bool]:
|
||
"""Predicate ``skip(provider, model, base_url="")`` → True when a candidate must be skipped for the failed
|
||
route. Scope: ``failed_model`` → model-scoped (only that deployment; timeout/connection/rate-limit);
|
||
None → credential-wide (whole provider; auth/payment)."""
|
||
from agent.backend_identity import BackendIdentity, FailureScope, should_skip_candidate
|
||
skip_model = (failed_model or "").strip().lower() or None
|
||
failed_ident = BackendIdentity.build(
|
||
provider=failed_provider, model=skip_model, base_url=failed_base_url)
|
||
failure_scope = failure_scope or (FailureScope.MODEL if skip_model else FailureScope.CREDENTIAL)
|
||
|
||
def _skip(provider: str, model: Optional[str], base_url: str = "") -> bool:
|
||
return should_skip_candidate(
|
||
BackendIdentity.build(provider=provider, model=model, base_url=base_url), failed_ident, failure_scope,
|
||
)
|
||
return _skip
|
||
|
||
|
||
def _try_main_agent_model_fallback(
|
||
failed_provider: str, task: str = None, reason: str = "error",
|
||
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Last-resort fallback to the main agent provider + model after the configured chain is exhausted.
|
||
``failed_model`` scoping per ``_failed_backend_skip``; same-URL custom endpoints serve many models,
|
||
so a hung aux model says nothing about the main model's health. Returns (client, model, label) or (None, None, "")."""
|
||
main_provider = (_read_main_provider() or "").strip()
|
||
main_model = (_read_main_model() or "").strip()
|
||
if main_provider.lower() == "moa":
|
||
# MoA virtual provider: fall back to the preset's aggregator (the acting model).
|
||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||
if not _agg_provider or not _agg_model:
|
||
return None, None, ""
|
||
main_provider, main_model = _agg_provider, _agg_model
|
||
if not main_provider or not main_model or main_provider.lower() in {"auto", ""}:
|
||
return None, None, ""
|
||
if task == "vision" and (
|
||
main_provider in _PROVIDERS_WITHOUT_VISION or not _main_model_supports_vision(main_provider, main_model)):
|
||
# Same capability gate as the auto-route (_vision_main_provider_client): handing an image to a
|
||
# text-only main model turns a transient 429 into a guaranteed 400 (#108349).
|
||
logger.info("Auxiliary vision: %s on %s — main agent provider %s accepts no image input, not falling back",
|
||
reason, failed_provider, main_provider)
|
||
return None, None, ""
|
||
main_base_url = _custom_health_base_url(main_provider)
|
||
if _failed_backend_skip(
|
||
failed_provider, failed_model, failed_base_url=failed_base_url,
|
||
failure_scope=failure_scope)(main_provider, main_model, main_base_url):
|
||
return None, None, ""
|
||
if _is_provider_unhealthy(main_provider, main_base_url):
|
||
_log_skip_unhealthy(main_provider, task, base_url=main_base_url)
|
||
return None, None, ""
|
||
try:
|
||
client, resolved_model = resolve_provider_client(provider=main_provider, model=main_model)
|
||
except Exception:
|
||
client, resolved_model = None, None
|
||
if client is None:
|
||
return None, None, ""
|
||
label = f"main-agent({main_provider})"
|
||
logger.info("Auxiliary %s: %s on %s — falling back to main agent model %s (%s)",
|
||
task or "call", reason, failed_provider, label, resolved_model or main_model)
|
||
return client, resolved_model or main_model, label
|
||
|
||
|
||
# Context-window screening for runtime fallback chains: the startup feasibility check filters
|
||
# too-small aux models; runtime chains must too, or compression stops at a reachable-but-too-small
|
||
# candidate. ``None`` (unknown) passes through.
|
||
|
||
# ── Context-window screening for runtime fallback chains (issue #52392) ── When the runtime auxiliary
|
||
# fallback chain selects a candidate that is reachable but has a context window smaller than the compression
|
||
# task requires, the call errors out instead of continuing to the next, viable candidate. The startup
|
||
# feasibility check in ``agent.conversation_compression.check_compression_model_feasibility`` already
|
||
# filters too-small auxiliary models at startup, but the runtime fallback chain
|
||
# (``_try_configured_fallback_chain`` and ``_try_main_fallback_chain``) does not apply the same filter, so
|
||
# compression can stop at the first alive door even if the room behind it is too small. The helpers below
|
||
# screen each candidate by its effective context window before it is returned. ``None`` results from
|
||
# ``get_model_context_length`` are passed through (we cannot prove a model is too small, so we do not block
|
||
# it). This preserves the existing fallback surface for unrecognised/custom models while closing the gap on
|
||
# the well-known ones.
|
||
def _task_minimum_context_length(task: Optional[str]) -> Optional[int]:
|
||
"""Minimum context length for an auxiliary task; None = no floor (only ``compression`` has one)."""
|
||
return MINIMUM_CONTEXT_LENGTH if task == "compression" else None
|
||
|
||
|
||
def _candidate_context_window(provider: str, model: str, base_url: str = "", api_key: str = "") -> Optional[int]:
|
||
"""Best-effort context window for a fallback candidate; ``None`` = unknown (never raises; callers pass it through)."""
|
||
if not model:
|
||
return None
|
||
try:
|
||
ctx = get_model_context_length(model, base_url=base_url, api_key=api_key, provider=provider)
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary fallback: could not resolve context window for %s/%s: %s", provider, model, exc)
|
||
return None
|
||
return ctx if isinstance(ctx, int) and ctx > 0 else None
|
||
|
||
|
||
def _context_too_small(
|
||
entry: Dict[str, Any], provider: str, model: str, min_ctx: Optional[int], *,
|
||
task: Optional[str], label: str, name_model: bool = False,
|
||
) -> Optional[str]:
|
||
"""Screen one fallback candidate by context window; returns the ``tried`` note when it is too small."""
|
||
if min_ctx is None:
|
||
return None
|
||
fb_ctx = _candidate_context_window(
|
||
provider, model, base_url=str(entry.get("base_url") or ""), api_key=_fallback_entry_api_key(entry) or "")
|
||
if fb_ctx is None or fb_ctx >= min_ctx:
|
||
return None
|
||
if name_model:
|
||
logger.info("Auxiliary %s: skipping %s (%s context=%d < min=%d), continuing chain",
|
||
task, label, model, fb_ctx, min_ctx)
|
||
else:
|
||
logger.info("Auxiliary %s: skipping %s (context=%d < min=%d), continuing chain",
|
||
task or "call", label, fb_ctx, min_ctx)
|
||
return f"{label} (context too small: {fb_ctx}<{min_ctx})"
|
||
|
||
|
||
def _try_configured_fallback_chain(
|
||
task: str, failed_provider: str, reason: str = "error", failed_model: Optional[str] = None, *,
|
||
failed_base_url: str = "", failure_scope: Any = None,
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Try auxiliary.<task>.fallback_chain entries in order (each needs ``provider``; model/base_url/api_key optional).
|
||
``failed_model`` scoping per ``_failed_backend_skip`` (sibling models on the same provider still
|
||
run after a model-scoped failure). Returns (client, model, provider_label) or (None, None, "")."""
|
||
if not task:
|
||
return None, None, ""
|
||
chain = _get_auxiliary_task_config(task).get("fallback_chain")
|
||
if not chain or not isinstance(chain, list):
|
||
return None, None, ""
|
||
skip = _failed_backend_skip(
|
||
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||
tried = []
|
||
min_ctx = _task_minimum_context_length(task)
|
||
for i, entry in enumerate(chain):
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
fb_provider = str(entry.get("provider", "")).strip()
|
||
if not fb_provider:
|
||
continue
|
||
fb_model_raw = str(entry.get("model", "")).strip()
|
||
fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
|
||
if skip(fb_provider, fb_model_raw, fb_base_url):
|
||
continue
|
||
if _is_provider_unhealthy(fb_provider, fb_base_url):
|
||
_log_skip_unhealthy(fb_provider, task, base_url=fb_base_url)
|
||
tried.append(f"fallback_chain[{i}]({fb_provider}) (unhealthy)")
|
||
continue
|
||
fb_model = fb_model_raw or None
|
||
label = f"fallback_chain[{i}]({fb_provider})"
|
||
try:
|
||
fb_client, resolved_model = _resolve_fallback_entry(entry)
|
||
except Exception:
|
||
fb_client, resolved_model = None, None
|
||
if fb_client is not None:
|
||
too_small = _context_too_small(
|
||
entry, fb_provider, resolved_model, min_ctx, task=task, label=label, name_model=True,
|
||
) if resolved_model else None
|
||
if too_small:
|
||
tried.append(too_small)
|
||
continue
|
||
logger.info("Auxiliary %s: %s on %s — configured fallback to %s (%s)",
|
||
task, reason, failed_provider, label, resolved_model or fb_model or "default")
|
||
return fb_client, resolved_model or fb_model, label
|
||
tried.append(label)
|
||
if tried:
|
||
logger.debug("Auxiliary %s: configured fallback_chain exhausted (tried: %s)", task, ", ".join(tried))
|
||
return None, None, ""
|
||
|
||
|
||
def _try_configured_fallback_for_unavailable_client(
|
||
task: Optional[str], failed_provider: str
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Task fallback_chain when an explicit aux provider cannot build a client (no key/OAuth/pool creds);
|
||
stops at the per-task chain — the main-agent model stays the runtime last resort."""
|
||
explicit = (failed_provider or "").strip().lower()
|
||
if not task or not explicit or explicit in {"auto"}:
|
||
return None, None, ""
|
||
return _try_configured_fallback_chain(task, explicit, reason="provider unavailable")
|
||
|
||
|
||
def _fallback_entry_api_key(entry: Dict[str, Any]) -> Optional[str]:
|
||
"""Resolve inline or env-backed API key via the secret-scope-aware resolver (no raw os.getenv under multiplexing)."""
|
||
from hermes_cli.fallback_config import resolve_entry_api_key
|
||
return resolve_entry_api_key(entry)
|
||
|
||
|
||
def _resolve_fallback_entry(entry: Dict[str, Any]) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Resolve one fallback entry through the central provider router."""
|
||
provider = str(entry.get("provider") or "").strip()
|
||
model = str(entry.get("model") or "").strip() or None
|
||
if not provider or not model:
|
||
return None, None
|
||
client, resolved_model = resolve_provider_client(
|
||
provider, model=model, explicit_base_url=str(entry.get("base_url") or "").strip() or None,
|
||
explicit_api_key=_fallback_entry_api_key(entry),
|
||
api_mode=str(entry.get("api_mode") or entry.get("transport") or "").strip() or None,
|
||
)
|
||
if client is not None:
|
||
with contextlib.suppress(Exception):
|
||
client._hermes_fallback_destination = _fallback_destination_from_entry(entry, client, resolved_model)
|
||
return client, resolved_model
|
||
|
||
|
||
def _try_main_fallback_chain(
|
||
task: Optional[str], failed_provider: str = "", reason: str = "error", *,
|
||
failed_model: Optional[str] = None, failed_base_url: str = "", failure_scope: Any = None,
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Top-level main-agent fallback chain for a ``provider: auto`` auxiliary call: auto tasks honour the
|
||
user's main fallback policy before the built-in discovery chain; read via ``get_fallback_chain`` so
|
||
``fallback_providers`` and legacy ``fallback_model`` keep the main agent's order."""
|
||
try:
|
||
from hermes_cli.config import load_config_readonly
|
||
from hermes_cli.fallback_config import get_fallback_chain
|
||
chain = get_fallback_chain(load_config_readonly())
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary %s: could not load main fallback chain: %s", task or "call", exc)
|
||
return None, None, ""
|
||
if not chain:
|
||
return None, None, ""
|
||
skip = _failed_backend_skip(
|
||
failed_provider, failed_model, failed_base_url=failed_base_url, failure_scope=failure_scope)
|
||
tried: List[str] = []
|
||
min_ctx = _task_minimum_context_length(task)
|
||
for i, entry in enumerate(chain):
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
fb_provider = str(entry.get("provider") or "").strip()
|
||
fb_model = str(entry.get("model") or "").strip()
|
||
if not fb_provider or not fb_model:
|
||
continue
|
||
fb_norm = fb_provider.lower()
|
||
label = f"fallback_providers[{i}]({fb_provider})"
|
||
fb_base_url = _custom_health_base_url(fb_provider, entry.get("base_url"))
|
||
if fb_norm == "auto" or skip(fb_provider, fb_model, fb_base_url):
|
||
tried.append(f"{label} (skipped)")
|
||
continue
|
||
if _is_provider_unhealthy(fb_norm, fb_base_url):
|
||
_log_skip_unhealthy(fb_norm, task, base_url=fb_base_url)
|
||
tried.append(f"{label} (unhealthy)")
|
||
continue
|
||
try:
|
||
fb_client, resolved_model = _resolve_fallback_entry(entry)
|
||
except Exception as exc:
|
||
logger.debug("Auxiliary %s: main fallback %s failed to resolve: %s", task or "call", label, exc)
|
||
fb_client, resolved_model = None, None
|
||
if fb_client is not None:
|
||
too_small = _context_too_small(
|
||
entry, fb_provider, resolved_model or fb_model, min_ctx, task=task, label=label,
|
||
)
|
||
if too_small:
|
||
tried.append(too_small)
|
||
continue
|
||
logger.info("Auxiliary %s: %s on %s — main fallback chain to %s (%s)",
|
||
task or "call", reason, failed_provider or "auto", label, resolved_model or fb_model)
|
||
return fb_client, resolved_model or fb_model, fb_provider
|
||
tried.append(label)
|
||
if tried:
|
||
logger.debug("Auxiliary %s: main fallback chain exhausted (tried: %s)", task or "call", ", ".join(tried))
|
||
return None, None, ""
|
||
|
||
|
||
def _warn_stale_openai_base_url(runtime_provider: str) -> None:
|
||
"""Warn once when OPENAI_BASE_URL is set but config.yaml names a non-custom provider (a stale
|
||
~/.hermes/.env value after `hermes model` poisons routing)."""
|
||
global _stale_base_url_warned
|
||
if _stale_base_url_warned:
|
||
return
|
||
_env_base = os.getenv("OPENAI_BASE_URL", "").strip()
|
||
_cfg_provider = runtime_provider or _read_main_provider()
|
||
if (_env_base and _cfg_provider and _cfg_provider != "custom" and not _cfg_provider.startswith("custom:")):
|
||
logger.warning(
|
||
"OPENAI_BASE_URL is set (%s) but model.provider is '%s'. "
|
||
"Auxiliary clients may route to the wrong endpoint. "
|
||
"Run: hermes model to reconfigure, or remove "
|
||
"OPENAI_BASE_URL from ~/.hermes/.env",
|
||
_env_base, _cfg_provider,
|
||
)
|
||
_stale_base_url_warned = True
|
||
|
||
|
||
def _main_route_target(runtime: Dict[str, Any], task: Optional[str]) -> Tuple[str, str, str, Any, str]:
|
||
"""Step-1 target: (provider, model, base_url, api_key, api_mode) of the main runtime, after the
|
||
fast-model opt-in and the MoA aggregator substitution."""
|
||
main_provider = str(runtime.get("provider", "") or _read_main_provider() or "")
|
||
main_model = str(runtime.get("model") or _read_main_model() or "")
|
||
runtime_base_url = str(runtime.get("base_url") or "")
|
||
runtime_api_key = runtime.get("api_key", "")
|
||
runtime_api_mode = str(runtime.get("api_mode") or "")
|
||
# Latency-critical tasks (titling only) opt in to the provider's fast model. Opt-in only:
|
||
# every settings surface defines "auto" as the main model.
|
||
if _task_prefers_fast_model(task) and main_provider and main_provider not in {"auto", ""}:
|
||
fast_model = _get_aux_model_for_provider(main_provider, prefer_fast=True)
|
||
if fast_model and fast_model != main_model:
|
||
logger.debug("Auxiliary task %s: preferring fast model %s over main model %s",
|
||
task, fast_model, main_model)
|
||
main_model = fast_model
|
||
# MoA virtual provider: the preset name is not a wire model; run aux on the aggregator and drop
|
||
# the facade's "moa://local" base_url / placeholder key so it uses its own credentials.
|
||
if main_provider == "moa":
|
||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||
if _agg_provider and _agg_model:
|
||
main_provider, main_model = _agg_provider, _agg_model
|
||
runtime_base_url = runtime_api_key = runtime_api_mode = ""
|
||
return main_provider, main_model, runtime_base_url, runtime_api_key, runtime_api_mode
|
||
|
||
|
||
def _try_main_provider_route(
|
||
main_provider: str, main_model: str, runtime_base_url: str, runtime_api_key: Any, runtime_api_mode: str,
|
||
) -> Optional[Tuple[Any, str, str]]:
|
||
"""Step 1: route aux onto the main provider + main model; None if unusable."""
|
||
if not (main_provider and main_model and main_provider not in {"auto", ""}):
|
||
return None
|
||
resolved_provider = main_provider
|
||
explicit_base_url = runtime_base_url or None
|
||
health_base_url = _custom_health_base_url(main_provider, explicit_base_url)
|
||
explicit_api_key = None
|
||
if runtime_base_url and main_provider == "custom":
|
||
# Anonymous custom endpoint — pass through explicit base_url + api_key.
|
||
explicit_api_key = runtime_api_key or None
|
||
elif main_provider.startswith("custom:"):
|
||
# Named custom provider (custom_providers / providers dict entry).
|
||
_has_named_entry = False
|
||
with contextlib.suppress(ImportError):
|
||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||
_has_named_entry = _get_named_custom_provider(main_provider) is not None
|
||
if _has_named_entry:
|
||
# KEEP the full ``custom:<name>`` so the named arm honours the entry's api_mode
|
||
# (collapsing to "custom" strips /anthropic → 404s). base_url/api_key come from the entry.
|
||
explicit_base_url = None
|
||
elif runtime_base_url:
|
||
# Config-less named custom provider (live runtime only): anonymous custom arm + runtime key.
|
||
# See #34777.
|
||
resolved_provider = "custom"
|
||
explicit_api_key = runtime_api_key or None
|
||
elif runtime_api_key:
|
||
explicit_api_key = runtime_api_key
|
||
elif runtime_api_key:
|
||
# Pin aux to the main session's working key, not a re-selected (maybe exhausted) pool key.
|
||
explicit_api_key = runtime_api_key
|
||
# Skip if the main provider was recently 402'd (unhealthy TTL bounds the bypass).
|
||
main_chain_label = _normalize_chain_label(resolved_provider)
|
||
if main_chain_label and _is_provider_unhealthy(main_chain_label, health_base_url):
|
||
_log_skip_unhealthy(main_chain_label, base_url=health_base_url)
|
||
return None
|
||
client, resolved = resolve_provider_client(
|
||
resolved_provider, main_model, explicit_base_url=explicit_base_url,
|
||
explicit_api_key=explicit_api_key, api_mode=runtime_api_mode or None,
|
||
)
|
||
if client is None:
|
||
return None
|
||
logger.info("Auxiliary auto-detect: using main provider %s (%s)", main_provider, resolved or main_model)
|
||
return client, resolved or main_model, resolved_provider
|
||
|
||
|
||
def _discovery_chain_allowed(main_provider: str, task: Optional[str] = None) -> bool:
|
||
"""The built-in discovery chain is a convenience for installs with NO selected main provider.
|
||
Once the user picked one, every auxiliary route must be a provider they configured (main,
|
||
``auxiliary.<task>``, ``fallback_providers``); guessing "whatever else is logged in" bills an
|
||
account they never pointed this session at (xAI OAuth session with a dead token → every
|
||
compression silently charged to a Nous Portal balance)."""
|
||
if (main_provider or "").strip().lower() in {"", "auto"}:
|
||
return True
|
||
logger.warning(
|
||
"Auxiliary %s: main provider %s is unavailable and no fallback_chain / fallback_providers is "
|
||
"configured — refusing to guess another logged-in provider. Re-authenticate (`hermes model`) "
|
||
"or declare a fallback.", task or "call", main_provider)
|
||
return False
|
||
|
||
|
||
def _try_discovery_chain() -> Tuple[Optional[OpenAI], Optional[str], str]:
|
||
"""Step 3: hardcoded aggregator/fallback chain, skipping unhealthy providers."""
|
||
tried = []
|
||
for label, try_fn in _get_provider_chain():
|
||
candidate_base_url = _custom_health_base_url(label)
|
||
if _is_provider_unhealthy(label, candidate_base_url):
|
||
_log_skip_unhealthy(label, base_url=candidate_base_url)
|
||
tried.append(f"{label} (unhealthy)")
|
||
continue
|
||
client, model = try_fn()
|
||
if client is not None:
|
||
if tried:
|
||
logger.info("Auxiliary auto-detect: using %s (%s) — skipped: %s",
|
||
label, model or "default", ", ".join(tried))
|
||
else:
|
||
logger.info("Auxiliary auto-detect: using %s (%s)", label, model or "default")
|
||
return client, model, label
|
||
tried.append(label)
|
||
logger.warning("Auxiliary auto-detect: no provider available (tried: %s). "
|
||
"Compression, summarization, and memory flush will not work. "
|
||
"Set OPENROUTER_API_KEY or configure a local model in config.yaml.", ", ".join(tried))
|
||
return None, None, ""
|
||
|
||
|
||
def _resolve_auto_route(
|
||
main_runtime: Optional[Dict[str, Any]] = None, task: Optional[str] = None
|
||
) -> Tuple[Optional[OpenAI], Optional[str], str]:
|
||
"""Full auto-detection chain, including the selected provider identity. Priority: (1) main provider +
|
||
main model, regardless of provider type ("auto" means "my main model for side tasks too"; explicit
|
||
per-task overrides still win); (2) configured fallback policy — task chain, then the main agent's
|
||
top-level chain; (3) OpenRouter → Nous → custom → Codex → API-key providers, only with no policy
|
||
and no working main client."""
|
||
global auxiliary_is_nous
|
||
auxiliary_is_nous = False # Reset — _try_nous() will set True if it wins
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
_warn_stale_openai_base_url(runtime.get("provider", ""))
|
||
main_provider, main_model, base_url, api_key, api_mode = _main_route_target(runtime, task)
|
||
routed = _try_main_provider_route(main_provider, main_model, base_url, api_key, api_mode)
|
||
if routed is not None:
|
||
return routed
|
||
if task:
|
||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||
task, main_provider or "auto", reason="main provider unavailable")
|
||
if fb_client is not None:
|
||
return fb_client, fb_model, _fallback_provider_from_label(fb_label)
|
||
fb_client, fb_model, fb_label = _try_main_fallback_chain(
|
||
task, main_provider or "auto", reason="main provider unavailable")
|
||
if fb_client is not None:
|
||
return fb_client, fb_model, fb_label
|
||
if not _discovery_chain_allowed(main_provider, task):
|
||
return None, None, ""
|
||
return _try_discovery_chain()
|
||
|
||
|
||
def _effective_provider_for_client(client: Any, fallback: str) -> str:
|
||
"""Return the concrete provider selected for an auto-routed client."""
|
||
effective_provider = getattr(client, "_hermes_aux_effective_provider", "")
|
||
if isinstance(effective_provider, str) and effective_provider:
|
||
return effective_provider
|
||
return str(fallback or "")
|
||
|
||
|
||
# Centralized Provider Router: resolve_provider_client() is the single entry point for building a configured
|
||
# client (auth, base URL, headers, API format) from (provider, model). Never read auth env vars ad-hoc.
|
||
|
||
|
||
def _to_async_client(sync_client, model: str, is_vision: bool = False):
|
||
"""Sync client → async counterpart, preserving Codex routing (``is_vision`` adds the Copilot vision header)."""
|
||
from openai import AsyncOpenAI
|
||
if isinstance(sync_client, _AuxProbeClientStub):
|
||
return sync_client, model
|
||
if isinstance(sync_client, CodexAuxiliaryClient):
|
||
return AsyncCodexAuxiliaryClient(sync_client), model
|
||
if isinstance(sync_client, AnthropicAuxiliaryClient):
|
||
return AsyncAnthropicAuxiliaryClient(sync_client), model
|
||
if isinstance(sync_client, BedrockAuxiliaryClient):
|
||
return AsyncBedrockAuxiliaryClient(sync_client), model
|
||
with contextlib.suppress(ImportError):
|
||
from agent.gemini_native_adapter import GeminiNativeClient, AsyncGeminiNativeClient
|
||
if isinstance(sync_client, GeminiNativeClient):
|
||
return AsyncGeminiNativeClient(sync_client), model
|
||
# ACP shims (subprocess, not an HTTP pool) are already async-safe and opt out of the wrapper.
|
||
if _client_declares(sync_client, "HERMES_SKIP_ASYNC_WRAP"):
|
||
return sync_client, model
|
||
sync_base_url = str(sync_client.base_url)
|
||
# A key_cmd/Entra client keeps its credential in the SDK's per-request provider slot, not in
|
||
# ``.api_key`` (which stays ""); rebuilding from the snapshot alone ships NO Authorization
|
||
# header. Configured default_headers (a named entry's extra_headers) are likewise carried over,
|
||
# merged last exactly as the SDK merges them on the sync client. See #109595.
|
||
from agent.auxiliary_async_rebuild import async_api_key, configured_default_headers
|
||
async_kwargs = {"api_key": async_api_key(sync_client), "base_url": sync_base_url}
|
||
if base_url_host_matches(sync_base_url, "openrouter.ai"):
|
||
headers = _apply_user_default_headers(build_or_headers())
|
||
elif _is_official_codex_base_url(sync_base_url):
|
||
headers = _apply_user_default_headers(_codex_cloudflare_headers(sync_client.api_key, base_url=sync_base_url))
|
||
else:
|
||
# Provider for the profile-header fallback is inferred from the hostname.
|
||
try:
|
||
from agent.model_metadata import _infer_provider_from_url
|
||
inferred = _infer_provider_from_url(sync_base_url) or ""
|
||
except Exception:
|
||
inferred = ""
|
||
headers = _endpoint_default_headers(sync_base_url, inferred, is_vision=is_vision, xai=True)
|
||
headers = {**(headers or {}), **configured_default_headers(sync_client)}
|
||
if headers:
|
||
async_kwargs["default_headers"] = headers
|
||
_apply_required_codex_headers(async_kwargs, access_token=sync_client.api_key, base_url=sync_base_url)
|
||
async_kwargs = {**_openai_http_client_kwargs(sync_base_url, async_mode=True), **async_kwargs}
|
||
# Hermes owns the auxiliary retry/timeout budget; disable SDK-internal retries.
|
||
# See #54465.
|
||
async_kwargs.setdefault("max_retries", 0)
|
||
return AsyncOpenAI(**async_kwargs), model
|
||
|
||
|
||
def _normalize_resolved_model(model_name: Optional[str], provider: str) -> Optional[str]:
|
||
"""Normalize a resolved model for the provider that will receive it."""
|
||
if not model_name:
|
||
return model_name
|
||
try:
|
||
from hermes_cli.model_normalize import normalize_model_for_provider
|
||
return normalize_model_for_provider(model_name, provider)
|
||
except Exception:
|
||
return model_name
|
||
|
||
|
||
def _named_custom_api_key(custom_entry: Dict[str, Any], provider: str, custom_base: str) -> Any:
|
||
"""Credential for a named custom provider: inline api_key → key_env → key_cmd → credential pool → placeholder.
|
||
Aux resolves named custom providers here, not via _resolve_named_custom_runtime, so key_cmd must be
|
||
honoured at the same precedence or every aux call 401s."""
|
||
custom_key: Any = (custom_entry.get("api_key") or "").strip()
|
||
custom_key_env = (custom_entry.get("key_env") or custom_entry.get("api_key_env") or "").strip()
|
||
if not custom_key and custom_key_env:
|
||
custom_key = _scoped_key_env(custom_key_env)
|
||
custom_key_cmd = str(custom_entry.get("key_cmd", "") or "").strip()
|
||
if custom_key_cmd:
|
||
from agent.command_token_source import build_command_token_provider
|
||
custom_key = build_command_token_provider(custom_key_cmd, custom_entry.get("name") or provider) or custom_key
|
||
if not custom_key:
|
||
with contextlib.suppress(Exception):
|
||
from agent.credential_pool import custom_provider_pool_key_candidates
|
||
pool_name = custom_entry.get("provider_key") or custom_entry.get("name") or provider
|
||
for pool_key in custom_provider_pool_key_candidates(custom_base, pool_name):
|
||
try:
|
||
pool = load_pool(pool_key)
|
||
except Exception:
|
||
continue
|
||
if not pool.has_credentials():
|
||
continue
|
||
pool_entry = pool.select()
|
||
if pool_entry is None:
|
||
continue
|
||
pool_api_key = getattr(pool_entry, "runtime_api_key", None) or getattr(pool_entry, "access_token", "") or ""
|
||
if str(pool_api_key).strip():
|
||
custom_key = str(pool_api_key).strip()
|
||
break
|
||
return custom_key or "no-key-required"
|
||
|
||
|
||
def _build_bedrock_client(provider: str, model: Optional[str], *, raw_codex: bool) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""AWS Bedrock: Claude → Anthropic Bedrock SDK (prompt caching, thinking); OpenAI models
|
||
(GPT-5.5/5.6) → Bedrock Mantle's OpenAI Responses endpoint; everything else → Converse API."""
|
||
try:
|
||
from agent.bedrock_adapter import (
|
||
has_aws_credentials, is_anthropic_bedrock_model, resolve_bedrock_runtime_region,
|
||
is_openai_bedrock_model, bedrock_openai_base_url, resolve_bedrock_bearer_token,
|
||
configure_bedrock_openai_client_kwargs,
|
||
)
|
||
from agent.anthropic_adapter import build_anthropic_bedrock_client
|
||
except ImportError:
|
||
logger.warning("resolve_provider_client: bedrock requested but boto3, httpx/openai, or anthropic SDK not installed")
|
||
return None, None
|
||
if not has_aws_credentials():
|
||
logger.debug("resolve_provider_client: bedrock requested but no AWS credentials found")
|
||
return None, None
|
||
# Region must match the main runtime's resolution (bedrock.region in config first, then
|
||
# env/profile) so aux calls never leave the primary runtime's configured region.
|
||
# See #53880, #65076.
|
||
region = resolve_bedrock_runtime_region()
|
||
default_model = "anthropic.claude-haiku-4-5-20251001-v1:0"
|
||
final_model = _normalize_resolved_model(model or default_model, provider) or default_model
|
||
if is_openai_bedrock_model(final_model):
|
||
# Module-level lazy ``OpenAI`` proxy on purpose so tests can patch("agent.auxiliary_client.OpenAI").
|
||
client_kwargs: Dict[str, Any] = {
|
||
"api_key": resolve_bedrock_bearer_token() or "aws-sdk",
|
||
"base_url": bedrock_openai_base_url(region),
|
||
}
|
||
configure_bedrock_openai_client_kwargs(client_kwargs)
|
||
client = OpenAI(**client_kwargs)
|
||
logger.debug("resolve_provider_client: bedrock-openai (%s, %s)", final_model, region)
|
||
return (client if raw_codex else CodexAuxiliaryClient(client, final_model)), final_model
|
||
base_url = f"https://bedrock-runtime.{region}.amazonaws.com"
|
||
if is_anthropic_bedrock_model(final_model):
|
||
try:
|
||
real_client = build_anthropic_bedrock_client(region)
|
||
except ImportError as exc:
|
||
logger.warning("resolve_provider_client: cannot create Bedrock client: %s", exc)
|
||
return None, None
|
||
client = AnthropicAuxiliaryClient(real_client, final_model, api_key="aws-sdk", base_url=base_url)
|
||
logger.debug("resolve_provider_client: bedrock anthropic (%s, %s)", final_model, region)
|
||
else:
|
||
client = BedrockAuxiliaryClient(region, final_model)
|
||
logger.debug("resolve_provider_client: bedrock converse (%s, %s)", final_model, region)
|
||
return client, final_model
|
||
|
||
|
||
def _build_vertex_client(provider: str, model: Optional[str]) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Google Vertex AI: Gemini via the OpenAI-compatible endpoint with an OAuth2 bearer (standard OpenAI client)."""
|
||
try:
|
||
from agent.vertex_adapter import get_vertex_config, has_vertex_credentials
|
||
except ImportError:
|
||
logger.warning("resolve_provider_client: vertex requested but google-auth not installed")
|
||
return None, None
|
||
if not has_vertex_credentials():
|
||
logger.debug("resolve_provider_client: vertex requested but no GCP credentials found")
|
||
return None, None
|
||
token, base_url = get_vertex_config()
|
||
if not token or not base_url:
|
||
logger.warning("resolve_provider_client: vertex requested but could not mint token / resolve project")
|
||
return None, None
|
||
final_model = _normalize_resolved_model(model or "google/gemini-3-flash-preview", provider)
|
||
try:
|
||
# Aliased import: a bare `from openai import OpenAI` would shadow the module-level lazy proxy.
|
||
from openai import OpenAI as _VertexOpenAI
|
||
client = _VertexOpenAI(api_key=token, base_url=base_url)
|
||
except Exception as exc:
|
||
logger.warning("resolve_provider_client: cannot create Vertex client: %s", exc)
|
||
return None, None
|
||
logger.debug("resolve_provider_client: vertex (%s)", final_model)
|
||
return client, final_model
|
||
|
||
|
||
class _ResolveRequest(NamedTuple):
|
||
"""Normalized resolve_provider_client() arguments shared by the per-provider branch helpers."""
|
||
provider: str
|
||
original_provider: str
|
||
model: Optional[str]
|
||
async_mode: bool
|
||
raw_codex: bool
|
||
explicit_base_url: Optional[str]
|
||
explicit_api_key: Optional[Union[str, Callable[[], str]]]
|
||
api_mode: Optional[str]
|
||
main_runtime: Optional[Dict[str, Any]]
|
||
is_vision: bool
|
||
task: Optional[str]
|
||
|
||
|
||
_ResolveResult = Tuple[Optional[Any], Optional[str]]
|
||
|
||
|
||
def _normalize_api_key(raw: Any) -> Union[str, Callable[[], str]]:
|
||
"""A key_cmd/Entra callable passes through uncalled; strings are stripped; anything else is ''."""
|
||
if callable(raw) and not isinstance(raw, str):
|
||
return raw
|
||
return raw.strip() if isinstance(raw, str) else ""
|
||
|
||
|
||
def _log_once_debug(seen: set, key: Any, msg: str, *args: Any) -> None:
|
||
"""Debug-log ``msg`` the first time ``key`` is seen so per-call retries stay silent."""
|
||
if key not in seen:
|
||
seen.add(key)
|
||
logger.debug(msg, *args)
|
||
|
||
|
||
def _is_actual_auxiliary_route(req: _ResolveRequest, base_url: str) -> bool:
|
||
from hermes_cli.auth import normalize_actual_base_url
|
||
from hermes_cli.providers import is_actual_route
|
||
from hermes_cli.route_identity import normalize_route_base_url
|
||
|
||
if is_actual_route(req.provider, base_url):
|
||
return True
|
||
runtime = _normalize_main_runtime(req.main_runtime)
|
||
return bool(
|
||
base_url
|
||
and is_actual_route(runtime.get("provider", ""), runtime.get("base_url", ""))
|
||
and normalize_route_base_url(normalize_actual_base_url(base_url))
|
||
== normalize_route_base_url(
|
||
normalize_actual_base_url(runtime.get("base_url", ""))
|
||
)
|
||
)
|
||
|
||
|
||
def _wrap_transport(req: _ResolveRequest, client_obj: Any, final_model_str: str,
|
||
base_url_str: str = "", api_key_str: str = ""):
|
||
"""Wrap a plain OpenAI client in the right transport adapter; specialized wrappers pass through.
|
||
Codex (Responses API): explicit ``api_mode=codex_responses``, else — with no
|
||
explicit api_mode — api.openai.com + codex model. Anthropic (Messages): ``api_mode=anthropic_messages``,
|
||
any ``/anthropic`` suffix, ``api.kimi.com/coding``, or ``api.anthropic.com``."""
|
||
if _is_actual_auxiliary_route(req, base_url_str):
|
||
client = (
|
||
client_obj._real_client
|
||
if isinstance(client_obj, CodexAuxiliaryClient)
|
||
else client_obj
|
||
)
|
||
client._hermes_aux_effective_provider = "actual"
|
||
return client
|
||
# OpenCode relay targets pick the wire per model; a task/provider-level api_mode is stale for
|
||
# every other model (#98799), so it is re-derived here like the main runtime does.
|
||
from agent.opencode_affinity import opencode_transport
|
||
_oc_mode, _oc_base = opencode_transport(req.provider, final_model_str, base_url_str)
|
||
if _oc_mode:
|
||
req, base_url_str = req._replace(api_mode=_oc_mode), _oc_base
|
||
needs_codex = not (
|
||
isinstance(client_obj, CodexAuxiliaryClient) or req.raw_codex
|
||
) and (
|
||
req.api_mode == "codex_responses"
|
||
or (
|
||
not req.api_mode
|
||
and base_url_hostname(base_url_str) == "api.openai.com"
|
||
and "codex" in (final_model_str or "").lower()
|
||
)
|
||
)
|
||
if needs_codex:
|
||
logger.debug("resolve_provider_client: wrapping client in CodexAuxiliaryClient "
|
||
"(api_mode=%s, model=%s, base_url=%s)",
|
||
req.api_mode or "auto-detected", final_model_str, base_url_str[:60] if base_url_str else "")
|
||
return CodexAuxiliaryClient(client_obj, final_model_str)
|
||
# A profile that declares the Messages wire (commandcode-anthropic) is on it whatever the URL
|
||
# looks like; the same declaration gates ``_reasoning_config`` in _build_call_kwargs.
|
||
api_mode = req.api_mode or _profile_declared_messages_wire(req.provider)
|
||
return _maybe_wrap_anthropic(client_obj, final_model_str, api_key_str, base_url_str, api_mode)
|
||
|
||
|
||
def _profile_declared_messages_wire(provider: str) -> Optional[str]:
|
||
"""``"anthropic_messages"`` when the registered profile declares that api_mode, else None."""
|
||
from providers import get_provider_profile
|
||
profile = get_provider_profile(str(provider or "").strip().lower())
|
||
return "anthropic_messages" if profile is not None and profile.api_mode == "anthropic_messages" else None
|
||
|
||
|
||
def _route_client(req: _ResolveRequest, client_obj: Any, final_model_str: Optional[str]) -> _ResolveResult:
|
||
"""Return (client, model), converting to the async wrapper when ``req.async_mode``."""
|
||
if req.async_mode:
|
||
return _to_async_client(client_obj, final_model_str, is_vision=req.is_vision)
|
||
return client_obj, final_model_str
|
||
|
||
|
||
def _route_or_warn(req: _ResolveRequest, client: Any, default: Optional[str], unavailable_msg: str, *args: Any) -> _ResolveResult:
|
||
"""Route ``client`` on ``req.model or default``; warn and return (None, None) when the provider produced no client."""
|
||
if client is None:
|
||
logger.warning(unavailable_msg, *args)
|
||
return None, None
|
||
return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
|
||
|
||
|
||
def _resolve_auto_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""Auto: try all providers in priority order; tag the client with the effective provider (survives cache reuse)."""
|
||
client, resolved, effective_provider = _resolve_auto_route(main_runtime=req.main_runtime, task=req.task)
|
||
if client is None:
|
||
return None, None
|
||
model = req.model
|
||
# An OpenRouter-format model override won't work on a non-OpenRouter provider (e.g. local
|
||
# server); drop it for the provider's default.
|
||
if model and "/" in model and resolved and "/" not in resolved:
|
||
logger.debug("Dropping OpenRouter-format model %r for non-OpenRouter "
|
||
"auxiliary provider (using %r instead)", model, resolved)
|
||
model = None
|
||
routed_client, routed_model = _route_client(req, client, model or resolved)
|
||
if routed_client is not None and effective_provider:
|
||
try:
|
||
setattr(routed_client, "_hermes_aux_effective_provider", effective_provider)
|
||
except (AttributeError, TypeError):
|
||
logger.debug("Auxiliary client %s cannot retain effective provider %s",
|
||
type(routed_client).__name__, effective_provider)
|
||
return routed_client, routed_model
|
||
|
||
|
||
def _resolve_openrouter_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""OpenRouter."""
|
||
client, default = _try_openrouter(explicit_api_key=req.explicit_api_key, model=req.model,
|
||
explicit_base_url=req.explicit_base_url)
|
||
if client is None:
|
||
logger.warning("resolve_provider_client: openrouter requested but %s",
|
||
_describe_openrouter_unavailable(model=req.model))
|
||
return None, None
|
||
return _route_client(req, client, _normalize_resolved_model(req.model or default, req.provider))
|
||
|
||
|
||
def _resolve_nous_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""Nous Portal (OAuth)."""
|
||
model = req.model
|
||
# Vision: caller flag, _PROVIDER_VISION_MODELS override, or a known vision id.
|
||
client, default = _try_nous(vision=(req.is_vision or model in _PROVIDER_VISION_MODELS.values()
|
||
or (model or "").strip().lower() == "mimo-v2-omni"))
|
||
if client is None:
|
||
logger.warning("resolve_provider_client: nous requested but Nous Portal not configured (run: hermes auth)")
|
||
return None, None
|
||
final_model = _normalize_resolved_model(model or default, req.provider)
|
||
# Dual-wire: anthropic/* → /v1/messages, else /chat/completions. Derive from the catalog id
|
||
# (not a stale api_mode) so aux matches the main agent.
|
||
from hermes_cli.providers import nous_api_mode
|
||
client = _maybe_wrap_anthropic(
|
||
client, final_model, str(getattr(client, "api_key", "") or ""),
|
||
str(getattr(client, "base_url", "") or ""), nous_api_mode(final_model),
|
||
)
|
||
return _route_client(req, client, final_model)
|
||
|
||
|
||
def _resolve_openai_codex_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""OpenAI Codex (OAuth → Responses API)."""
|
||
model = req.model
|
||
if not model:
|
||
logger.warning("resolve_provider_client: openai-codex requested without a "
|
||
"model; pass model explicitly (e.g. model.model in config.yaml "
|
||
"or auxiliary.<task>.model for per-task aux routing).")
|
||
return None, None
|
||
no_token_msg = "resolve_provider_client: openai-codex requested but no Codex OAuth token found (run: hermes model)"
|
||
if req.raw_codex:
|
||
# Raw OpenAI client for callers needing responses.stream() (main agent loop).
|
||
codex_token, base_url = _resolve_codex_credential_and_base()
|
||
if not codex_token:
|
||
logger.warning(no_token_msg)
|
||
return None, None
|
||
raw_client = _create_openai_client(api_key=codex_token, base_url=base_url,
|
||
default_headers=_codex_cloudflare_headers(codex_token, base_url=base_url))
|
||
return raw_client, _normalize_resolved_model(model, req.provider)
|
||
client, default = _build_codex_client(model)
|
||
return _route_or_warn(req, client, default, no_token_msg)
|
||
|
||
|
||
def _resolve_xai_oauth_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""xAI Grok OAuth (device code → Responses API). Without this branch xai-oauth falls to the generic
|
||
oauth_external arm, returns (None, None), and silently re-routes every aux task to the Step-2 fallback."""
|
||
client, default = _build_xai_oauth_aux_client(req.model)
|
||
return _route_or_warn(req, client, default,
|
||
"resolve_provider_client: xai-oauth requested but no xAI "
|
||
"OAuth token found (run: hermes model -> xAI Grok OAuth — SuperGrok / Premium+)")
|
||
|
||
|
||
def _resolve_custom_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""Custom endpoint (OPENAI_BASE_URL + OPENAI_API_KEY)."""
|
||
provider, model, main_runtime = req.provider, req.model, req.main_runtime
|
||
# wrap_base: base for the Anthropic-wrap decision. anthropic_messages must keep the raw
|
||
# /anthropic base while the plain OpenAI client uses the /v1-rewritten custom_base (never
|
||
# /anthropic/chat/completions). Empty means "use custom_base".
|
||
custom_base = custom_key = wrap_base = ""
|
||
if req.explicit_base_url:
|
||
custom_base = _to_openai_base_url(req.explicit_base_url).strip()
|
||
# Ollama/vLLM/llama.cpp serve the OpenAI surface under /v1; a bare host posts to the
|
||
# native API and 404s (#106010). Only the alias group gets the tail — ``custom`` is verbatim.
|
||
if req.original_provider in _LOCAL_SERVER_ALIASES and not urlparse(custom_base).path.strip("/"):
|
||
custom_base = custom_base.rstrip("/") + "/v1"
|
||
if req.api_mode == "anthropic_messages":
|
||
wrap_base = (req.explicit_base_url or "").strip().rstrip("/")
|
||
if req.original_provider in _LOCAL_SERVER_ALIASES:
|
||
# SECURITY: a local-server alias never borrows OPENAI_API_KEY or the main key —
|
||
# the alias means "this is my own server"; sending an OpenAI secret to whatever
|
||
# host base_url names is never intended. Explicit api_key or the placeholder only.
|
||
custom_key = _normalize_api_key(req.explicit_api_key) or "no-key-required"
|
||
else:
|
||
custom_key = (
|
||
_normalize_api_key(req.explicit_api_key)
|
||
or _scoped_key_env("OPENAI_API_KEY")
|
||
or _read_main_api_key_if_same_host(custom_base)
|
||
or "no-key-required" # local servers don't need auth
|
||
)
|
||
if not custom_base:
|
||
logger.warning("resolve_provider_client: explicit custom endpoint requested but base_url is empty")
|
||
return None, None
|
||
elif main_runtime:
|
||
# Reuse main_runtime's concrete base_url + api_key for a named custom provider;
|
||
# re-resolving from bare "custom" loses the name and lands on the wrong provider.
|
||
# Re-resolution loses the provider name and falls back to OpenRouter or a wrong API-key provider —
|
||
# the main agent already solved this, we just need to reuse its answer. (#45472)
|
||
_main_base = str(main_runtime.get("base_url") or "").strip().rstrip("/")
|
||
_main_key = _normalize_api_key(main_runtime.get("api_key"))
|
||
if _main_base and _main_key:
|
||
custom_base, custom_key = _main_base, _main_key
|
||
if custom_base and custom_key:
|
||
if _is_actual_auxiliary_route(req, custom_base):
|
||
from hermes_cli.auth import normalize_actual_base_url
|
||
custom_base = normalize_actual_base_url(custom_base)
|
||
final_model = _normalize_resolved_model(
|
||
model or (main_runtime.get("model") if main_runtime else None) or "gpt-4o-mini", provider,
|
||
)
|
||
extra = {}
|
||
_clean_base, _dq = _extract_url_query_params(custom_base)
|
||
if _dq:
|
||
extra["default_query"] = _dq
|
||
_custom_headers = _endpoint_default_headers(custom_base, provider, is_vision=req.is_vision)
|
||
if _custom_headers:
|
||
extra["default_headers"] = _custom_headers
|
||
client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **extra)
|
||
client = _wrap_transport(req, client, final_model, wrap_base or custom_base, custom_key)
|
||
return _route_client(req, client, final_model)
|
||
# Try custom first, then API-key providers (Codex excluded here:
|
||
# falling through to Codex with no model is a stale-constant trap).
|
||
for try_fn in (_try_custom_endpoint, _resolve_api_key_provider):
|
||
client, default = try_fn()
|
||
if client is not None:
|
||
final_model = _normalize_resolved_model(model or default, provider)
|
||
# ``client.api_key`` may be a callable (Azure Entra bearer provider);
|
||
# wrapping decisions only need base_url + api_mode.
|
||
_raw_ckey = getattr(client, "api_key", "")
|
||
_ckey = "" if (callable(_raw_ckey) and not isinstance(_raw_ckey, str)) else str(_raw_ckey or "")
|
||
client = _wrap_transport(req, client, final_model, str(getattr(client, "base_url", "") or ""), _ckey)
|
||
return _route_client(req, client, final_model)
|
||
logger.warning("resolve_provider_client: custom/main requested but no endpoint credentials found")
|
||
return None, None
|
||
|
||
|
||
def _named_custom_openai_wire_client(custom_base: str, custom_key: Any, extra_headers: Optional[Dict[str, str]] = None):
|
||
"""Plain OpenAI client on the /v1 equivalent of a named custom entry's base URL.
|
||
|
||
``extra_headers`` is the entry's own header block (gateway routing tags, proxy auth): the main
|
||
runtime lifts it onto every request, so aux must too (#109595). SECURITY: never log the values."""
|
||
_clean_base, _dq = _extract_url_query_params(_to_openai_base_url(custom_base))
|
||
_extra = {"default_query": _dq} if _dq else {}
|
||
_headers = _apply_user_default_headers(None)
|
||
if extra_headers:
|
||
_headers = {**(_headers or {}), **extra_headers}
|
||
if _headers:
|
||
_extra["default_headers"] = _headers
|
||
return _create_openai_client(api_key=custom_key, base_url=_clean_base, **_extra)
|
||
|
||
|
||
def _resolve_named_custom_branch(req: _ResolveRequest) -> Optional[_ResolveResult]:
|
||
"""Named custom provider (config.yaml providers dict / custom_providers list); None if no entry matches."""
|
||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||
provider = req.provider
|
||
# If the raw name is an alias (``kimi`` → ``kimi-coding``) and a custom_providers entry exists
|
||
# under it, the custom entry wins over alias rewriting. Only for aliases, so entries matching a
|
||
# canonical name (e.g. ``nous``) still defer to the built-in.
|
||
custom_entry = None
|
||
if req.original_provider and req.original_provider != provider:
|
||
custom_entry = _get_named_custom_provider(req.original_provider)
|
||
if custom_entry is None:
|
||
custom_entry = _get_named_custom_provider(provider)
|
||
if not custom_entry:
|
||
return None
|
||
# A per-task/explicit base_url or api_key composes OVER the named entry's defaults: the entry supplies
|
||
# whatever the caller left blank, never replaces what the caller set (compression prompts carry
|
||
# conversation history, so a silently swapped destination is a data-routing bug, not a nuisance).
|
||
custom_base = (req.explicit_base_url or custom_entry.get("base_url") or "").strip()
|
||
custom_key = _normalize_api_key(req.explicit_api_key) or _named_custom_api_key(custom_entry, provider, custom_base)
|
||
if custom_key == "no-key-required":
|
||
logger.warning("resolve_provider_client: named custom provider %r has no resolvable "
|
||
"api_key — request will be sent with placeholder no-key-required "
|
||
"and will 401 on auth-required endpoints", custom_entry.get("name") or provider)
|
||
# Actual's wire protocol takes precedence over persisted task/provider modes.
|
||
entry_api_mode = (req.api_mode or custom_entry.get("api_mode") or "").strip()
|
||
if _is_actual_auxiliary_route(req, custom_base):
|
||
from hermes_cli.auth import normalize_actual_base_url
|
||
custom_base = normalize_actual_base_url(custom_base)
|
||
entry_api_mode = "chat_completions"
|
||
if not custom_base:
|
||
logger.warning("resolve_provider_client: named custom provider %r has no base_url", provider)
|
||
return None, None
|
||
from hermes_cli.config import normalize_extra_headers
|
||
entry_headers = normalize_extra_headers(custom_entry.get("extra_headers"))
|
||
final_model = _normalize_resolved_model(
|
||
req.model
|
||
or custom_entry.get("model")
|
||
or (req.main_runtime.get("model") if req.main_runtime else None)
|
||
or _read_main_model_for_aux()
|
||
or "gpt-4o-mini",
|
||
provider,
|
||
)
|
||
# An OpenCode-family entry (``opencode-go-bridge``, #85589) persisted the api_mode of whichever
|
||
# model was selected at save time; the relay picks the wire per model (#98799).
|
||
from agent.opencode_affinity import opencode_transport
|
||
_oc_mode, _oc_base = opencode_transport(provider, final_model, custom_base)
|
||
if _oc_mode:
|
||
entry_api_mode, custom_base = _oc_mode, _oc_base
|
||
logger.debug("resolve_provider_client: named custom provider %r (%s, api_mode=%s)",
|
||
provider, final_model, entry_api_mode or "chat_completions")
|
||
# anthropic_messages: route via AnthropicAuxiliaryClient (mirrors _try_custom_endpoint);
|
||
# the Anthropic SDK sees the original (un-rewritten) URL.
|
||
# Mirrors the anonymous-custom branch in _try_custom_endpoint(). See #15033.
|
||
if entry_api_mode == "anthropic_messages":
|
||
try:
|
||
from agent.anthropic_adapter import build_anthropic_client
|
||
from agent.anthropic_credentials import anthropic_route_is_oauth
|
||
real_client = build_anthropic_client(custom_key, custom_base)
|
||
if entry_headers:
|
||
# Same entry headers as the two OpenAI-wire arms; ``with_options`` merges onto the
|
||
# beta/credential-Omit headers the builder installed (#109595).
|
||
real_client = real_client.with_options(default_headers=entry_headers)
|
||
except ImportError:
|
||
logger.warning("Named custom provider %r declares api_mode=anthropic_messages but the anthropic SDK "
|
||
"is not installed — falling back to OpenAI-wire.", provider)
|
||
return _route_client(req, _named_custom_openai_wire_client(custom_base, custom_key, entry_headers), final_model)
|
||
return _route_client(
|
||
req, AnthropicAuxiliaryClient(real_client, final_model, custom_key, custom_base,
|
||
is_oauth=anthropic_route_is_oauth(custom_base, custom_key)), final_model)
|
||
client = _named_custom_openai_wire_client(custom_base, custom_key, entry_headers)
|
||
# codex_responses, or auto-detect via _wrap_transport (which reads the task-level api_mode).
|
||
if entry_api_mode == "codex_responses":
|
||
client = CodexAuxiliaryClient(client, final_model)
|
||
else:
|
||
client = _wrap_transport(req, client, final_model, custom_base, custom_key)
|
||
return _route_client(req, client, final_model)
|
||
|
||
|
||
def _resolve_azure_foundry_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""Azure Foundry via the runtime resolver: the generic PROVIDER_REGISTRY path only knows the static
|
||
AZURE_FOUNDRY_API_KEY env var, missing ``auth_mode: entra_id`` (callable bearer) and config base_url overrides."""
|
||
client, default_model = _try_azure_foundry(model=req.model, explicit_api_key=req.explicit_api_key,
|
||
explicit_base_url=req.explicit_base_url, api_mode=req.api_mode)
|
||
return _route_or_warn(req, client, default_model,
|
||
"resolve_provider_client: azure-foundry requested but "
|
||
"runtime resolution failed (run: hermes doctor for diagnostics)")
|
||
|
||
|
||
def _api_key_profile_supplied_client(provider: str, **client_kwargs: Any) -> Any | None:
|
||
"""Registered profile's own client for an ``api_key`` aux route, or ``None``.
|
||
|
||
Same registration seam as ``agent_runtime_helpers._provider_supplied_client`` (main agent)
|
||
and the ``external_process`` branch below: a profile whose wire protocol is not
|
||
OpenAI-over-HTTP overrides ``ProviderProfile.create_client()`` to supply its transport.
|
||
A profile that raises is logged and skipped — a third-party plugin can only fail to
|
||
provide a client, never take the auxiliary resolution down."""
|
||
try:
|
||
from providers import get_provider_profile
|
||
profile = get_provider_profile(provider)
|
||
except Exception:
|
||
return None
|
||
if profile is None:
|
||
return None
|
||
try:
|
||
return profile.create_client(**client_kwargs)
|
||
except Exception:
|
||
logger.warning("resolve_provider_client: provider profile %r failed to create an "
|
||
"auxiliary client; falling back to the standard client path",
|
||
provider, exc_info=True)
|
||
return None
|
||
|
||
|
||
def _resolve_api_key_branch(req: _ResolveRequest, pconfig: Any, resolve_creds: Callable) -> _ResolveResult:
|
||
"""PROVIDER_REGISTRY ``api_key`` providers (Anthropic via its own resolver), honouring explicit overrides."""
|
||
provider = req.provider
|
||
if provider == "anthropic":
|
||
client, default_model = _try_anthropic(explicit_api_key=req.explicit_api_key,
|
||
explicit_base_url=req.explicit_base_url)
|
||
return _route_or_warn(req, client, default_model,
|
||
"resolve_provider_client: anthropic requested but no Anthropic credentials found")
|
||
creds = resolve_creds(provider)
|
||
api_key = str(creds.get("api_key", "")).strip()
|
||
# Explicit api_key override (fallback_model / custom_providers entry) lets callers
|
||
# authenticate where no built-in credential is registered for this alias.
|
||
api_key = _normalize_api_key(req.explicit_api_key) or api_key
|
||
raw_base_url = str(creds.get("base_url", "")).strip().rstrip("/") or pconfig.inference_base_url
|
||
if req.explicit_base_url:
|
||
raw_base_url = req.explicit_base_url.strip().rstrip("/")
|
||
if provider == "actual":
|
||
with contextlib.suppress(Exception):
|
||
from hermes_cli.auth import (
|
||
ACTUAL_LOCAL_NOAUTH_PLACEHOLDER, is_actual_local_base_url, normalize_actual_base_url
|
||
)
|
||
raw_base_url = normalize_actual_base_url(raw_base_url)
|
||
if not api_key and is_actual_local_base_url(raw_base_url):
|
||
api_key = ACTUAL_LOCAL_NOAUTH_PLACEHOLDER
|
||
if not api_key:
|
||
tried_sources = list(pconfig.api_key_env_vars) + (["gh auth token"] if provider == "copilot" else [])
|
||
logger.debug("resolve_provider_client: provider %s has no API key configured (tried: %s)",
|
||
provider, ", ".join(tried_sources))
|
||
return None, None
|
||
base_url = _to_openai_base_url(raw_base_url)
|
||
# Explicit base_url override: a fallback_model/custom_providers entry pointing a built-in name elsewhere.
|
||
if req.explicit_base_url and provider != "actual":
|
||
base_url = _to_openai_base_url(req.explicit_base_url.strip().rstrip("/"))
|
||
final_model = _normalize_resolved_model(req.model or _get_aux_model_for_provider(provider), provider)
|
||
# Consulted before the built-in gemini/OpenAI ladder so a registered native transport wins (#112384).
|
||
profile_client = _api_key_profile_supplied_client(provider, api_key=api_key, base_url=base_url)
|
||
if profile_client is not None:
|
||
logger.debug("resolve_provider_client: %s native client from provider profile (%s)", provider, final_model)
|
||
return _route_client(req, profile_client, final_model)
|
||
if provider == "gemini":
|
||
from agent.gemini_native_adapter import GeminiNativeClient, is_native_gemini_base_url
|
||
if is_native_gemini_base_url(base_url):
|
||
client = GeminiNativeClient(api_key=api_key, base_url=base_url)
|
||
logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
|
||
return _route_client(req, client, final_model)
|
||
headers = _endpoint_default_headers(base_url, provider, is_vision=req.is_vision, xai=True)
|
||
client = _create_openai_client(api_key=api_key, base_url=base_url, **({"default_headers": headers} if headers else {}))
|
||
# Copilot GPT-5+ models (except gpt-5-mini) are only reachable via the Responses API;
|
||
# wrap so call_llm() transparently routes through responses.stream().
|
||
if provider == "copilot" and final_model and not req.raw_codex:
|
||
with contextlib.suppress(ImportError):
|
||
from hermes_cli.models import _should_use_copilot_responses_api
|
||
if _should_use_copilot_responses_api(final_model):
|
||
logger.debug("resolve_provider_client: copilot model %s needs "
|
||
"Responses API — wrapping with CodexAuxiliaryClient", final_model)
|
||
client = CodexAuxiliaryClient(client, final_model)
|
||
# api_mode handling for any API-key provider (direct OpenAI + codex model) and Anthropic-wire
|
||
# endpoints (api.kimi.com/coding, /anthropic gateways) without per-provider branches.
|
||
client = _wrap_transport(req, client, final_model, raw_base_url, api_key)
|
||
logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
|
||
return _route_client(req, client, final_model)
|
||
|
||
|
||
def _resolve_external_process_branch(req: _ResolveRequest, creds: Dict[str, Any]) -> _ResolveResult:
|
||
"""PROVIDER_REGISTRY ``external_process`` providers, served via their registered profile."""
|
||
provider = req.provider
|
||
final_model = _normalize_resolved_model(
|
||
req.model or (req.main_runtime.get("model") if req.main_runtime else None) or _read_main_model_for_aux(),
|
||
provider,
|
||
)
|
||
# Keyed on the registered profile, not a provider name, so an out-of-tree ACP provider reaches
|
||
# the auxiliary path (compression, vision, background review) exactly like the in-tree one.
|
||
try:
|
||
from providers import get_provider_profile as _get_provider_profile
|
||
_extproc_profile = _get_provider_profile(provider)
|
||
except Exception:
|
||
_extproc_profile = None
|
||
if _extproc_profile is not None:
|
||
api_key = str(creds.get("api_key", "")).strip()
|
||
base_url = str(creds.get("base_url", "")).strip()
|
||
if not final_model:
|
||
logger.warning("resolve_provider_client: %s requested but no model was provided or configured", provider)
|
||
return None, None
|
||
if not api_key or not base_url:
|
||
logger.warning("resolve_provider_client: %s requested but external process credentials are incomplete", provider)
|
||
return None, None
|
||
try:
|
||
client = _extproc_profile.create_client(
|
||
api_key=api_key, base_url=base_url,
|
||
command=str(creds.get("command", "")).strip() or None, args=list(creds.get("args") or []))
|
||
except Exception:
|
||
logger.warning("resolve_provider_client: profile %r failed to create an external-process client",
|
||
provider, exc_info=True)
|
||
client = None
|
||
if client is not None:
|
||
logger.debug("resolve_provider_client: %s (%s)", provider, final_model)
|
||
return _route_client(req, client, final_model)
|
||
_log_once_debug(_LOGGED_UNSUPPORTED_EXTPROC_KEYS, provider,
|
||
"resolve_provider_client: external-process provider %s not "
|
||
"directly supported", provider)
|
||
return None, None
|
||
|
||
|
||
def _resolve_registry_branch(req: _ResolveRequest) -> _ResolveResult:
|
||
"""PROVIDER_REGISTRY providers, dispatched on ``auth_type``; unknown providers log once."""
|
||
provider = req.provider
|
||
try:
|
||
from hermes_cli.auth import (
|
||
PROVIDER_REGISTRY, resolve_api_key_provider_credentials,
|
||
resolve_external_process_provider_credentials,
|
||
)
|
||
except ImportError:
|
||
logger.debug("hermes_cli.auth not available for provider %s", provider)
|
||
return None, None
|
||
pconfig = PROVIDER_REGISTRY.get(provider)
|
||
if pconfig is None:
|
||
_log_once_debug(_LOGGED_UNKNOWN_PROVIDER_KEYS, provider,
|
||
"resolve_provider_client: unknown provider %r", provider)
|
||
return None, None
|
||
auth_type = pconfig.auth_type
|
||
if auth_type == "api_key":
|
||
return _resolve_api_key_branch(req, pconfig, resolve_api_key_provider_credentials)
|
||
if auth_type == "external_process":
|
||
return _resolve_external_process_branch(req, resolve_external_process_provider_credentials(provider))
|
||
if auth_type == "vertex":
|
||
client, final_model = _build_vertex_client(provider, req.model)
|
||
elif auth_type == "aws_sdk":
|
||
client, final_model = _build_bedrock_client(provider, req.model, raw_codex=req.raw_codex)
|
||
elif auth_type in {"oauth_device_code", "oauth_external"}:
|
||
# nous / openai-codex / xai-oauth already returned from their explicit branches.
|
||
_log_once_debug(_LOGGED_UNSUPPORTED_OAUTH_KEYS, provider,
|
||
"resolve_provider_client: OAuth provider %s not "
|
||
"directly supported, try 'auto'", provider)
|
||
return None, None
|
||
else:
|
||
# The first occurrence surfaces a real schema-drift bug; per-call retries stay silent.
|
||
_log_once_debug(_LOGGED_UNHANDLED_AUTHTYPE_KEYS, (auth_type, provider),
|
||
"resolve_provider_client: unhandled auth_type %s for %s",
|
||
auth_type, provider)
|
||
return None, None
|
||
return _route_client(req, client, final_model) if client is not None else (None, None)
|
||
|
||
|
||
# Explicit providers with a dedicated branch; anything else falls through to named custom
|
||
# providers → azure-foundry → PROVIDER_REGISTRY (order preserved from the original if-chain).
|
||
_EXPLICIT_PROVIDER_BRANCHES: Dict[str, Callable[[_ResolveRequest], _ResolveResult]] = {
|
||
"auto": _resolve_auto_branch,
|
||
"openrouter": _resolve_openrouter_branch,
|
||
"nous": _resolve_nous_branch,
|
||
"openai-codex": _resolve_openai_codex_branch,
|
||
"xai-oauth": _resolve_xai_oauth_branch,
|
||
"custom": _resolve_custom_branch,
|
||
}
|
||
|
||
|
||
def resolve_provider_client(
|
||
provider: str, model: str = None, async_mode: bool = False, raw_codex: bool = False,
|
||
explicit_base_url: str = None, explicit_api_key: Optional[Union[str, Callable[[], str]]] = None,
|
||
api_mode: str = None, main_runtime: Optional[Dict[str, Any]] = None, is_vision: bool = False,
|
||
task: Optional[str] = None,
|
||
) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Central router: return a configured client (auth, base URL, API format) for a provider + optional model.
|
||
The client always exposes ``.chat.completions.create()``; Codex/Responses providers get an adapter.
|
||
``provider``: built-in name, ``custom:<name>``, "custom" (OPENAI_BASE_URL + OPENAI_API_KEY) or "auto"
|
||
(full auto-detection chain). ``model=None`` → provider's default aux model. ``raw_codex`` → bare OpenAI
|
||
client for ``responses.stream()`` callers. ``api_mode`` forces "codex_responses"/"chat_completions"/
|
||
"anthropic_messages" instead of auto-detect. Returns (client, resolved_model) or (None, None)."""
|
||
_validate_proxy_env_urls()
|
||
# Keep the pre-alias name so a custom_providers entry named like a built-in alias
|
||
# (e.g. "kimi" → "kimi-coding") is still reachable via the named-custom branch.
|
||
original_provider = (provider or "").strip().lower()
|
||
provider = _normalize_aux_provider(provider)
|
||
api_mode = _canonical_api_mode(str(api_mode or "")).lower() or None
|
||
# MoA chokepoint: "moa" is not an HTTP provider; resolve to the aggregator so direct callers don't
|
||
# dead-end in unknown-provider. Unresolvable preset → leave untouched for the normal diagnostic.
|
||
if provider == "moa":
|
||
_agg_provider, _agg_model = _resolve_moa_aggregator(model)
|
||
if _agg_provider and _agg_model:
|
||
original_provider = _agg_provider.strip().lower()
|
||
provider = _normalize_aux_provider(_agg_provider)
|
||
model = _agg_model
|
||
# The moa:// facade endpoint/key belong to the virtual runtime, not the aggregator.
|
||
if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"):
|
||
explicit_base_url = None
|
||
explicit_api_key = None
|
||
# Model for concrete providers: caller ``model`` → catalog default (empty for OAuth-gated providers whose
|
||
# lists drift) → configured main model (MoA → aggregator), keeping OAuth aux tasks off the Step-2 fallback.
|
||
# Excluded: ``auto`` (a stale main slug could pair with any picked provider) and Nous + vision (the
|
||
# Portal's tier-aware vision recommendation must win over a text-only model).
|
||
if not model and provider != "auto" and not (provider == "nous" and is_vision):
|
||
# ``auto`` is intentionally excluded: `_resolve_auto_route(main_runtime=...)` returns the model paired
|
||
# with the provider it actually selected. Pre-filling an auto call from `_read_main_model()` can
|
||
# leak a stale process-global runtime into a different provider (for example Claude model slug on
|
||
# Codex OAuth) and override that correctly resolved model. 1. ``model`` argument (caller knew what
|
||
# they wanted) 2. Provider's catalog default — cheap/fast model the provider registered via
|
||
# ``ProviderProfile.default_aux_model`` or the legacy ``_API_KEY_PROVIDER_AUX_MODELS_FALLBACK``
|
||
# dict. 3. User's main model from ``model.model`` in config.yaml. This is the load-bearing step for
|
||
# OAuth providers: an xai-oauth user with grok-4.3 configured gets grok-4.3 for title generation
|
||
# instead of silently dropping to whatever Step-2 fallback (#31845). When the main provider is MoA,
|
||
# ``_read_main_model_for_aux()`` substitutes the preset's aggregator model — the preset NAME is
|
||
# never a valid wire model id, so unset aux models default to the preset's acting model instead.
|
||
# Each provider branch below sees a non-empty ``model`` whenever the user has *anything* configured
|
||
# — no provider-specific empty-model guards needed. When the user has NOTHING configured (fresh
|
||
# install, main_model also empty), the branches still hit their own missing-credentials returns and
|
||
# ``_resolve_auto_route`` falls through to the Step-2 chain as before. Do NOT pre-fill a blank ``auto``
|
||
# request from the config/main default here. Claude model sent to Codex after the main lane fell
|
||
# back to gpt-5.5). Let _resolve_auto_route() return the actual current runtime model when the caller did
|
||
# not explicitly request one. (# compression-current-model) Nous + vision is the one carve-out: the
|
||
# branch below resolves its model from the Portal's tier-aware vision recommendation
|
||
# (``_try_nous(vision= True)``), and ``final_model = model or default`` means anything pre-filled
|
||
# here wins over that. The main chat model is routinely text-only (e.g. a ``:free`` chat SKU), so
|
||
# pre-filling it sends the image to a model that cannot accept one and the Portal 404s. Leave
|
||
# ``model`` unset and let the Portal slot through; only an explicit caller model may override it.
|
||
model = _get_aux_model_for_provider(provider) or _read_main_model_for_aux() or model
|
||
req = _ResolveRequest(
|
||
provider, original_provider, model, async_mode, raw_codex,
|
||
explicit_base_url, explicit_api_key, api_mode, main_runtime, is_vision, task,
|
||
)
|
||
branch = _EXPLICIT_PROVIDER_BRANCHES.get(provider)
|
||
alias_identity = original_provider.removeprefix("custom:")
|
||
# A configured provider whose name collides with a local-server alias is still a named
|
||
# provider. Resolve it before the alias's generic ``custom`` branch, which otherwise loses
|
||
# the configured base_url and can fall through to an unrelated credentialed provider.
|
||
if branch is None or alias_identity in _LOCAL_SERVER_ALIASES:
|
||
try:
|
||
result = _resolve_named_custom_branch(req)
|
||
except ImportError:
|
||
result = None
|
||
if result is not None:
|
||
return result
|
||
if branch is not None:
|
||
return branch(req)
|
||
if provider == "azure-foundry":
|
||
return _resolve_azure_foundry_branch(req)
|
||
return _resolve_registry_branch(req)
|
||
|
||
|
||
# ── Public API ──────────────────────────────────────────────────────────────
|
||
|
||
def get_text_auxiliary_client(task: str = "", *, main_runtime: Optional[Dict[str, Any]] = None) -> Tuple[Optional[OpenAI], Optional[str]]:
|
||
"""Return (client, default_model_slug) for text-only aux tasks; ``task`` selects auxiliary.<task> overrides."""
|
||
provider, model, base_url, api_key, api_mode = _resolve_task_provider_model(task or None)
|
||
return resolve_provider_client(
|
||
provider, model=model, explicit_base_url=base_url, explicit_api_key=api_key,
|
||
api_mode=api_mode, main_runtime=main_runtime,
|
||
)
|
||
|
||
|
||
_VISION_AUTO_PROVIDER_ORDER = ("openrouter", "nous", "deepinfra")
|
||
|
||
|
||
def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool:
|
||
"""True when ``provider``/``model`` is known to accept image input; unknown capability → True (attempt the call)."""
|
||
try:
|
||
from agent.image_routing import _lookup_supports_vision
|
||
from hermes_cli.config import load_config_readonly
|
||
except ImportError:
|
||
return True
|
||
try:
|
||
supports = _lookup_supports_vision(provider, model, load_config_readonly())
|
||
except Exception: # pragma: no cover - defensive
|
||
return True
|
||
return True if supports is None else bool(supports)
|
||
|
||
|
||
def _normalize_vision_provider(provider: Optional[str]) -> str:
|
||
return _normalize_aux_provider(provider)
|
||
|
||
|
||
def _deepinfra_strict_vision_backend(model: Optional[str]) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""DeepInfra vision: default model is discovered live via default_vision_model() so no hardcoded id can rot."""
|
||
vision_model = model or _resolve_provider_vision_default("deepinfra")
|
||
if not vision_model:
|
||
logger.debug("Vision auto-detect: deepinfra catalog unreachable or returned no vision-tagged models — skipping")
|
||
return None, None
|
||
return resolve_provider_client("deepinfra", vision_model, is_vision=True)
|
||
|
||
|
||
# Strict (explicitly requested) vision backends by normalized provider name. nous MUST go
|
||
# through resolve_provider_client so anthropic/* picks wrap onto /v1/messages (a bare _try_nous
|
||
# client 404s). openai-codex has no safe default model; callers set auxiliary.<task>.model.
|
||
_STRICT_VISION_BACKENDS: Dict[str, Callable[[Optional[str]], Tuple[Optional[Any], Optional[str]]]] = {
|
||
"copilot": lambda model: resolve_provider_client("copilot", model, is_vision=True),
|
||
"openrouter": lambda model: _try_openrouter(model=model),
|
||
"nous": lambda model: resolve_provider_client("nous", model, is_vision=True),
|
||
"openai-codex": lambda model: resolve_provider_client("openai-codex", model, is_vision=True),
|
||
"anthropic": lambda model: _try_anthropic(),
|
||
"deepinfra": _deepinfra_strict_vision_backend,
|
||
"custom": lambda model: _try_custom_endpoint(),
|
||
}
|
||
|
||
|
||
def _resolve_strict_vision_backend(provider: str, model: Optional[str] = None) -> Tuple[Optional[Any], Optional[str]]:
|
||
backend = _STRICT_VISION_BACKENDS.get(_normalize_vision_provider(provider))
|
||
return backend(model) if backend is not None else (None, None)
|
||
|
||
|
||
def get_available_vision_backends() -> List[str]:
|
||
"""Available vision backends in auto-selection order (active provider → OpenRouter → Nous → DeepInfra).
|
||
|
||
Single source of truth for setup, tool gating, and runtime auto-routing.
|
||
"""
|
||
available: List[str] = []
|
||
main_provider = _read_main_provider()
|
||
if main_provider and main_provider not in {"auto", ""}:
|
||
if main_provider in _VISION_AUTO_PROVIDER_ORDER:
|
||
main_ok = _resolve_strict_vision_backend(main_provider)[0] is not None
|
||
else:
|
||
main_ok = resolve_provider_client(main_provider, _read_main_model())[0] is not None
|
||
if main_ok:
|
||
available.append(main_provider)
|
||
for p in _VISION_AUTO_PROVIDER_ORDER: # skip if already covered by main provider
|
||
if p not in available and _resolve_strict_vision_backend(p)[0] is not None:
|
||
available.append(p)
|
||
return available
|
||
|
||
|
||
def _finalize_vision_client(
|
||
resolved_provider: str, sync_client: Any, default_model: Optional[str],
|
||
resolved_model: Optional[str], async_mode: bool,
|
||
) -> Tuple[Optional[str], Optional[Any], Optional[str]]:
|
||
"""Apply the explicit model override (and async wrapping) to a resolved vision client."""
|
||
if sync_client is None:
|
||
return resolved_provider, None, None
|
||
final_model = resolved_model or default_model
|
||
if async_mode:
|
||
async_client, async_model = _to_async_client(sync_client, final_model, is_vision=True)
|
||
return resolved_provider, async_client, async_model
|
||
return resolved_provider, sync_client, final_model
|
||
|
||
|
||
def _vision_main_provider_client(
|
||
main_provider: str, main_model: str, runtime: Dict[str, Any], resolved_model: Optional[str],
|
||
resolved_api_mode: Optional[str],
|
||
) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Auto-detect step 1: try the main provider; (None, None) falls through to the aggregator chain."""
|
||
# A provider vision default (static override or catalog discovery) is a *known* multimodal
|
||
# model; the pinned chat model usually isn't, so only fall back to it when no default exists.
|
||
provider_vision_default = _resolve_provider_vision_default(main_provider)
|
||
vision_model = provider_vision_default or main_model
|
||
if main_provider == "nous":
|
||
# Nous picks its vision model from Portal tier-aware slots inside _try_nous(vision=True);
|
||
# passing the chat model would override that and 404. Only auxiliary.vision.model may.
|
||
sync_client, default_model = _resolve_strict_vision_backend(main_provider, resolved_model or provider_vision_default)
|
||
if sync_client is None:
|
||
return None, None
|
||
logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, default_model or resolved_model or main_model)
|
||
return sync_client, default_model
|
||
if main_provider in _PROVIDERS_WITHOUT_VISION: # endpoint rejects image input entirely
|
||
logger.debug("Vision auto-detect: skipping main provider %s (no vision support) — falling through to aggregator chain", main_provider)
|
||
return None, None
|
||
if not _main_model_supports_vision(main_provider, vision_model):
|
||
# Known text-only model. Log only the provider name (CodeQL clear-text-logging FPs).
|
||
logger.debug(
|
||
"Vision auto-detect: skipping main provider %s (reports no vision capability) — falling through to aggregator chain",
|
||
main_provider,
|
||
)
|
||
return None, None
|
||
# Custom endpoints carry no built-in base_url/api_key: recover the live main endpoint from
|
||
# set_runtime_main() or, with no live runtime recorded, the configured custom endpoint.
|
||
rpc_base_url = rpc_api_key = None
|
||
rpc_api_mode = resolved_api_mode
|
||
if main_provider == "custom" or main_provider.startswith("custom:"):
|
||
if runtime.get("base_url"):
|
||
custom_base, custom_key, custom_mode = runtime.get("base_url"), runtime.get("api_key") or None, runtime.get("api_mode")
|
||
else:
|
||
custom_base, custom_key, custom_mode = _resolve_custom_runtime()
|
||
if custom_base:
|
||
rpc_base_url, rpc_api_key = custom_base, custom_key
|
||
rpc_api_mode = resolved_api_mode or custom_mode or None
|
||
rpc_client, rpc_model = resolve_provider_client(
|
||
main_provider, vision_model, api_mode=rpc_api_mode, explicit_base_url=rpc_base_url,
|
||
explicit_api_key=rpc_api_key, main_runtime=runtime, is_vision=True)
|
||
if rpc_client is None:
|
||
return None, None
|
||
logger.info("Vision auto-detect: using main provider %s (%s)", main_provider, rpc_model or vision_model)
|
||
return rpc_client, rpc_model or vision_model
|
||
|
||
|
||
def _vision_auto_route(
|
||
runtime: Dict[str, Any], resolved_model: Optional[str], resolved_api_mode: Optional[str],
|
||
async_mode: bool,
|
||
) -> Tuple[Optional[str], Optional[Any], Optional[str]]:
|
||
"""Auto-detect order: 1. main provider + model, 2. OpenRouter, 3. Nous Portal, 4. DeepInfra, 5. stop."""
|
||
main_provider = str(runtime.get("provider") or _read_main_provider())
|
||
main_model = str(runtime.get("model") or _read_main_model())
|
||
if main_provider.strip().lower() == "moa":
|
||
# MoA main_model is a preset NAME, not a wire model — unwrap to the preset's aggregator
|
||
# slot. The moa:// facade endpoint belongs to the virtual provider, not the real one.
|
||
_agg_provider, _agg_model = _resolve_moa_aggregator(main_model)
|
||
if _agg_provider and _agg_model:
|
||
main_provider, main_model = _agg_provider, _agg_model
|
||
runtime = dict(runtime, base_url="", api_key="", api_mode="")
|
||
if main_provider and main_provider not in {"auto", "", "moa"}:
|
||
client, default_model = _vision_main_provider_client(main_provider, main_model, runtime, resolved_model, resolved_api_mode)
|
||
if client is not None:
|
||
return _finalize_vision_client(main_provider, client, default_model, resolved_model, async_mode)
|
||
# Aggregators use their dedicated vision model, not the user's main model.
|
||
for candidate in _VISION_AUTO_PROVIDER_ORDER:
|
||
if candidate == main_provider:
|
||
continue # already tried above
|
||
sync_client, default_model = _resolve_strict_vision_backend(candidate)
|
||
if sync_client is not None:
|
||
return _finalize_vision_client(candidate, sync_client, default_model, resolved_model, async_mode)
|
||
logger.debug("Auxiliary vision client: none available")
|
||
return None, None, None
|
||
|
||
|
||
# ZAI vision must use the OpenAI-compatible endpoint: the Anthropic wire rejects max_tokens on
|
||
# multimodal calls (error 1210).
|
||
_ZAI_OPENAI_VISION_URLS = ("https://open.bigmodel.cn/api/paas/v4", "https://api.z.ai/api/paas/v4")
|
||
|
||
|
||
def resolve_vision_provider_client(
|
||
provider: Optional[str] = None, model: Optional[str] = None, *, base_url: Optional[str] = None,
|
||
api_key: Optional[str] = None, async_mode: bool = False,
|
||
main_runtime: Optional[Dict[str, Any]] = None,
|
||
) -> Tuple[Optional[str], Optional[Any], Optional[str]]:
|
||
"""Resolve the client actually used for vision tasks.
|
||
|
||
Direct endpoint overrides beat provider selection; explicit providers may force
|
||
experimental backends; auto mode only tries backends known to work.
|
||
"""
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
requested, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
|
||
"vision", provider, model, base_url, api_key
|
||
)
|
||
requested = _normalize_vision_provider(requested)
|
||
if resolved_base_url:
|
||
provider_for_base_override = requested if requested and requested not in {"", "auto"} else "custom"
|
||
client, final_model = resolve_provider_client(
|
||
provider_for_base_override, model=resolved_model, async_mode=async_mode,
|
||
explicit_base_url=resolved_base_url, explicit_api_key=resolved_api_key,
|
||
api_mode=resolved_api_mode, main_runtime=runtime,
|
||
)
|
||
return provider_for_base_override, client, (final_model if client is not None else None)
|
||
if requested == "auto":
|
||
return _vision_auto_route(runtime, resolved_model, resolved_api_mode, async_mode)
|
||
if requested in _VISION_AUTO_PROVIDER_ORDER:
|
||
sync_client, default_model = _resolve_strict_vision_backend(requested, resolved_model)
|
||
return _finalize_vision_client(requested, sync_client, default_model, resolved_model, async_mode)
|
||
if requested == "zai":
|
||
for _zai_url in _ZAI_OPENAI_VISION_URLS:
|
||
client, final_model = _get_cached_client(
|
||
requested, resolved_model, async_mode, base_url=_zai_url,
|
||
api_key=resolved_api_key or None, api_mode="chat_completions", main_runtime=runtime,
|
||
is_vision=True,
|
||
)
|
||
if client is not None:
|
||
return _finalize_vision_client(requested, client, final_model, resolved_model, async_mode)
|
||
# Fallback: try without explicit base_url (old behavior)
|
||
client, final_model = _get_cached_client(
|
||
requested, resolved_model, async_mode, api_mode=resolved_api_mode, main_runtime=runtime, is_vision=True,
|
||
)
|
||
return requested, client, (final_model if client is not None else None)
|
||
|
||
|
||
def get_auxiliary_extra_body() -> dict:
|
||
"""Return extra_body kwargs (Nous Portal product tags when Nous-backed, else {})."""
|
||
return _nous_extra_body() if auxiliary_is_nous else {}
|
||
|
||
|
||
def auxiliary_max_tokens_param(value: int, *, model: Optional[str] = None) -> dict:
|
||
"""Max-tokens kwarg for the auxiliary provider: direct OpenAI/Copilot and newer OpenAI-family
|
||
models (by ``model`` name, so custom endpoints fronting gpt-5.x are caught) need max_completion_tokens."""
|
||
_custom_host = base_url_hostname(_current_custom_base_url()) or ""
|
||
direct_openai_family = (
|
||
not _scoped_key_env("OPENROUTER_API_KEY") and _read_nous_auth() is None
|
||
and (_custom_host in ("api.openai.com", "api.githubcopilot.com") or _custom_host.endswith(".githubcopilot.com"))
|
||
)
|
||
if direct_openai_family or model_forces_max_completion_tokens(model):
|
||
return {"max_completion_tokens": value}
|
||
return {"max_tokens": value}
|
||
|
||
|
||
# ── Centralized LLM Call API: call_llm()/async_call_llm() own resolve → cached client → shape
|
||
# request → call → return. Every auxiliary LLM consumer should use these.
|
||
|
||
# Client cache: (provider, async_mode, base_url, api_key, api_mode, runtime_key) -> (client, default_model, loop)
|
||
# Loop identity is NOT part of the key: stale-loop entries are replaced in place on async hits,
|
||
# bounding growth to one entry per provider config (avoids fd accumulation in gateways).
|
||
# This bounds cache growth to one entry per unique provider config rather than one per (config ×
|
||
# event-loop), which previously caused unbounded fd accumulation in long-running gateway processes (#10200).
|
||
_client_cache: Dict[tuple, tuple] = {}
|
||
_client_cache_lock = threading.Lock()
|
||
_CLIENT_CACHE_MAX_SIZE = 64 # safety belt — evict oldest when exceeded
|
||
|
||
|
||
class _CallableCacheDiscriminator:
|
||
"""Hash a credential callback by identity without exposing its state."""
|
||
|
||
__slots__ = ("_callback",)
|
||
|
||
def __init__(self, callback: Any) -> None:
|
||
self._callback = callback # retained so its id cannot be reused while cached
|
||
|
||
def __hash__(self) -> int:
|
||
return id(self._callback)
|
||
|
||
def __eq__(self, other: object) -> bool:
|
||
return isinstance(other, _CallableCacheDiscriminator) and self._callback is other._callback
|
||
|
||
def __repr__(self) -> str:
|
||
return "<callable-api-key>"
|
||
|
||
|
||
def _runtime_cache_discriminator(field: str, value: Any) -> Any:
|
||
"""Return a hashable, secret-safe runtime cache-key component."""
|
||
if field == "api_key" and callable(value):
|
||
return _CallableCacheDiscriminator(value)
|
||
if field == "api_key" and isinstance(value, str) and value:
|
||
return ("api-key-digest", hashlib.blake2b(value.encode("utf-8"), digest_size=16).digest())
|
||
return value
|
||
|
||
|
||
def _client_cache_key(
|
||
provider: str, *, async_mode: bool, base_url: Optional[str] = None,
|
||
api_key: Optional[str] = None, api_mode: Optional[str] = None,
|
||
main_runtime: Optional[Dict[str, Any]] = None, is_vision: bool = False,
|
||
task: Optional[str] = None, model: Optional[str] = None,
|
||
) -> tuple:
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
# `auto` resolves through the main runtime and task-specific policy, so both join the key.
|
||
runtime_key = tuple(_runtime_cache_discriminator(f, runtime.get(f, "")) for f in _MAIN_RUNTIME_FIELDS) if provider == "auto" else ()
|
||
task_key = (task or "", _task_prefers_fast_model(task)) if provider == "auto" else ""
|
||
pool_hint = _pool_cache_hint(provider, main_runtime=main_runtime)
|
||
# Model MUST be in the key: concurrent calls to the same endpoint with different models would
|
||
# share an entry, and the second builder's _store_cached_client would close the first's client.
|
||
model_key = model or runtime.get("model", "")
|
||
api_key_key = _runtime_cache_discriminator("api_key", api_key or "")
|
||
# Profile home leads the key: callers that omit api_key (pool / Nous auth.json paths) would
|
||
# otherwise share one client across multiplex profiles holding different credentials.
|
||
return (hermes_home_key(), provider, async_mode, base_url or "", api_key_key, api_mode or "", runtime_key, is_vision, task_key, pool_hint, model_key)
|
||
|
||
|
||
def _current_event_loop() -> Any:
|
||
"""``asyncio.get_event_loop()`` or None when no loop can be obtained (async cache-key binding)."""
|
||
try:
|
||
import asyncio as _aio
|
||
return _aio.get_event_loop()
|
||
except RuntimeError:
|
||
return None
|
||
|
||
|
||
def _store_cached_client(cache_key: tuple, client: Any, default_model: Optional[str], *, bound_loop: Any = None) -> None:
|
||
if isinstance(client, _AuxProbeClientStub):
|
||
return # probe stubs must never be cached — the next hit would get a dud client
|
||
with _client_cache_lock:
|
||
old_entry = _client_cache.get(cache_key)
|
||
if old_entry is not None and old_entry[0] is not client:
|
||
_close_cached_client(old_entry[0])
|
||
_client_cache[cache_key] = (client, default_model, bound_loop)
|
||
|
||
|
||
def _refresh_nous_auxiliary_client(
|
||
*, cache_provider: str, model: Optional[str], async_mode: bool, base_url: Optional[str] = None,
|
||
api_key: Optional[str] = None, api_mode: Optional[str] = None,
|
||
main_runtime: Optional[Dict[str, Any]] = None, is_vision: bool = False,
|
||
lookup_model: Optional[str] = None, lookup_task: Optional[str] = None,
|
||
) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Refresh Nous runtime creds, rebuild the client, and replace the cache entry.
|
||
|
||
``model`` is the resolved wire model stored as the entry's usable model and returned. The
|
||
cache KEY MUST be built from ``lookup_model``/``lookup_task`` — the model and task as passed
|
||
to ``_get_cached_client`` when the stale client was acquired — so the fresh client overwrites
|
||
the exact entry the stale one is served from. Keying on the resolved model or an empty task
|
||
would leave the expired client immortal and every auxiliary call 401ing forever.
|
||
|
||
See #56889.
|
||
For ``provider == "auto"`` the task participates in the cache key (task-specific fallback policy), so it
|
||
MUST be carried into the key here for the same reason as ``lookup_model``; otherwise an auto-provider
|
||
client refreshed on a 401 lands under the ``task=""`` key while the stale entry survives under the
|
||
task-scoped key (#58894).
|
||
"""
|
||
runtime = _resolve_nous_runtime_api(force_refresh=True, stale_access_token=api_key)
|
||
if runtime is None:
|
||
return None, model
|
||
fresh_key, fresh_base_url = runtime
|
||
sync_client = _create_openai_client(api_key=fresh_key, base_url=fresh_base_url)
|
||
current_loop = _current_event_loop() if async_mode else None
|
||
if async_mode:
|
||
client, final_model = _to_async_client(sync_client, model or "", is_vision=is_vision)
|
||
else:
|
||
client, final_model = sync_client, model
|
||
cache_key = _client_cache_key(
|
||
cache_provider, async_mode=async_mode, base_url=base_url, api_key=api_key,
|
||
api_mode=api_mode, main_runtime=main_runtime, is_vision=is_vision, task=lookup_task,
|
||
model=lookup_model,
|
||
)
|
||
_store_cached_client(cache_key, client, final_model, bound_loop=current_loop)
|
||
return client, final_model
|
||
|
||
|
||
def neuter_async_httpx_del() -> None:
|
||
"""Monkey-patch ``AsyncHttpxClientWrapper.__del__`` to be a no-op.
|
||
|
||
The SDK's ``__del__`` schedules ``aclose()`` on the *running* loop, but the transport is
|
||
bound to the loop the client was created on; when that loop is dead this raises "Event loop
|
||
is closed" into prompt_toolkit's loop. Safe because cached clients are closed explicitly and
|
||
the OS reaps the rest. Call once at CLI startup, before any ``AsyncOpenAI`` is created.
|
||
"""
|
||
try:
|
||
from openai._base_client import AsyncHttpxClientWrapper
|
||
AsyncHttpxClientWrapper.__del__ = lambda self: None # type: ignore[assignment]
|
||
except (ImportError, AttributeError):
|
||
pass # Graceful degradation if the SDK changes its internals
|
||
|
||
|
||
def _force_close_async_httpx(client: Any) -> None:
|
||
"""Mark the httpx AsyncClient inside an AsyncOpenAI client as closed so ``__del__`` won't
|
||
schedule ``aclose()`` on a dead loop. Skips the full async close — the OS drops connections."""
|
||
with contextlib.suppress(Exception):
|
||
from httpx._client import ClientState
|
||
inner = getattr(client, "_client", None)
|
||
if inner is not None and not getattr(inner, "is_closed", True):
|
||
inner._state = ClientState.CLOSED
|
||
|
||
|
||
def _schedule_async_close(close_result: Any, client: Any) -> None:
|
||
"""Finish an async close without leaking an unawaited coroutine."""
|
||
async def _await_close() -> None:
|
||
try:
|
||
await close_result
|
||
except Exception:
|
||
pass
|
||
finally:
|
||
_force_close_async_httpx(client)
|
||
runner = _await_close()
|
||
try:
|
||
import asyncio as _aio
|
||
try:
|
||
loop = _aio.get_running_loop()
|
||
except RuntimeError:
|
||
_aio.run(runner)
|
||
else:
|
||
task = loop.create_task(runner)
|
||
|
||
def _consume(completed_task) -> None:
|
||
with contextlib.suppress(BaseException):
|
||
completed_task.exception()
|
||
task.add_done_callback(_consume)
|
||
runner = None
|
||
except Exception:
|
||
if runner is not None:
|
||
with contextlib.suppress(Exception):
|
||
runner.close()
|
||
_force_close_async_httpx(client)
|
||
|
||
|
||
def _close_cached_client(client: Any, *, close_async: bool = False) -> None:
|
||
"""Close one cached client, awaiting async transports only when safe."""
|
||
if client is None:
|
||
return
|
||
close_fn = getattr(client, "close", None)
|
||
if not callable(close_fn):
|
||
_force_close_async_httpx(client)
|
||
return
|
||
try:
|
||
close_result = close_fn()
|
||
except Exception:
|
||
_force_close_async_httpx(client)
|
||
return
|
||
if inspect.isawaitable(close_result):
|
||
if close_async:
|
||
_schedule_async_close(close_result, client)
|
||
else:
|
||
# Never await a client owned by another live loop; close the coroutine (no
|
||
# unawaited warning) and neuter the transport.
|
||
with contextlib.suppress(Exception):
|
||
close_result.close()
|
||
_force_close_async_httpx(client)
|
||
return
|
||
_force_close_async_httpx(client)
|
||
|
||
|
||
def shutdown_cached_clients() -> None:
|
||
"""Close all cached clients; call at CLI shutdown *before* the loop closes.
|
||
|
||
Snapshot+clear under the lock, close outside it: async teardown can block while an owner
|
||
loop drains, and holding the lock would convoy every caller.
|
||
"""
|
||
with _client_cache_lock:
|
||
clients = [(entry[0], entry[2]) for entry in _client_cache.values() if entry[0] is not None]
|
||
_client_cache.clear()
|
||
try:
|
||
import asyncio as _aio
|
||
running_loop = _aio.get_running_loop()
|
||
except RuntimeError:
|
||
running_loop = None
|
||
for client, owner_loop in clients:
|
||
# A live foreign loop owns its transport — neuter only and let it finish teardown.
|
||
# Closed loops and the current loop are safe to drain here.
|
||
close_async = owner_loop is not None and (owner_loop.is_closed() or owner_loop is running_loop)
|
||
_close_cached_client(client, close_async=close_async)
|
||
|
||
|
||
def cleanup_stale_async_clients() -> None:
|
||
"""Force-close cached async clients whose loop is closed; call after each agent turn
|
||
(defense-in-depth behind ``neuter_async_httpx_del``)."""
|
||
with _client_cache_lock:
|
||
stale = [(key, entry[0]) for key, entry in _client_cache.items() if entry[2] is not None and entry[2].is_closed()]
|
||
for key, _client in stale:
|
||
del _client_cache[key]
|
||
for _key, client in stale:
|
||
_close_cached_client(client, close_async=True)
|
||
|
||
|
||
def _compat_model(client: Any, model: Optional[str], cached_default: Optional[str]) -> Optional[str]:
|
||
"""Keep slash-bearing model IDs only for cached clients that accept ``vendor/model`` (OpenRouter
|
||
or a slash-bearing default). Mirrors the resolve_provider_client() guard, which cache hits skip."""
|
||
if model and "/" in model:
|
||
accepts_slash = any(
|
||
obj and base_url_host_matches(str(getattr(obj, "base_url", "") or ""), "openrouter.ai")
|
||
for obj in (client, getattr(client, "_client", None), getattr(client, "client", None))
|
||
) or bool(cached_default and "/" in cached_default)
|
||
if not accepts_slash:
|
||
return cached_default
|
||
return model or cached_default
|
||
|
||
|
||
def _get_cached_client(
|
||
provider: str, model: str = None, async_mode: bool = False, base_url: str = None,
|
||
api_key: str = None, api_mode: str = None, main_runtime: Optional[Dict[str, Any]] = None,
|
||
is_vision: bool = False, task: Optional[str] = None,
|
||
) -> Tuple[Optional[Any], Optional[str]]:
|
||
"""Get or create a cached client for the given provider.
|
||
|
||
Async clients bind to the loop they were created on, so every async hit validates the cached
|
||
loop is the current, open loop; stale entries are replaced in place (bounded, no cross-loop reuse).
|
||
|
||
This keeps cache size bounded to one entry per unique provider config, preventing the fd-exhaustion that
|
||
previously occurred in long-running gateways where recycled worker threads created unbounded entries
|
||
(#10200).
|
||
"""
|
||
current_loop = _current_event_loop() if async_mode else None
|
||
runtime = _normalize_main_runtime(main_runtime)
|
||
cache_key = _client_cache_key(
|
||
provider, async_mode=async_mode, base_url=base_url, api_key=api_key, api_mode=api_mode,
|
||
main_runtime=main_runtime, is_vision=is_vision, task=task, model=model,
|
||
)
|
||
with _client_cache_lock:
|
||
if cache_key in _client_cache:
|
||
cached_client, cached_default, cached_loop = _client_cache[cache_key]
|
||
loop_ok = not async_mode or (
|
||
cached_loop is not None and cached_loop is current_loop and not cached_loop.is_closed()
|
||
)
|
||
if loop_ok:
|
||
return cached_client, _compat_model(cached_client, model, cached_default)
|
||
# Stale async entry — evict. Only a closed owner loop may be awaited here; a live
|
||
# foreign loop stays force-neutered.
|
||
_close_cached_client(cached_client, close_async=cached_loop is not None and cached_loop.is_closed())
|
||
del _client_cache[cache_key]
|
||
# Build outside the lock. For pool-backed providers derive the key from the pool entry:
|
||
# resolve_api_key_provider_credentials prefers env vars, which would bypass pool rotation
|
||
# and retry an exhausted key.
|
||
effective_api_key = api_key
|
||
if not effective_api_key:
|
||
_pe = _peek_pool_entry(_normalize_aux_provider(provider))
|
||
if _pe is not None:
|
||
effective_api_key = _pool_runtime_api_key(_pe) or api_key
|
||
client, default_model = resolve_provider_client(
|
||
provider, model, async_mode, explicit_base_url=base_url, explicit_api_key=effective_api_key,
|
||
api_mode=api_mode, main_runtime=runtime, is_vision=is_vision, task=task,
|
||
)
|
||
if client is not None and _aux_probe_active():
|
||
# Availability probes answer "resolvable?" and must leave the cache untouched: the
|
||
# probe stub (bare, or wrapped in a Codex/Anthropic adapter whose leaf is the stub)
|
||
# shares the runtime key, and a cached one is served to every later caller — the
|
||
# next probe dies in _compat_model() on stub attribute access, so check_fns flip to
|
||
# False and vision tools vanish for the process lifetime (#87654).
|
||
return client, model or default_model
|
||
if client is not None:
|
||
with _client_cache_lock:
|
||
if cache_key not in _client_cache:
|
||
# FIFO safety-belt eviction. Do NOT close evicted clients: another caller may be
|
||
# mid-request on one; refcount/GC handles it.
|
||
while len(_client_cache) >= _CLIENT_CACHE_MAX_SIZE:
|
||
del _client_cache[next(iter(_client_cache))]
|
||
_client_cache[cache_key] = (client, default_model, current_loop)
|
||
else:
|
||
built_client = client
|
||
client, default_model, _ = _client_cache[cache_key]
|
||
# Race loser was never exposed to a caller — safe to close now.
|
||
_close_cached_client(built_client, close_async=async_mode)
|
||
return client, _compat_model(client, model, default_model)
|
||
|
||
|
||
# MoA virtual provider: an *explicit* `provider: moa` override (either the caller-passed `provider` arg or
|
||
# `auxiliary.<task>.provider` in config.yaml) reaches this function directly — it never goes through
|
||
# _resolve_auto_route(), which only unwraps the *implicit* "main provider is moa" case (#53827). Left as-is, "moa"
|
||
# is returned verbatim and resolve_provider_client() looks it up in PROVIDER_REGISTRY (which has no "moa"
|
||
# entry — it's not a real HTTP provider), falls to the unknown-provider dead end, and call_llm surfaces a
|
||
# nonsensical "MOA_API_KEY environment variable" error for a provider that was never meant to be reached
|
||
# over the wire. Auxiliary tasks don't need the reference fan-out — resolve to the preset's aggregator slot
|
||
# instead, exactly like the implicit path does (shared helper: _resolve_moa_aggregator).
|
||
def _unwrap_moa_provider(prov: str, mdl: Optional[str]) -> Tuple[str, Optional[str]]:
|
||
"""Resolve an *explicit* ``provider: moa`` to its preset's aggregator slot (_resolve_auto_route()
|
||
only unwraps the implicit case; "moa" isn't in PROVIDER_REGISTRY and would dead-end)."""
|
||
if prov.strip().lower() != "moa":
|
||
return prov, mdl
|
||
agg_provider, agg_model = _resolve_moa_aggregator(mdl)
|
||
if agg_provider and agg_model:
|
||
return agg_provider, agg_model
|
||
return prov, mdl
|
||
|
||
|
||
def _preserve_provider_with_base_url(prov: Optional[str]) -> bool:
|
||
"""True when a first-class provider keeps its identity alongside an explicit base_url."""
|
||
normalized = str(prov or "").strip().lower()
|
||
if normalized in {"", "auto", "custom"} or normalized.startswith("custom:"):
|
||
return False
|
||
if normalized in _LOCAL_SERVER_ALIASES:
|
||
return True # the custom branch applies the /v1 tail only when it still sees the alias
|
||
try:
|
||
from hermes_cli.providers import get_provider
|
||
return get_provider(normalized) is not None
|
||
except Exception: # keep provider-backed routes safe when the catalog can't load
|
||
return normalized in {
|
||
"anthropic", "copilot", "copilot-acp", "minimax-oauth", "nous", "openai-codex", "qwen-oauth", "xai-oauth",
|
||
}
|
||
|
||
|
||
def _resolve_task_provider_model(
|
||
task: str = None, provider: str = None, model: str = None, base_url: Optional[str] = None,
|
||
api_key: Optional[str] = None,
|
||
) -> Tuple[str, Optional[str], Optional[str], Optional[str], Optional[str]]:
|
||
"""Determine (provider, model, base_url, api_key, api_mode) for a call.
|
||
|
||
Priority: explicit args > config auxiliary.{task}.* > "auto". A bare base_url means custom,
|
||
but a first-class provider + base_url keeps the provider identity so its auth/transport
|
||
shaping still applies. api_mode is "chat_completions", "codex_responses", or None (auto).
|
||
"""
|
||
cfg_provider = cfg_model = cfg_base_url = cfg_api_key = resolved_api_mode = None
|
||
if task:
|
||
task_config = _get_auxiliary_task_config(task)
|
||
cfg_provider = str(task_config.get("provider", "")).strip() or None
|
||
cfg_model = str(task_config.get("model", "")).strip() or None
|
||
cfg_base_url = str(task_config.get("base_url", "")).strip() or None
|
||
cfg_api_key = str(task_config.get("api_key", "")).strip() or None
|
||
if not cfg_api_key: # key_env → env var when api_key is not set directly
|
||
cfg_key_env = str(task_config.get("key_env") or task_config.get("api_key_env") or "").strip()
|
||
if cfg_key_env:
|
||
cfg_api_key = _scoped_key_env(cfg_key_env) or None
|
||
# User-facing spellings (``responses``, ``anthropic``, …) canonicalize here so every
|
||
# branch downstream compares against the transport names only (#39750).
|
||
resolved_api_mode = _canonical_api_mode(str(task_config.get("api_mode") or "")).lower() or None
|
||
# 'auto' is a sentinel ("inherit / auto-detect"), not a model id — leaking it to the wire
|
||
# yields a 200 with an error-text body that consumers accept as output. The explicit `model`
|
||
# kwarg needs the same normalization: MoA slots forward preset `model:` fields through it.
|
||
if model and model.lower() == "auto":
|
||
model = None
|
||
if cfg_model and cfg_model.lower() == "auto":
|
||
cfg_model = None
|
||
resolved_model = model or cfg_model
|
||
# Any moa:// facade endpoint belongs to the facade, not the aggregator's real provider —
|
||
# drop it (mirrors _resolve_auto_route()).
|
||
if provider and str(provider).strip().lower() == "moa":
|
||
provider, resolved_model = _unwrap_moa_provider(provider, resolved_model)
|
||
if provider and provider.lower() != "moa":
|
||
base_url = None
|
||
api_key = None
|
||
elif cfg_provider and str(cfg_provider).strip().lower() == "moa":
|
||
cfg_provider, cfg_model = _unwrap_moa_provider(cfg_provider, resolved_model)
|
||
if cfg_provider and cfg_provider.lower() != "moa":
|
||
resolved_model = cfg_model
|
||
cfg_base_url = None
|
||
cfg_api_key = None
|
||
# One shared alias table with resolve_runtime_provider(): ``provider: openai`` routes the same
|
||
# way here (compression/vision/title) and on the runtime path (background review, curator, MoA).
|
||
from hermes_cli.runtime_provider_custom import expand_direct_api_alias
|
||
if provider:
|
||
provider, base_url = expand_direct_api_alias(provider, base_url)
|
||
if cfg_provider:
|
||
cfg_provider, cfg_base_url = expand_direct_api_alias(cfg_provider, cfg_base_url)
|
||
# An explicit provider without base_url adopts the task's configured endpoint (same or
|
||
# unnamed provider) so the early return below carries it. Explicit "auto" is excluded — it
|
||
# must keep flowing through auto-resolution.
|
||
# See #58515.
|
||
if provider and provider != "auto" and not base_url and cfg_base_url and cfg_provider in (None, provider):
|
||
base_url = cfg_base_url
|
||
if not api_key:
|
||
api_key = cfg_api_key
|
||
if base_url:
|
||
kept = provider if _preserve_provider_with_base_url(provider) else "custom"
|
||
return kept, resolved_model, base_url, api_key, resolved_api_mode
|
||
if provider:
|
||
return provider, resolved_model, base_url, api_key, resolved_api_mode
|
||
if cfg_base_url and cfg_api_key:
|
||
kept = cfg_provider if str(cfg_provider or "").strip().lower() in _LOCAL_SERVER_ALIASES else "custom"
|
||
return kept, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
|
||
if cfg_base_url and cfg_provider and cfg_provider != "auto":
|
||
# base_url without api_key: keep the provider so it can resolve credentials from env
|
||
# vars instead of locking into "custom".
|
||
return cfg_provider, resolved_model, cfg_base_url, None, resolved_api_mode
|
||
if cfg_provider and cfg_provider != "auto":
|
||
return cfg_provider, resolved_model, cfg_base_url, cfg_api_key, resolved_api_mode
|
||
return "auto", resolved_model, None, None, resolved_api_mode
|
||
|
||
|
||
_DEFAULT_AUX_TIMEOUT = 30.0
|
||
|
||
# Reasoning compression models can exceed the default 120 s config timeout, falling back to the
|
||
# deterministic marker. Bounded *floor* for config-derived compression timeouts only; never
|
||
# overrides an explicit per-call timeout.
|
||
# Compression summarises large conversation histories; a reasoning auxiliary model (e.g. Codex / GPT-5.5)
|
||
# can legitimately take longer than the default ``auxiliary.compression.timeout`` (120 s), causing the
|
||
# stream to time out and the compressor to fall back to the deterministic context marker (#54915). A floor
|
||
# is harmless for fast compression models (they finish before the deadline) and is a minimum, so a higher
|
||
# config value is kept unchanged.
|
||
_COMPRESSION_TIMEOUT_FLOOR_SECONDS = 300.0
|
||
|
||
|
||
def _get_auxiliary_task_config(task: str) -> Dict[str, Any]:
|
||
"""Config dict for auxiliary.<task>, or {} when unavailable. Plugin-registered tasks get their
|
||
declared defaults layered under user config (user wins); built-in defaults live in DEFAULT_CONFIG."""
|
||
if not task:
|
||
return {}
|
||
try:
|
||
from hermes_cli.config import load_config_readonly
|
||
config = load_config_readonly()
|
||
except ImportError:
|
||
return {}
|
||
aux = config.get("auxiliary", {}) if isinstance(config, dict) else {}
|
||
task_config = aux.get(task, {}) if isinstance(aux, dict) else {}
|
||
if not isinstance(task_config, dict):
|
||
task_config = {}
|
||
try:
|
||
from hermes_cli.plugins import get_plugin_auxiliary_tasks
|
||
for _entry in get_plugin_auxiliary_tasks():
|
||
if _entry.get("key") == task:
|
||
_defaults = _entry.get("defaults") or {}
|
||
if isinstance(_defaults, dict):
|
||
return {**_defaults, **task_config}
|
||
break
|
||
except Exception:
|
||
pass # plugin discovery failure must not break aux task config reads
|
||
return task_config
|
||
|
||
|
||
class CompressionFastLane(NamedTuple):
|
||
"""Explicit, non-reasoning compression route."""
|
||
|
||
certified_non_reasoning: bool
|
||
reasoning_config: Optional[Dict[str, Any]]
|
||
|
||
|
||
def _fast_lane_config_fields(config: Dict[str, Any]) -> tuple[str, str, bool]:
|
||
"""Only explicit reasoning disablement certifies a non-reasoning route."""
|
||
from hermes_constants import parse_reasoning_effort
|
||
provider = str(config.get("provider") or "").strip().lower()
|
||
model = str(config.get("model") or "").strip()
|
||
parsed_effort = parse_reasoning_effort(config.get("reasoning_effort"))
|
||
non_reasoning = parsed_effort is not None and parsed_effort.get("enabled") is False
|
||
return provider, model, non_reasoning
|
||
|
||
|
||
def resolve_compression_fast_lane(
|
||
actual_provider: str, actual_model: Optional[str], *, requested_provider: Optional[str] = None,
|
||
requested_model: Optional[str] = None, route_config: Optional[Dict[str, Any]] = None,
|
||
) -> CompressionFastLane:
|
||
"""Certify explicit non-reasoning settings only on the matching destination."""
|
||
config = route_config if route_config is not None else _get_auxiliary_task_config("compression")
|
||
cfg_provider, cfg_model, non_reasoning = _fast_lane_config_fields(config)
|
||
provider = str(requested_provider or "").strip().lower() or cfg_provider
|
||
model = str(requested_model or "").strip() or cfg_model
|
||
explicit_route = provider not in {"", "auto"} and model.lower() not in {"", "auto"}
|
||
actual_norm = _normalize_aux_provider(_fallback_provider_from_label(str(actual_provider or "")))
|
||
provider_matches = actual_norm == _normalize_aux_provider(provider)
|
||
model_matches = str(actual_model or "").strip().lower() == model.lower()
|
||
if explicit_route and provider_matches and model_matches and non_reasoning:
|
||
return CompressionFastLane(True, {"enabled": False, "effort": "none"})
|
||
return CompressionFastLane(False, None)
|
||
|
||
|
||
def _compression_config_claims_fast_lane(config: Dict[str, Any]) -> bool:
|
||
"""Whether task config declares fast-only controls that cannot leak."""
|
||
provider, model, non_reasoning = _fast_lane_config_fields(config)
|
||
return provider not in {"", "auto"} and model.lower() not in {"", "auto"} and non_reasoning
|
||
|
||
|
||
def _compression_fast_lane_controls(
|
||
task: str | None, *, actual_provider: str, actual_model: str | None,
|
||
requested_provider: str | None, requested_model: str | None, route_config: Dict[str, Any],
|
||
leak_guard_config: Dict[str, Any], max_tokens: int | None, extra_body: Dict[str, Any],
|
||
) -> tuple[int | None, Dict[str, Any]]:
|
||
"""Apply the certified compression controls to one resolved route."""
|
||
if task != "compression" or max_tokens is not None:
|
||
return max_tokens, extra_body
|
||
body = dict(extra_body)
|
||
lane = resolve_compression_fast_lane(
|
||
actual_provider, actual_model, requested_provider=requested_provider, requested_model=requested_model, route_config=route_config,
|
||
)
|
||
if lane.reasoning_config is not None:
|
||
if "reasoning" not in body:
|
||
body["reasoning"] = lane.reasoning_config
|
||
elif _compression_config_claims_fast_lane(leak_guard_config):
|
||
body.pop("reasoning", None)
|
||
return max_tokens, body
|
||
|
||
|
||
def _get_task_no_progress_timeout(task: str) -> Optional[float]:
|
||
"""``auxiliary.<task>.no_progress_timeout`` from config, or None when unset/invalid
|
||
(the Codex stream guard then keeps its built-in ``_AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS``
|
||
default). Lets an operator widen the substantive-progress window independently of the
|
||
overall request timeout — see #108104."""
|
||
if not task:
|
||
return None
|
||
raw = _get_auxiliary_task_config(task).get("no_progress_timeout")
|
||
if raw is None:
|
||
return None
|
||
try:
|
||
value = float(raw)
|
||
except (ValueError, TypeError):
|
||
value = 0.0
|
||
if isinstance(raw, bool) or value <= 0:
|
||
# Fail clearly: a typo here silently leaving the 60s default is exactly the
|
||
# "why did my 600s request abort after 60s" confusion the key exists to remove.
|
||
logger.warning(
|
||
"auxiliary.%s.no_progress_timeout=%r is not a positive number of seconds; "
|
||
"using the built-in %.0fs default", task, raw, _AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS)
|
||
return None
|
||
return value
|
||
|
||
|
||
def _get_task_timeout(task: str, default: float = _DEFAULT_AUX_TIMEOUT) -> float:
|
||
"""``auxiliary.<task>.timeout`` from config, else *default*."""
|
||
if not task:
|
||
return default
|
||
raw = _get_auxiliary_task_config(task).get("timeout")
|
||
if raw is not None:
|
||
with contextlib.suppress(ValueError, TypeError):
|
||
return float(raw)
|
||
return default
|
||
|
||
|
||
def _effective_aux_timeout(task: str, timeout: Optional[float]) -> float:
|
||
"""Explicit ``timeout`` wins, else config; compression gets a floor so a reasoning model
|
||
summarising a large context isn't cut off."""
|
||
if timeout is not None:
|
||
return timeout
|
||
effective = _get_task_timeout(task)
|
||
return max(effective, _COMPRESSION_TIMEOUT_FLOOR_SECONDS) if task == "compression" else effective
|
||
|
||
|
||
def _get_task_extra_body(task: str) -> Dict[str, Any]:
|
||
"""Shallow copy of ``auxiliary.<task>.extra_body`` with ``reasoning_effort`` folded into
|
||
``reasoning`` unless one is configured (more specific wins). MoA tasks are excluded: their
|
||
reasoning depth is per-slot in the preset."""
|
||
task_config = _get_auxiliary_task_config(task)
|
||
raw = task_config.get("extra_body")
|
||
result = dict(raw) if isinstance(raw, dict) else {}
|
||
if "reasoning" in result:
|
||
return result
|
||
effort = task_config.get("reasoning_effort")
|
||
if effort is None or effort == "":
|
||
return result
|
||
if task in ("moa_reference", "moa_aggregator"):
|
||
logger.warning(
|
||
"auxiliary.%s.reasoning_effort is not supported — MoA reasoning depth is per-slot: set reasoning_effort "
|
||
"on the preset's reference_models entries / aggregator instead (moa.presets.<name>...). Ignoring.",
|
||
task,
|
||
)
|
||
return result
|
||
from hermes_constants import parse_reasoning_effort
|
||
parsed = parse_reasoning_effort(effort)
|
||
if parsed is not None:
|
||
result["reasoning"] = parsed
|
||
else:
|
||
logger.warning(
|
||
"auxiliary.%s.reasoning_effort %r is not a valid level (none, minimal, low, medium, high, xhigh, max, ultra) — ignoring",
|
||
task, effort,
|
||
)
|
||
return result
|
||
|
||
|
||
# Per-task concurrency limiting: many sessions can spawn unbounded background aux calls, each
|
||
# retrying across the fallback chain during incidents.
|
||
# During provider incidents each call also retries / fans out across the fallback chain, multiplying request
|
||
# volume on already-degraded endpoints. A per-task semaphore caps in-flight calls so retry amplification
|
||
# stays bounded. See #23324.
|
||
# Keyed by profile home as well: the limit is the profile's ``auxiliary.<task>.max_concurrency``, and two
|
||
# multiplexed profiles with different limits would otherwise rebuild (and reset) one shared semaphore.
|
||
_aux_sync_semaphores: Dict[Tuple[str, str], Tuple[int, threading.BoundedSemaphore]] = {}
|
||
_aux_async_semaphores: Dict[Tuple[str, str, int], Tuple[int, Any]] = {}
|
||
_aux_sem_lock = threading.Lock()
|
||
|
||
|
||
def _get_task_max_concurrency(task: Optional[str]) -> Optional[int]:
|
||
"""``auxiliary.<task>.max_concurrency`` as a positive int, or None. Vision uses this key for
|
||
its encode/resize CPU pool; its LLM calls stay concurrent."""
|
||
if not task or task == "vision":
|
||
return None
|
||
try:
|
||
value = int(_get_auxiliary_task_config(task).get("max_concurrency"))
|
||
except (TypeError, ValueError): # missing (None) or malformed
|
||
return None
|
||
return value if value > 0 else None
|
||
|
||
|
||
def _cached_semaphore(store: dict, key: Any, limit: int, factory: Callable[[int], Any]) -> Any:
|
||
"""Return the cached semaphore for ``key``, rebuilding it when the limit changed."""
|
||
with _aux_sem_lock:
|
||
entry = store.get(key)
|
||
if entry is None or entry[0] != limit:
|
||
store[key] = entry = (limit, factory(limit))
|
||
return entry[1]
|
||
|
||
|
||
def _acquire_sync_aux_semaphore(task: Optional[str]) -> Optional[threading.BoundedSemaphore]:
|
||
"""Get a per-task sync semaphore, rebuilding it after a config change."""
|
||
limit = _get_task_max_concurrency(task)
|
||
if limit is None:
|
||
return None
|
||
from hermes_constants import hermes_home_key
|
||
return _cached_semaphore(_aux_sync_semaphores, (hermes_home_key(), task), limit, threading.BoundedSemaphore)
|
||
|
||
|
||
def _acquire_async_aux_semaphore(task: Optional[str]):
|
||
"""Get a per-task, per-event-loop async semaphore after config lookup."""
|
||
limit = _get_task_max_concurrency(task)
|
||
if limit is None:
|
||
return None
|
||
import asyncio
|
||
try:
|
||
loop = asyncio.get_running_loop()
|
||
except RuntimeError:
|
||
return None
|
||
from hermes_constants import hermes_home_key
|
||
return _cached_semaphore(_aux_async_semaphores, (hermes_home_key(), task, id(loop)), limit, asyncio.Semaphore)
|
||
|
||
|
||
def _reset_aux_semaphores() -> None:
|
||
"""Drop cached semaphores (test helper)."""
|
||
with _aux_sem_lock:
|
||
_aux_sync_semaphores.clear()
|
||
_aux_async_semaphores.clear()
|
||
|
||
|
||
# Anthropic-compatible endpoints reached via the OpenAI SDK wrapper; their image content blocks
|
||
# must use Anthropic format.
|
||
_ANTHROPIC_COMPAT_PROVIDERS = frozenset({"minimax", "minimax-oauth", "minimax-cn"})
|
||
|
||
|
||
def _is_anthropic_compat_endpoint(provider: str, base_url: str) -> bool:
|
||
"""True for known Anthropic-compatible providers or any ``/anthropic`` URL path."""
|
||
return provider in _ANTHROPIC_COMPAT_PROVIDERS or "/anthropic" in (base_url or "").lower()
|
||
|
||
|
||
# OpenAI block type → (Anthropic block type, default media type for data: URLs). MiniMax's
|
||
# Anthropic-compatible endpoint wants type="video" (not "video_url"/"input_video") with the same
|
||
# ``source`` shape as "image".
|
||
_ANTHROPIC_MEDIA_BLOCKS = {"image_url": ("image", "image/png"), "video_url": ("video", "video/mp4")}
|
||
|
||
|
||
def _convert_openai_images_to_anthropic(messages: list) -> list:
|
||
"""Convert OpenAI ``image_url``/``video_url`` blocks to Anthropic ``image``/``video``;
|
||
only list-content messages with such blocks change."""
|
||
converted = []
|
||
for msg in messages:
|
||
content = msg.get("content")
|
||
if not isinstance(content, list):
|
||
converted.append(msg)
|
||
continue
|
||
new_content = []
|
||
changed = False
|
||
for block in content:
|
||
block_type = block.get("type")
|
||
if block_type not in _ANTHROPIC_MEDIA_BLOCKS:
|
||
new_content.append(block)
|
||
continue
|
||
url = (block.get(block_type) or {}).get("url", "")
|
||
anth_type, media_type = _ANTHROPIC_MEDIA_BLOCKS[block_type]
|
||
if url.startswith("data:"):
|
||
header, _, b64data = url.partition(",")
|
||
if ":" in header and ";" in header:
|
||
media_type = header.split(":", 1)[1].split(";", 1)[0]
|
||
source = {"type": "base64", "media_type": media_type, "data": b64data}
|
||
else:
|
||
source = {"type": "url", "url": url}
|
||
new_content.append({"type": anth_type, "source": source})
|
||
changed = True
|
||
converted.append({**msg, "content": new_content} if changed else msg)
|
||
return converted
|
||
|
||
|
||
_PROFILE_REASONING_KEYS = {
|
||
"reasoning", "reasoning_effort", "thinking", "thinking_config", "thinkingconfig",
|
||
"thinking_budget", "thinkingbudget", "enable_thinking", "think", "verbosity",
|
||
}
|
||
|
||
|
||
def _contains_profile_reasoning_fields(value: Any) -> bool:
|
||
"""Return whether a profile payload contains a reasoning wire control (recursive)."""
|
||
if not isinstance(value, dict):
|
||
return False
|
||
return any(
|
||
str(key).strip().lower() in _PROFILE_REASONING_KEYS or _contains_profile_reasoning_fields(nested)
|
||
for key, nested in value.items()
|
||
)
|
||
|
||
|
||
_NOUS_PROVIDER_NAMES = frozenset({"nous", "nous-portal", "nousresearch"})
|
||
|
||
|
||
def _nous_on_messages_wire(provider_norm: str, model: str) -> bool:
|
||
"""True when a Nous Portal route serves ``model`` over /v1/messages (dual-wire catalog)."""
|
||
if provider_norm not in _NOUS_PROVIDER_NAMES:
|
||
return False
|
||
from hermes_cli.providers import nous_api_mode
|
||
return nous_api_mode(model) == "anthropic_messages"
|
||
|
||
|
||
_NVIDIA_PROVIDER_NAMES = {"nvidia", "nvidia-nim", "nim", "build-nvidia", "nemotron"}
|
||
_GEMINI_NATIVE_PROVIDER_NAMES = {"gemini", "google", "google-gemini", "google-ai-studio"}
|
||
|
||
|
||
def _is_gemini_native_route(provider_norm: str, effective_base: str) -> bool:
|
||
"""Gemini native by provider name, else (best-effort) by base URL shape."""
|
||
if provider_norm in _GEMINI_NATIVE_PROVIDER_NAMES:
|
||
return True
|
||
if not effective_base:
|
||
return False
|
||
try:
|
||
from agent.gemini_native_adapter import is_native_gemini_base_url
|
||
return is_native_gemini_base_url(effective_base)
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _forwards_max_tokens(provider: str, provider_norm: str, model: str, effective_base: str, task: Optional[str]) -> bool:
|
||
"""Whether an explicit max_tokens is forwarded on this route.
|
||
|
||
No default cap elsewhere (omitted = provider default; avoids max_completion_tokens / ZAI-vision
|
||
quirks). Forward only where mandatory or honored: Anthropic Messages wire (400 without it);
|
||
NVIDIA NIM (empty choices[] when omitted); MoA reference slots; Gemini native (fixed 65,535
|
||
ceiling otherwise); OpenRouter (budgets the FULL window when omitted → 402 on low credit);
|
||
managed local llama-server (uncapped decode with no EOS burns the GPU to the context window).
|
||
"""
|
||
return (
|
||
_is_anthropic_compat_endpoint(provider, effective_base)
|
||
or _nous_on_messages_wire(provider_norm, model)
|
||
or provider_norm in _NVIDIA_PROVIDER_NAMES
|
||
or base_url_host_matches(effective_base, "integrate.api.nvidia.com")
|
||
or str(task) == "moa_reference"
|
||
or _is_gemini_native_route(provider_norm, effective_base)
|
||
or provider_norm == "openrouter"
|
||
or base_url_host_matches(effective_base, "openrouter.ai")
|
||
or _is_managed_local_endpoint(effective_base)
|
||
)
|
||
|
||
|
||
def _dedupe_tool_names(tools: list, provider: str, model: str) -> list:
|
||
"""Drop duplicate tool names (Vertex/Azure/Bedrock 400 on them) with a warning."""
|
||
seen: set = set()
|
||
deduped: list = []
|
||
for tool in tools:
|
||
name = (tool.get("function") or {}).get("name", "")
|
||
if name and name in seen:
|
||
logger.warning("_build_call_kwargs: duplicate tool name '%s' removed (provider=%s model=%s)", name, provider, model)
|
||
continue
|
||
if name:
|
||
seen.add(name)
|
||
deduped.append(tool)
|
||
return deduped
|
||
|
||
|
||
class _ProfileProjection(NamedTuple):
|
||
body: Dict[str, Any]
|
||
reasoning_extra: Dict[str, Any]
|
||
top_level: Dict[str, Any]
|
||
handles_reasoning: bool
|
||
messages_wire: bool = False
|
||
|
||
|
||
def _routes_to_custom_endpoint(provider_norm: str) -> bool:
|
||
"""True when a profile-less provider name is really a configured OpenAI-compatible custom endpoint.
|
||
|
||
Keyed ``providers:`` / ``custom_providers`` entries (by bare key, or ``main``/``auto`` resolving to
|
||
one). An unconfigured name with only an explicit base_url keeps the generic nested fallback: the
|
||
operator declared nothing about that endpoint's wire, and the fireworks control contract pins it.
|
||
"""
|
||
name = _normalize_aux_provider(provider_norm)
|
||
if name == "custom":
|
||
return True
|
||
from hermes_cli.runtime_provider import _get_named_custom_provider
|
||
return _get_named_custom_provider(name) is not None
|
||
|
||
|
||
def _project_provider_profile(
|
||
provider: str, provider_norm: str, model: str, effective_base: str, reasoning_config: Optional[dict],
|
||
) -> _ProfileProjection:
|
||
"""Provider profile's extra_body / kwargs projection; partial on failure."""
|
||
body: Dict[str, Any] = {}
|
||
reasoning_extra: Dict[str, Any] = {}
|
||
top_level: Dict[str, Any] = {}
|
||
handles_reasoning = False
|
||
messages_wire = False
|
||
try:
|
||
from providers import get_provider_profile
|
||
from providers.base import ProviderProfile
|
||
profile = get_provider_profile(provider_norm)
|
||
if profile is None and _routes_to_custom_endpoint(provider_norm):
|
||
# A keyed ``providers:`` entry referenced by its bare key (or via ``main``/``auto``) is
|
||
# the same OpenAI-compatible custom endpoint the main path already projects with the
|
||
# ``custom`` profile (``custom:<key>`` falls back inside get_provider_profile). Without
|
||
# it the generic nested ``extra_body.reasoning`` fallback below ships to a strict
|
||
# gateway that only accepts top-level ``reasoning_effort`` -> 400 (#75089).
|
||
profile = get_provider_profile("custom")
|
||
if profile is not None:
|
||
messages_wire = profile.api_mode == "anthropic_messages"
|
||
body = profile.build_extra_body(model=model, base_url=effective_base, reasoning_config=reasoning_config) or {}
|
||
reasoning_extra, top_level = profile.build_api_kwargs_extras(
|
||
reasoning_config=reasoning_config, supports_reasoning=reasoning_config is not None,
|
||
model=model, base_url=effective_base,
|
||
)
|
||
reasoning_extra = reasoning_extra or {}
|
||
top_level = top_level or {}
|
||
handles_reasoning = (
|
||
type(profile).build_api_kwargs_extras is not ProviderProfile.build_api_kwargs_extras
|
||
or _contains_profile_reasoning_fields(body)
|
||
or _contains_profile_reasoning_fields(reasoning_extra)
|
||
or _contains_profile_reasoning_fields(top_level)
|
||
)
|
||
except Exception as exc:
|
||
logger.debug("_build_call_kwargs: provider profile projection failed for %s: %s", provider, exc)
|
||
return _ProfileProjection(body, reasoning_extra, top_level, handles_reasoning, messages_wire)
|
||
|
||
|
||
def _merge_aux_extra_body(
|
||
extra_body: Optional[dict], projection: _ProfileProjection, reasoning_config: Optional[dict], provider_norm: str,
|
||
) -> Dict[str, Any]:
|
||
"""Caller extra_body + profile body/reasoning + generic reasoning fallback + Nous tags."""
|
||
merged_extra = dict(extra_body or {})
|
||
caller_reasoning_fields = {
|
||
key: value for key, value in merged_extra.items()
|
||
if str(key).strip().lower() in _PROFILE_REASONING_KEYS and str(key).strip().lower() != "reasoning"
|
||
}
|
||
caller_disabled = isinstance(reasoning_config, dict) and reasoning_config.get("enabled") is False
|
||
if caller_disabled:
|
||
# The caller's thinking-off beats ``auxiliary.<task>.reasoning_effort`` (folded into
|
||
# ``extra_body.reasoning`` by _get_task_extra_body). Dropped BEFORE the profile merge so a
|
||
# profile that projects disabled reasoning onto its own wire (custom: top-level
|
||
# ``reasoning_effort=none``) never ships beside a task-level ``reasoning.effort`` — strict
|
||
# gateways 400 on the contradiction (#114020) — while a profile whose disabled shape IS
|
||
# ``extra_body.reasoning`` (OpenRouter) still lands it below.
|
||
merged_extra.pop("reasoning", None)
|
||
merged_extra.update(projection.body)
|
||
merged_extra.update(projection.reasoning_extra)
|
||
# Profiles supply route defaults, but an explicit vendor wire control in the task/call config
|
||
# is already provider-specific and must not be replaced by that default.
|
||
merged_extra.update(caller_reasoning_fields)
|
||
if reasoning_config and isinstance(reasoning_config, dict) and not projection.handles_reasoning:
|
||
if caller_disabled:
|
||
merged_extra["reasoning"] = {"enabled": False}
|
||
else:
|
||
# ``reasoning_config`` is already clamped to the OpenAI-compat wire by _build_call_kwargs.
|
||
merged_extra["reasoning"] = {"enabled": True, "effort": reasoning_config.get("effort") or "medium"}
|
||
# Caller/task ``extra_body.reasoning`` (``auxiliary.<task>.reasoning_effort`` folds in here via
|
||
# _get_task_extra_body) takes the same wire clamp: Hermes-only ``ultra`` never reaches the
|
||
# OpenAI-compat wire from any aux task (#112010).
|
||
if isinstance(merged_extra.get("reasoning"), dict):
|
||
from agent.reasoning_effort import clamp_reasoning_config
|
||
merged_extra["reasoning"] = clamp_reasoning_config(merged_extra["reasoning"])
|
||
# Portal tags + sticky session_id fallback when the profile didn't supply them; session_id
|
||
# keeps aux calls on the main turn's upstream instance (cache warmth) — tags alone are not
|
||
# enough on /v1/messages.
|
||
if provider_norm in _NOUS_PROVIDER_NAMES:
|
||
if "tags" not in merged_extra:
|
||
merged_extra["tags"] = _nous_portal_tags()
|
||
if "session_id" not in merged_extra:
|
||
try:
|
||
from agent.portal_tags import get_conversation_context
|
||
sticky_key = get_conversation_context()
|
||
except Exception:
|
||
sticky_key = None
|
||
if sticky_key:
|
||
merged_extra["session_id"] = sticky_key
|
||
return merged_extra
|
||
|
||
|
||
def _build_call_kwargs(
|
||
provider: str, model: str, messages: list, temperature: Optional[float] = None,
|
||
max_tokens: Optional[int] = None, tools: Optional[list] = None, timeout: float = 30.0,
|
||
extra_body: Optional[dict] = None, reasoning_config: Optional[dict] = None,
|
||
base_url: Optional[str] = None, task: Optional[str] = None,
|
||
no_progress_timeout: Optional[float] = None,
|
||
) -> dict:
|
||
"""Build kwargs for .chat.completions.create() with model/provider adjustments.
|
||
``no_progress_timeout`` is a Codex-Responses-only extra (consumed by
|
||
``_CodexCompletionsAdapter.create``'s ``**kwargs`` catch-all); callers must only pass it
|
||
when the resolved client is a ``CodexAuxiliaryClient`` — real SDK clients don't accept it."""
|
||
kwargs: Dict[str, Any] = {"model": model, "messages": messages, "timeout": timeout}
|
||
if no_progress_timeout is not None:
|
||
kwargs["no_progress_timeout"] = no_progress_timeout
|
||
effective_base = base_url or (_current_custom_base_url() if provider == "custom" else "")
|
||
# Per-model fixed/omitted temperature, then Opus 4.7+ sampling bans: it rejects any
|
||
# non-default temperature/top_p/top_k, so drop silently rather than 400 when the aux model flips.
|
||
fixed_temperature = _fixed_temperature_for_model(model, effective_base, provider)
|
||
if fixed_temperature is OMIT_TEMPERATURE:
|
||
temperature = None # strip — let server choose
|
||
elif fixed_temperature is not None:
|
||
temperature = fixed_temperature
|
||
if temperature is not None:
|
||
from agent.anthropic_adapter import _forbids_sampling_params
|
||
if not _forbids_sampling_params(model):
|
||
kwargs["temperature"] = temperature
|
||
provider_norm = str(provider or "").strip().lower()
|
||
if max_tokens is not None and _forwards_max_tokens(provider, provider_norm, model, effective_base, task):
|
||
kwargs.update(auxiliary_max_tokens_param(max_tokens, model=model)) # picks max_completion_tokens where needed
|
||
if tools:
|
||
kwargs["tools"] = _dedupe_tool_names(tools, provider, model)
|
||
# Provider profiles are the source of truth for reasoning wire shapes (top-level, nested body,
|
||
# or extra_body.reasoning); providers without a reasoning-aware profile keep the generic
|
||
# ``extra_body.reasoning`` fallback. Clamp Hermes-internal levels (``ultra``) to the
|
||
# OpenAI-compat wire ONCE here, before either path sees the config — the same entry clamp the
|
||
# main transport applies (#89503); MoA aggregator/reference and aux calls 400'd without it (#112010).
|
||
from agent.reasoning_effort import clamp_reasoning_config
|
||
from agent.auxiliary_reasoning_floor import known_reasoning_floor
|
||
if isinstance(extra_body, dict):
|
||
task_reasoning = extra_body.get("reasoning")
|
||
if isinstance(task_reasoning, dict) and "enabled" in task_reasoning:
|
||
extra_body = dict(extra_body)
|
||
extra_body.pop("reasoning")
|
||
if reasoning_config is None:
|
||
reasoning_config = task_reasoning
|
||
reasoning_config = clamp_reasoning_config(
|
||
known_reasoning_floor(reasoning_config, provider_norm, effective_base, model, task))
|
||
projection = _project_provider_profile(provider, provider_norm, model, effective_base, reasoning_config)
|
||
kwargs.update(projection.top_level)
|
||
merged_extra = _merge_aux_extra_body(extra_body, projection, reasoning_config, provider_norm)
|
||
if "response_format" in merged_extra:
|
||
from agent.auxiliary_structured_output import without_unsupported_response_format
|
||
merged_extra = without_unsupported_response_format(merged_extra, provider_norm, effective_base, model, task)
|
||
if merged_extra:
|
||
kwargs["extra_body"] = merged_extra
|
||
# Anthropic Messages adapters take reasoning via a private kwarg that plain OpenAI SDK clients
|
||
# would reject; Portal Claude is dual-wire, so include it only when the catalog id selects
|
||
# /v1/messages. A profile declaring api_mode=anthropic_messages (commandcode-anthropic) is on
|
||
# that wire regardless of URL shape — once it overrides build_api_kwargs_extras the generic
|
||
# ``extra_body.reasoning`` fallback the adapter used to read is gone, so this is the adapter's
|
||
# only path. _wrap_transport wraps such providers on the same declaration.
|
||
if reasoning_config and isinstance(reasoning_config, dict):
|
||
raw_base = base_url or ""
|
||
if (
|
||
provider_norm == "anthropic" or projection.messages_wire or _nous_on_messages_wire(provider_norm, model)
|
||
or _endpoint_speaks_anthropic_messages(raw_base) or _is_anthropic_compat_endpoint(provider_norm, raw_base)
|
||
):
|
||
kwargs["_reasoning_config"] = dict(reasoning_config)
|
||
# Conversation affinity (OpenCode relay, opt-in custom-provider header) — same key as the main
|
||
# turn so compression/title/vision calls stay on the conversation's warm backend.
|
||
from agent.opencode_affinity import merge_session_affinity_headers
|
||
return merge_session_affinity_headers(kwargs, provider, base_url, _runtime_main_value("session_id") or None)
|
||
|
||
|
||
def _validate_llm_response(
|
||
response: Any, task: Optional[str] = None, provider: Optional[str] = None, base_url: Optional[str] = None,
|
||
) -> Any:
|
||
"""Validate the .choices[0].message shape (fail fast, not a downstream AttributeError).
|
||
|
||
Also the single aux-usage accounting chokepoint: every successful non-streaming response
|
||
passes here exactly once; *provider*/*base_url* are optional hints.
|
||
|
||
See #7264.
|
||
Recording is best-effort and never affects validation. *provider*/*base_url* are optional accounting
|
||
hints — fallback-path calls omit them and the row keeps the model (read from the response itself) with
|
||
an empty route. See #23270.
|
||
"""
|
||
if response is None:
|
||
raise RuntimeError(f"Auxiliary {task or 'call'}: LLM returned None response")
|
||
from agent.aux_accounting import record_aux_usage
|
||
record_aux_usage(response, task, provider=provider, base_url=base_url)
|
||
# Adapter SimpleNamespace responses are fine — they have .choices[0].message.
|
||
try:
|
||
choices = response.choices
|
||
if not choices or not hasattr(choices[0], "message"):
|
||
raise AttributeError("missing choices[0].message")
|
||
except (AttributeError, TypeError, IndexError) as exc:
|
||
recovered = _recover_aux_response_message(response)
|
||
if recovered is None:
|
||
raise RuntimeError(
|
||
f"Auxiliary {task or 'call'}: LLM returned invalid response (type={type(response).__name__}): "
|
||
f"{str(response)[:120]!r}. Expected object with .choices[0].message — check provider "
|
||
f"adapter or custom endpoint compatibility."
|
||
) from exc
|
||
response = recovered
|
||
from agent.transports.chat_completions import is_router_timeout_shim
|
||
if is_router_timeout_shim(response):
|
||
# HTTP-200 router failure shim (#68396): invalid like a malformed shape so the
|
||
# auxiliary fallback chain moves to the next candidate instead of titling with it.
|
||
raise RuntimeError(f"Auxiliary {task or 'call'}: provider returned a timeout shim instead of a completion")
|
||
# Retain the provider-reported model for terminal relay route attribution.
|
||
context = _RELAY_AUX_CALL_CONTEXT.get()
|
||
if context is not None:
|
||
model = _field(response, "model")
|
||
if isinstance(model, str) and model.strip():
|
||
context["response_model"] = model
|
||
_complete_relay_auxiliary_call()
|
||
return response
|
||
|
||
|
||
def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None:
|
||
"""Close one auxiliary logical call after acceptance or terminal failure."""
|
||
context = _RELAY_AUX_CALL_CONTEXT.get()
|
||
if context is None:
|
||
return
|
||
from agent import relay_llm
|
||
relay_llm.complete_logical_call(
|
||
str(context.get("request_id") or ""), outcome=outcome,
|
||
model_name=str(context.get("model") or "unknown"),
|
||
provider_name=str(context.get("provider") or "auxiliary"),
|
||
response_model_name=context.get("response_model"),
|
||
)
|
||
|
||
|
||
def _fail_relay_auxiliary_call() -> None:
|
||
"""Close a terminally failed call without replacing its original error."""
|
||
try:
|
||
_complete_relay_auxiliary_call(outcome="failed")
|
||
except Exception:
|
||
logger.warning("Relay auxiliary failure finalization failed", exc_info=True)
|
||
|
||
|
||
def _recover_aux_response_message(response: Any) -> Optional[Any]:
|
||
"""Synthesize chat-completions shape from Responses-style text (``output_text``,
|
||
``output`` items) that some compatible endpoints return outside ``choices``."""
|
||
text = _extract_aux_response_text(response)
|
||
if not text:
|
||
return None
|
||
choice = SimpleNamespace(message=SimpleNamespace(content=text), finish_reason=getattr(response, "finish_reason", None) or "stop")
|
||
try:
|
||
response.choices = [choice]
|
||
return response
|
||
except Exception:
|
||
return SimpleNamespace(
|
||
id=getattr(response, "id", ""), model=getattr(response, "model", ""),
|
||
object=getattr(response, "object", "chat.completion"), choices=[choice],
|
||
usage=getattr(response, "usage", None),
|
||
)
|
||
|
||
|
||
def _extract_aux_response_text(response: Any) -> str:
|
||
"""Text from Responses-style ``output_text`` or ``output[].content[].text``."""
|
||
output_text = _field(response, "output_text")
|
||
if isinstance(output_text, str) and output_text.strip():
|
||
return output_text.strip()
|
||
output = _field(response, "output")
|
||
if not isinstance(output, list):
|
||
return ""
|
||
parts: List[str] = []
|
||
for item in output:
|
||
item_type = _field(item, "type")
|
||
if item_type and item_type != "message":
|
||
continue
|
||
for part in (_field(item, "content") or []):
|
||
if _field(part, "type") in {"output_text", "text", None}:
|
||
text = _field(part, "text")
|
||
if isinstance(text, str) and text.strip():
|
||
parts.append(text.strip())
|
||
return "\n".join(parts).strip()
|
||
|
||
|
||
# Streamed aggregation for progress-hooked aux calls: ``timeout`` becomes an inter-chunk idle
|
||
# timeout (httpx read timeout is per read), each chunk ticks outer watchdogs; the total ceiling
|
||
# bounds trickles.
|
||
_AUX_STREAM_CEILING_FLOOR_SECONDS = 600.0
|
||
_AUX_STREAM_CEILING_MULTIPLIER = 4.0
|
||
|
||
|
||
def _aux_stream_total_ceiling(effective_timeout: Optional[float]) -> float:
|
||
"""Absolute wall-clock bound for a streamed aux call; generous by design (the idle
|
||
timeout is the real guard — this only stops a one-token-per-idle-window trickle)."""
|
||
try:
|
||
timeout = float(effective_timeout) if effective_timeout is not None else 0.0
|
||
except (TypeError, ValueError):
|
||
timeout = 0.0
|
||
return max(_AUX_STREAM_CEILING_FLOOR_SECONDS, _AUX_STREAM_CEILING_MULTIPLIER * timeout)
|
||
|
||
|
||
def _client_streams_internally(client: Any) -> bool:
|
||
"""Adapters that stream inside .create() tick the hook themselves (Codex, Anthropic) or
|
||
cannot stream (Bedrock); none accept ``stream=True`` from us."""
|
||
return isinstance(client, (CodexAuxiliaryClient, AnthropicAuxiliaryClient, BedrockAuxiliaryClient))
|
||
|
||
|
||
_MANAGED_LOCAL_STATE_TTL_S = 15.0
|
||
_managed_local_cache: "tuple[float, str]" = (0.0, "")
|
||
|
||
|
||
def _managed_local_netloc() -> str:
|
||
"""host:port of the managed local llama-server ("" when none), read with a short TTL from
|
||
the supervisor state file provider resolution also uses (exact match)."""
|
||
global _managed_local_cache
|
||
now = time.monotonic()
|
||
ts, cached = _managed_local_cache
|
||
if now - ts < _MANAGED_LOCAL_STATE_TTL_S:
|
||
return cached
|
||
try:
|
||
from hermes_cli.local_runtime.supervisor import state_path
|
||
|
||
raw = state_path().read_text(encoding="utf-8-sig")
|
||
base = str((json.loads(raw) or {}).get("base_url", ""))
|
||
netloc = urlparse(base).netloc.lower()
|
||
except Exception:
|
||
netloc = ""
|
||
_managed_local_cache = (now, netloc)
|
||
return netloc
|
||
|
||
|
||
def _is_managed_local_endpoint(base_url: Optional[str]) -> bool:
|
||
"""True when *base_url* targets the llama-server this Hermes manages."""
|
||
if not base_url:
|
||
return False
|
||
managed = _managed_local_netloc()
|
||
if not managed:
|
||
return False
|
||
try:
|
||
return urlparse(str(base_url)).netloc.lower() == managed
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _provider_requires_stream(provider: str, base_url: Optional[str]) -> bool:
|
||
"""Providers that only accept streaming (non-stream = 400): Tencent Copilot, any
|
||
``auxiliary.stream_only_base_urls`` substring, and the managed local llama-server
|
||
(streamed for cancellation — it only notices a dead client on socket write)."""
|
||
_url = str(base_url or "").lower()
|
||
if not _url:
|
||
return False
|
||
if base_url_host_matches(_url, "copilot.tencent.com") or _is_managed_local_endpoint(_url):
|
||
return True
|
||
try:
|
||
from hermes_cli.config import load_config
|
||
markers = (load_config() or {}).get("auxiliary", {}).get("stream_only_base_urls") or []
|
||
if isinstance(markers, (list, tuple)):
|
||
return any(
|
||
isinstance(marker, str) and marker.strip() and marker.strip().lower() in _url
|
||
for marker in markers)
|
||
except Exception:
|
||
pass # Config read is best-effort; never break an aux call over it.
|
||
return False
|
||
|
||
|
||
_AFFORDABLE_TOKENS_RE = re.compile(r"can only afford\s+([0-9][0-9,]*)", re.IGNORECASE)
|
||
# Below the floor the affordable budget can't fit a useful aux output — treat as exhaustion;
|
||
# the margin keeps provider-side token-count rounding from 402-ing the retry.
|
||
_AFFORDABLE_RETRY_FLOOR_TOKENS = 512
|
||
# See #49785.
|
||
_AFFORDABLE_RETRY_MARGIN_TOKENS = 64
|
||
|
||
|
||
def _affordable_max_tokens_from_error(exc: Exception) -> Optional[int]:
|
||
"""Affordable output budget (minus margin) from an OpenRouter credit-limited 402
|
||
("...but can only afford 7117": credit exists, the cap was too large); ``None``
|
||
when no count is present or the budget is too small to be useful."""
|
||
if not _is_payment_error(exc):
|
||
return None
|
||
match = _AFFORDABLE_TOKENS_RE.search(str(exc))
|
||
if not match:
|
||
return None
|
||
try:
|
||
affordable = int(match.group(1).replace(",", ""))
|
||
except (TypeError, ValueError):
|
||
return None
|
||
capped = affordable - _AFFORDABLE_RETRY_MARGIN_TOKENS
|
||
return capped if capped >= _AFFORDABLE_RETRY_FLOOR_TOKENS else None
|
||
|
||
|
||
def _create_with_progress(
|
||
client: Any, kwargs: Dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
|
||
) -> Any:
|
||
"""Credit-aware :func:`_create_with_progress_once`: a 402 naming an affordable
|
||
budget retries ONCE with that cap (only ever lowering); anything else re-raises."""
|
||
try:
|
||
return _create_with_progress_once(client, kwargs, task, force_stream=force_stream)
|
||
except Exception as exc:
|
||
affordable = _affordable_max_tokens_from_error(exc)
|
||
if affordable is None:
|
||
raise
|
||
existing_cap = kwargs.get("max_tokens") or kwargs.get("max_completion_tokens")
|
||
if isinstance(existing_cap, (int, float)) and 0 < existing_cap <= affordable:
|
||
raise # Already within budget — the error is something else; don't spin.
|
||
retry_kwargs = dict(kwargs)
|
||
retry_kwargs.pop("max_tokens", None)
|
||
retry_kwargs.pop("max_completion_tokens", None)
|
||
retry_kwargs.update(
|
||
auxiliary_max_tokens_param(affordable, model=str(kwargs.get("model") or "") or None))
|
||
logger.info("Auxiliary %s: credit-limited 402 (affordable=%d tokens); "
|
||
"retrying once with a clamped output cap instead of failing: %s",
|
||
task or "call", affordable, exc)
|
||
return _create_with_progress_once(client, retry_kwargs, task, force_stream=force_stream)
|
||
|
||
|
||
def _stream_request_plan(kwargs: Dict[str, Any]) -> "Tuple[Dict[str, Any], str, float]":
|
||
"""(stream kwargs, model name, total ceiling) for a streamed re-aggregation."""
|
||
stream_kwargs = dict(kwargs)
|
||
stream_kwargs["stream"] = True
|
||
stream_kwargs["stream_options"] = {"include_usage": True}
|
||
return (stream_kwargs, str(kwargs.get("model") or ""),
|
||
_aux_stream_total_ceiling(kwargs.get("timeout")))
|
||
|
||
|
||
def _create_with_progress_once(
|
||
client: Any, kwargs: Dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
|
||
) -> Any:
|
||
"""create() that streams (and re-aggregates, ticking the hook per substantive chunk) when a
|
||
progress hook is active or the provider is stream-only; plain ``create(**kwargs)`` otherwise
|
||
or when the adapter streams internally. Streaming rejections fall back to a plain call —
|
||
except under ``force_stream``.
|
||
|
||
Behavior is byte-for-byte identical to a plain ``create(**kwargs)`` when neither trigger applies (every
|
||
existing caller/task) or when the client's wire adapter streams internally. With a hook + a
|
||
chunk-capable client, the request is sent with ``stream=True`` and aggregated, ticking the hook only for
|
||
substantive chunks. The configured ``timeout`` acts per stream read (idle) rather than as a total
|
||
budget, and outer liveness watchdogs see tokens moving. ``force_stream=True`` (stream-only providers
|
||
such as Tencent Copilot — credit @kudi88, PR #60686) takes the same streamed path even without a hook.
|
||
Providers that reject the streamed request fall back to the plain non-streaming call — except under
|
||
``force_stream``, where a stream-only provider rejects the plain call by definition, so the original
|
||
error is surfaced to the normal recovery chains instead.
|
||
"""
|
||
kwargs = bypass_chat_sdk_request_transform(kwargs, client)
|
||
_notify_aux_dispatch()
|
||
# Dispatch alone is not forward progress: a 401/retry/fallback dispatch must not
|
||
# reset the compression inactivity fence, or a zero-output attempt runs to the
|
||
# total ceiling instead of idling out (#114938). Progress ticks only for
|
||
# substantive stream payloads or a completed usable response.
|
||
if (not _aux_progress_active() and not force_stream) or _client_streams_internally(client):
|
||
response = client.chat.completions.create(**kwargs)
|
||
if not _client_streams_internally(client):
|
||
_notify_aux_provider_response()
|
||
return response
|
||
stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs)
|
||
try:
|
||
chunks = client.chat.completions.create(**stream_kwargs)
|
||
except Exception as exc:
|
||
# Genuine provider failures aren't streaming's fault — surface unchanged so the
|
||
# recovery chains see the same error as a plain call.
|
||
if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
|
||
or _is_payment_error(exc) or _is_rate_limit_error(exc)):
|
||
raise
|
||
# Possibly a streaming-specific rejection: retry non-streaming once; a genuinely bad
|
||
# request reproduces the real error for the except-chains.
|
||
logger.debug("Auxiliary %s: streamed request failed (%s); retrying non-streaming",
|
||
task or "call", exc)
|
||
_notify_aux_dispatch()
|
||
response = client.chat.completions.create(**kwargs)
|
||
_notify_aux_provider_response()
|
||
return response
|
||
# Some shims (MoA quiet mode, defensive adapters) return a complete response despite
|
||
# stream=True; it counts as provider response + forward progress.
|
||
if hasattr(chunks, "choices"):
|
||
_notify_aux_provider_response()
|
||
return chunks
|
||
return _aggregate_chat_stream(chunks, model=model, total_ceiling=total_ceiling)
|
||
|
||
|
||
def _close_chunk_stream(chunks: Any, *, allow_aclose: bool = False) -> Any:
|
||
"""Best-effort ``close()`` (or ``aclose()``); returns a pending awaitable or None."""
|
||
close_fn = getattr(chunks, "close", None) or (
|
||
getattr(chunks, "aclose", None) if allow_aclose else None)
|
||
if not callable(close_fn):
|
||
return None
|
||
try:
|
||
result = close_fn()
|
||
except Exception:
|
||
return None
|
||
return result if inspect.isawaitable(result) else None
|
||
|
||
|
||
def _aggregate_chat_stream(
|
||
chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None
|
||
) -> Any:
|
||
"""Consume a chunk stream into a complete response; TimeoutError (phrased "timed out" so
|
||
``_is_timeout_error`` matches) when *total_ceiling* elapses."""
|
||
acc = _ChatStreamAccumulator(
|
||
model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline())
|
||
try:
|
||
for chunk in chunks:
|
||
acc.feed(chunk)
|
||
finally:
|
||
_close_chunk_stream(chunks)
|
||
return acc.finish()
|
||
|
||
|
||
# Reasoning-detail fields whose non-empty text counts as forward progress.
|
||
_REASONING_DETAIL_TEXT_FIELDS = ("summary", "thinking", "content", "text")
|
||
|
||
|
||
class _ChatStreamAccumulator:
|
||
"""Shared per-chunk accumulation so sync and async aggregation cannot drift."""
|
||
|
||
def __init__(self, model: str = "", total_ceiling: Optional[float] = None,
|
||
host_deadline: Optional[float] = None):
|
||
self._started = time.monotonic()
|
||
self._total_ceiling = total_ceiling
|
||
# Absolute instant the waiting host gives up; checked alongside (not instead of) the
|
||
# ceiling, and unaffected by pre-construction dispatch/TTFT.
|
||
# Checked as well as (not instead of) the ceiling above: the ceiling still bounds callers with no
|
||
# host deadline, and the host deadline is absolute, so it is unaffected by however long dispatch and
|
||
# TTFT took before this accumulator was constructed. See #99692.
|
||
self._host_deadline = host_deadline
|
||
self.content_parts: List[str] = []
|
||
self.reasoning_parts: List[str] = []
|
||
self.reasoning_details: List[Any] = []
|
||
self.tool_calls_acc: Dict[int, Dict[str, Any]] = {}
|
||
self.finish_reason = self.usage = None
|
||
self.resp_id = ""
|
||
self.resp_model = model or ""
|
||
|
||
def _check_deadlines(self) -> None:
|
||
"""Raise TimeoutError past the total ceiling or the host deadline."""
|
||
now = time.monotonic()
|
||
if self._total_ceiling is not None and (now - self._started) >= self._total_ceiling:
|
||
raise TimeoutError(f"Auxiliary streamed call timed out after {self._total_ceiling:.0f}s "
|
||
"total ceiling (stream still open but over budget)")
|
||
if self._host_deadline is not None and now >= self._host_deadline:
|
||
raise TimeoutError("Auxiliary streamed call timed out at the host compression "
|
||
f"deadline after {time.monotonic() - self._started:.0f}s "
|
||
"(the caller already stopped waiting; streaming on would only "
|
||
"pin its session lease)")
|
||
|
||
def _feed_reasoning_details(self, delta: Any) -> bool:
|
||
"""Collect ``reasoning_details`` (OpenRouter-style thinking); True only when a detail
|
||
carries text, so structural/signed envelopes can't keep a stall alive."""
|
||
reasoning_details = getattr(delta, "reasoning_details", None)
|
||
if reasoning_details is None:
|
||
model_extra = getattr(delta, "model_extra", None)
|
||
if isinstance(model_extra, dict):
|
||
reasoning_details = model_extra.get("reasoning_details")
|
||
if not isinstance(reasoning_details, list):
|
||
return False
|
||
made_progress = False
|
||
for detail in reasoning_details:
|
||
self.reasoning_details.append(detail)
|
||
if isinstance(detail, dict) and any(
|
||
isinstance(detail.get(f), str) and detail[f] for f in _REASONING_DETAIL_TEXT_FIELDS):
|
||
made_progress = True
|
||
return made_progress
|
||
|
||
def _feed_tool_calls(self, delta: Any) -> bool:
|
||
"""Merge tool-call fragments by index; True when any fragment carried data."""
|
||
made_progress = False
|
||
for tc in (getattr(delta, "tool_calls", None) or []):
|
||
idx = getattr(tc, "index", 0) or 0
|
||
acc = self.tool_calls_acc.setdefault(idx, {"id": "", "name": "", "arguments": []})
|
||
if getattr(tc, "id", None):
|
||
acc["id"] = tc.id
|
||
made_progress = True
|
||
fn = getattr(tc, "function", None)
|
||
if fn is not None:
|
||
if getattr(fn, "name", None):
|
||
acc["name"] = fn.name
|
||
made_progress = True
|
||
if getattr(fn, "arguments", None):
|
||
acc["arguments"].append(fn.arguments)
|
||
made_progress = True
|
||
return made_progress
|
||
|
||
def feed(self, chunk: Any) -> None:
|
||
# Every frame records transport timing (TTFP); only a substantive payload ticks the
|
||
# forward-progress hook that keeps compression alive.
|
||
_notify_aux_timing_response()
|
||
self._check_deadlines()
|
||
self.resp_id = getattr(chunk, "id", None) or self.resp_id
|
||
self.resp_model = getattr(chunk, "model", None) or self.resp_model
|
||
chunk_usage = getattr(chunk, "usage", None)
|
||
if chunk_usage:
|
||
self.usage = chunk_usage
|
||
choices = getattr(chunk, "choices", None) or []
|
||
if not choices:
|
||
return
|
||
choice = choices[0]
|
||
self.finish_reason = getattr(choice, "finish_reason", None) or self.finish_reason
|
||
delta = getattr(choice, "delta", None)
|
||
if delta is None:
|
||
return
|
||
made_progress = False
|
||
from agent.message_content import flatten_message_text
|
||
|
||
piece = flatten_message_text(getattr(delta, "content", None), sep="")
|
||
if piece:
|
||
self.content_parts.append(piece)
|
||
made_progress = True
|
||
reasoning_piece = getattr(delta, "reasoning", None) or getattr(delta, "reasoning_content", None)
|
||
if reasoning_piece is None and isinstance(getattr(delta, "model_extra", None), dict):
|
||
reasoning_piece = delta.model_extra.get("reasoning") or delta.model_extra.get("reasoning_content")
|
||
reasoning_piece = flatten_message_text(reasoning_piece, sep="")
|
||
if reasoning_piece:
|
||
self.reasoning_parts.append(reasoning_piece)
|
||
made_progress = True
|
||
# Evaluate both unconditionally: they accumulate state, not just progress.
|
||
made_progress |= self._feed_reasoning_details(delta)
|
||
made_progress |= self._feed_tool_calls(delta)
|
||
if made_progress:
|
||
_notify_aux_progress()
|
||
|
||
def finish(self) -> Any:
|
||
tool_calls = None
|
||
if self.tool_calls_acc:
|
||
tool_calls = [
|
||
SimpleNamespace(id=acc["id"], type="function", function=SimpleNamespace(
|
||
name=acc["name"], arguments="".join(acc["arguments"])))
|
||
for _idx, acc in sorted(self.tool_calls_acc.items())]
|
||
message = SimpleNamespace(
|
||
role="assistant", content="".join(self.content_parts), tool_calls=tool_calls,
|
||
reasoning="".join(self.reasoning_parts) or None,
|
||
reasoning_details=self.reasoning_details or None,
|
||
)
|
||
choice = SimpleNamespace(index=0, message=message, finish_reason=self.finish_reason or "stop")
|
||
return SimpleNamespace(id=self.resp_id, model=self.resp_model, object="chat.completion",
|
||
choices=[choice], usage=self.usage)
|
||
|
||
|
||
async def _aggregate_chat_stream_async(
|
||
chunks: Any, *, model: str = "", total_ceiling: Optional[float] = None
|
||
) -> Any:
|
||
"""Async mirror of :func:`_aggregate_chat_stream` (AsyncOpenAI streams need ``async for``)."""
|
||
acc = _ChatStreamAccumulator(
|
||
model=model, total_ceiling=total_ceiling, host_deadline=_current_aux_stream_deadline())
|
||
try:
|
||
async for chunk in chunks:
|
||
acc.feed(chunk)
|
||
finally:
|
||
pending = _close_chunk_stream(chunks, allow_aclose=True)
|
||
if pending is not None:
|
||
with contextlib.suppress(Exception):
|
||
await pending
|
||
return acc.finish()
|
||
|
||
|
||
async def _acreate_with_stream(client: Any, kwargs: Dict[str, Any], task: Optional[str] = None) -> Any:
|
||
"""Async create() for stream-only providers: ``stream=True`` + aggregate the async chunks."""
|
||
stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs)
|
||
chunks = await client.chat.completions.create(**stream_kwargs)
|
||
if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
|
||
return chunks
|
||
return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
|
||
|
||
|
||
def _async_client_streams_internally(client: Any) -> bool:
|
||
"""Async twin of :func:`_client_streams_internally` (the async adapters are separate classes)."""
|
||
return isinstance(client, (AsyncCodexAuxiliaryClient, AsyncAnthropicAuxiliaryClient, AsyncBedrockAuxiliaryClient))
|
||
|
||
|
||
async def _acreate_with_progress(
|
||
client: Any, kwargs: Dict[str, Any], task: Optional[str] = None, *, force_stream: bool = False
|
||
) -> Any:
|
||
"""Async :func:`_create_with_progress`: stream + re-aggregate (ticking the hook per substantive
|
||
chunk) when a progress hook is active or the provider is stream-only; plain create otherwise."""
|
||
kwargs = bypass_chat_sdk_request_transform(kwargs, client)
|
||
_notify_aux_dispatch()
|
||
# Same contract as the sync twin (#114938): dispatch alone is not progress.
|
||
if (not _aux_progress_active() and not force_stream) or _async_client_streams_internally(client):
|
||
response = await client.chat.completions.create(**kwargs)
|
||
if not _async_client_streams_internally(client):
|
||
_notify_aux_provider_response()
|
||
return response
|
||
stream_kwargs, model, total_ceiling = _stream_request_plan(kwargs)
|
||
try:
|
||
chunks = await client.chat.completions.create(**stream_kwargs)
|
||
except Exception as exc:
|
||
# Only a rejected stream NEGOTIATION falls back to a plain call (mirrors the sync wrapper); a
|
||
# failure mid-consumption below reaches the classified recovery ladder instead of silently
|
||
# re-sending the whole prompt non-streaming.
|
||
if (force_stream or _is_transient_transport_error(exc) or _is_auth_error(exc)
|
||
or _is_payment_error(exc) or _is_rate_limit_error(exc)):
|
||
raise
|
||
logger.debug("Auxiliary %s: streamed async request failed (%s); retrying non-streaming",
|
||
task or "call", exc)
|
||
_notify_aux_dispatch()
|
||
response = await client.chat.completions.create(**kwargs)
|
||
_notify_aux_provider_response()
|
||
return response
|
||
if hasattr(chunks, "choices"): # shims may hand back a complete response despite stream=True
|
||
_notify_aux_provider_response()
|
||
return chunks
|
||
return await _aggregate_chat_stream_async(chunks, model=model, total_ceiling=total_ceiling)
|
||
|
||
|
||
# Shared request head + recovery ladder for call_llm / async_call_llm: the entry points differ
|
||
# only in how a request is awaited, so route resolution and the ordered recovery ladder are
|
||
# written once. The ladder is a generator yielding ``_LadderStep`` requests and receiving the
|
||
# response (or thrown exception), so rung ORDER and accept/re-raise contracts match on both wires.
|
||
_ResolvedAuxRoute = NamedTuple("_ResolvedAuxRoute", [
|
||
("client", Any), ("final_model", Optional[str]), ("resolved_provider", str),
|
||
("effective_provider", str)])
|
||
|
||
|
||
def _resolve_call_client(
|
||
task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
|
||
api_key: Optional[str], resolved_provider: str, resolved_model: Optional[str],
|
||
resolved_base_url: Optional[str], resolved_api_key: Optional[str],
|
||
resolved_api_mode: Optional[str], main_runtime: Optional[Dict[str, Any]], async_mode: bool,
|
||
) -> _ResolvedAuxRoute:
|
||
"""Resolve the client for one aux call: vision chain, or cached text client with the
|
||
explicit-provider fallback_chain / auto-chain rescue; RuntimeError when nothing is configured."""
|
||
effective_provider = resolved_provider
|
||
if task == "vision":
|
||
effective_provider, client, final_model = resolve_vision_provider_client(
|
||
provider=resolved_provider if resolved_provider != "auto" else provider,
|
||
model=resolved_model or model, base_url=resolved_base_url or base_url,
|
||
api_key=resolved_api_key or api_key, async_mode=async_mode, main_runtime=main_runtime,
|
||
)
|
||
if client is None and resolved_provider != "auto" and not resolved_base_url:
|
||
logger.warning("Vision provider %s unavailable, falling back to auto vision backends",
|
||
resolved_provider)
|
||
effective_provider, client, final_model = resolve_vision_provider_client(
|
||
provider="auto", model=resolved_model, async_mode=async_mode,
|
||
main_runtime=main_runtime)
|
||
if client is not None:
|
||
resolved_provider = effective_provider or resolved_provider
|
||
else:
|
||
client, final_model = _get_cached_client(
|
||
resolved_provider, resolved_model, async_mode=async_mode, base_url=resolved_base_url,
|
||
api_key=resolved_api_key, api_mode=resolved_api_mode, main_runtime=main_runtime,
|
||
task=task)
|
||
effective_provider = _effective_provider_for_client(client, resolved_provider)
|
||
if client is None:
|
||
# Explicit provider with no credentials: honor the task fallback_chain before
|
||
# raising (fallback entries may use OAuth / credential-pool auth).
|
||
_explicit = (resolved_provider or "").strip().lower()
|
||
if _explicit and _explicit not in {"auto", "openrouter", "custom"}:
|
||
fb_client, fb_model, fb_label = _try_configured_fallback_for_unavailable_client(
|
||
task, _explicit)
|
||
if fb_client is None:
|
||
nous_detail = nous_credential_failure_detail() if _explicit == "nous" else None
|
||
raise AuxiliaryClientUnavailable(
|
||
nous_detail or missing_provider_credentials_message(_explicit))
|
||
client, final_model = fb_client, fb_model
|
||
if async_mode:
|
||
client, final_model = _to_async_client(
|
||
fb_client, fb_model or "", is_vision=(task == "vision"))
|
||
resolved_provider = fb_label or resolved_provider
|
||
effective_provider = resolved_provider
|
||
# Auto/custom with no credentials: walk the full auto chain (not just OpenRouter).
|
||
# model=None so each provider uses its own default.
|
||
if client is None and not resolved_base_url:
|
||
logger.info("Auxiliary %s: provider %s unavailable, trying auto-detection chain",
|
||
task or "call", resolved_provider)
|
||
client, final_model = _get_cached_client(
|
||
"auto", async_mode=async_mode, main_runtime=main_runtime, task=task)
|
||
effective_provider = _effective_provider_for_client(client, "auto")
|
||
if client is None:
|
||
raise AuxiliaryClientUnavailable(f"No LLM provider configured for task={task} "
|
||
f"provider={resolved_provider}. Run: hermes setup")
|
||
return _ResolvedAuxRoute(client, final_model, resolved_provider, effective_provider)
|
||
|
||
|
||
_PreparedAuxRequest = NamedTuple("_PreparedAuxRequest", [
|
||
("client", Any), ("final_model", Optional[str]), ("kwargs", Dict[str, Any]),
|
||
("resolved_provider", str), ("request_provider", str), ("resolved_model", Optional[str]),
|
||
("resolved_base_url", Optional[str]), ("resolved_api_key", Optional[str]),
|
||
("resolved_api_mode", Optional[str]), ("effective_timeout", float),
|
||
("effective_extra_body", Dict[str, Any]), ("base_info", str)])
|
||
|
||
|
||
def _prepare_aux_request(
|
||
task: Optional[str], *, provider: Optional[str], model: Optional[str], base_url: Optional[str],
|
||
api_key: Optional[str], main_runtime: Dict[str, Any], messages: list,
|
||
temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
|
||
timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
|
||
extra_headers: Optional[Dict[str, str]], api_mode: Optional[str],
|
||
route_info: Optional[Dict[str, str]], async_mode: bool,
|
||
) -> _PreparedAuxRequest:
|
||
"""Shared head of call_llm/async_call_llm: resolve route + client, publish it, build request kwargs.
|
||
Sync-only: compression fast lane, per-request ``extra_headers``, and ``base_info`` falling
|
||
back to the resolved base_url when the client exposes none."""
|
||
resolved_provider, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model(
|
||
task, provider, model, base_url, api_key)
|
||
if api_mode:
|
||
resolved_api_mode = api_mode
|
||
effective_extra_body = _get_task_extra_body(task)
|
||
effective_extra_body.update(extra_body or {})
|
||
client, final_model, resolved_provider, effective_provider = _resolve_call_client(
|
||
task, provider=provider, model=model, base_url=base_url, api_key=api_key,
|
||
resolved_provider=resolved_provider, resolved_model=resolved_model,
|
||
resolved_base_url=resolved_base_url, resolved_api_key=resolved_api_key,
|
||
resolved_api_mode=resolved_api_mode, main_runtime=main_runtime, async_mode=async_mode,
|
||
)
|
||
effective_timeout = _effective_aux_timeout(task, timeout)
|
||
# Codex-Responses-only: real SDK clients reject an unrecognized ``no_progress_timeout``
|
||
# kwarg, so only resolve/forward it when the route is actually a Codex stream (#108104).
|
||
no_progress_timeout = (
|
||
_get_task_no_progress_timeout(task)
|
||
if isinstance(client, (CodexAuxiliaryClient, AsyncCodexAuxiliaryClient)) else None
|
||
)
|
||
request_provider = effective_provider or resolved_provider
|
||
if not async_mode:
|
||
compression_config = _get_auxiliary_task_config("compression") if task == "compression" else {}
|
||
_, effective_extra_body = _compression_fast_lane_controls(
|
||
task, actual_provider=request_provider, actual_model=final_model,
|
||
requested_provider=provider, requested_model=model, route_config=compression_config,
|
||
leak_guard_config=compression_config, max_tokens=max_tokens,
|
||
extra_body=effective_extra_body,
|
||
)
|
||
_set_relay_auxiliary_route(request_provider, final_model, resolved_api_mode)
|
||
_record_route_info(route_info, _fallback_provider_from_label(request_provider), final_model)
|
||
if async_mode:
|
||
base_info = str(getattr(client, "base_url", "") or "")
|
||
else:
|
||
base_info = str(getattr(client, "base_url", resolved_base_url) or "")
|
||
if task:
|
||
logger.info("Auxiliary %s: using %s (%s)%s",
|
||
task, request_provider or "auto", final_model or "default",
|
||
f" at {base_info}" if base_info and "openrouter" not in base_info else "")
|
||
# Client's actual base_url so endpoint-specific temperature overrides work on
|
||
# auto-detected routes (api.moonshot.ai vs api.kimi.com/coding).
|
||
kwargs = _build_call_kwargs(
|
||
request_provider, final_model, messages, temperature=temperature, max_tokens=max_tokens,
|
||
tools=tools, timeout=effective_timeout, extra_body=effective_extra_body,
|
||
reasoning_config=reasoning_config, base_url=base_info or resolved_base_url, task=task,
|
||
no_progress_timeout=no_progress_timeout)
|
||
if extra_headers:
|
||
kwargs["extra_headers"] = dict(extra_headers)
|
||
# Convert image blocks for Anthropic-compatible endpoints (e.g. MiniMax)
|
||
client_base = str(getattr(client, "base_url", "") or "")
|
||
if _is_anthropic_compat_endpoint(request_provider, client_base):
|
||
kwargs["messages"] = _convert_openai_images_to_anthropic(kwargs["messages"])
|
||
return _PreparedAuxRequest(
|
||
client, final_model, kwargs, resolved_provider, request_provider, resolved_model,
|
||
resolved_base_url, resolved_api_key, resolved_api_mode, effective_timeout,
|
||
effective_extra_body, base_info)
|
||
|
||
|
||
class _LadderStep(NamedTuple):
|
||
"""A provider request the ladder asks its driver to perform. kind: "call" (client, kwargs) |
|
||
"retry_same_provider" (provider, model) | "fallback" (fb_client, fb_model, fb_label)."""
|
||
kind: str
|
||
args: tuple
|
||
|
||
|
||
# Ordered (predicate, reason) pairs for the provider-fallback rung: first match
|
||
# wins, so a payment-flavoured 429 reads as "payment error", not "rate limit".
|
||
_FALLBACK_REASONS: Tuple[Tuple[Callable[[Exception], bool], str], ...] = (
|
||
(_is_auth_error, "auth error"), (_is_payment_error, "payment error"),
|
||
(_is_rate_limit_error, "rate limit"), (_is_model_incompatible_error, "model incompatible with route"),
|
||
(_is_invalid_aux_response_error, "invalid provider response"),
|
||
# A status-less in-stream ``error`` event (SSE committed 200) is a route failure (#101538).
|
||
(_is_statusless_structured_provider_error, "structured provider error"),
|
||
# Before the connection-error rung (its superset): a full-budget timeout must be named as one, or
|
||
# a slow local model reads as an unreachable endpoint (#89445).
|
||
(_is_timeout_error, "request timed out"), (_is_connection_error, "connection error"),
|
||
)
|
||
|
||
|
||
def _rung(step: "_LadderStep", accept: Callable[[Exception], bool]):
|
||
"""One ladder rung: perform ``step``; yields ``(response, None)`` on success,
|
||
``(None, exc)`` when ``accept(exc)`` lets the next rung handle it, else re-raises."""
|
||
try:
|
||
result = yield step
|
||
except Exception as exc:
|
||
if not accept(exc):
|
||
raise
|
||
return None, exc
|
||
return result, None
|
||
|
||
|
||
def _param_rung_accepts(exc: Exception) -> bool:
|
||
"""After a parameter-strip retry: fall through to the max_tokens/payment/auth
|
||
chains with the stripped kwargs; re-raise anything those chains won't handle.
|
||
A 429 on the retry is the credential/provider-fallback rungs' job, so it falls
|
||
through too (the pre-ladder max_tokens rung accepted rate limits)."""
|
||
return (_is_payment_error(exc) or _is_connection_error(exc) or _is_auth_error(exc)
|
||
or _is_rate_limit_error(exc) or _is_statusless_structured_provider_error(exc)
|
||
or "max_tokens" in str(exc) or "unsupported_parameter" in str(exc)
|
||
# Parameter rungs chain in any order (a reasoning-strip retry can 400 on temperature,
|
||
# a temperature-strip retry on max_tokens), and a route-gating 400 after a strip still
|
||
# reaches the provider-fallback rung.
|
||
or _is_unsupported_parameter_error(exc, "temperature")
|
||
or _is_reasoning_field_rejection(exc) or _is_reasoning_required_rejection(exc)
|
||
or _is_structured_output_rejection(exc)
|
||
or _is_model_incompatible_error(exc))
|
||
|
||
|
||
def _credential_rung_accepts(exc: Exception) -> bool:
|
||
return _is_auth_error(exc) or _is_payment_error(exc) or _is_rate_limit_error(exc)
|
||
|
||
|
||
# Immutable route context shared by the recovery rungs.
|
||
_LadderRoute = NamedTuple("_LadderRoute", [
|
||
("client", Any), ("task", Optional[str]), ("tag", str), ("async_mode", bool), ("base_info", str),
|
||
("resolved_provider", str), ("resolved_model", Optional[str]), ("resolved_base_url", Optional[str]),
|
||
("resolved_api_key", Optional[str]), ("resolved_api_mode", Optional[str]),
|
||
("final_model", Optional[str]), ("main_runtime", Optional[Dict[str, Any]]),
|
||
("route_info", Optional[Dict[str, str]]), ("timeout", Optional[float]),
|
||
])
|
||
|
||
|
||
def _without_temperature(kwargs: dict) -> Optional[dict]:
|
||
"""Copy *kwargs* without ``temperature``; None when it was not sent."""
|
||
return {k: v for k, v in kwargs.items() if k != "temperature"} if "temperature" in kwargs else None
|
||
|
||
|
||
def _without_max_tokens(kwargs: dict) -> dict:
|
||
"""Copy *kwargs* without either output cap. Unlike the other strips this never returns None: a
|
||
route can translate the caller's cap into a field the wire kwargs no longer show (Codex
|
||
Responses), and the provider's own gateway may inject one — the 400 still names ``max_tokens``
|
||
and the identical request completes on retry (registry class #89897/#90257)."""
|
||
return {k: v for k, v in kwargs.items() if k not in ("max_tokens", "max_completion_tokens")}
|
||
|
||
|
||
def _is_max_tokens_rejection(exc: Exception, client: Any) -> bool:
|
||
err_str = str(exc)
|
||
# ZAI vision models reject max_tokens with code 1210 and a message that never
|
||
# mentions "max_tokens", so detect it explicitly.
|
||
is_zai_param_error = "1210" in err_str and "bigmodel" in str(getattr(client, "base_url", ""))
|
||
return ("max_tokens" in err_str or "unsupported_parameter" in err_str
|
||
or _is_unsupported_parameter_error(exc, "max_tokens") or is_zai_param_error)
|
||
|
||
|
||
def _parameter_rungs(client: Any, max_tokens: Optional[int]) -> tuple:
|
||
"""Ordered ``(matches, strip, log message, remember)`` parameter rungs; ``strip`` returns None
|
||
when the field was not on the wire, so an unchanged request is never re-sent; ``remember``
|
||
(optional) records the rejection per route so the next call omits the field up front."""
|
||
return (
|
||
(lambda exc: _is_unsupported_parameter_error(exc, "temperature"), _without_temperature,
|
||
"provider rejected temperature; retrying without it", remember_temperature_rejection),
|
||
(_is_structured_output_rejection, _without_structured_output_format,
|
||
"provider rejected the structured-output format field; retrying without it "
|
||
"(schema enforcement degrades to prompt compliance)", remember_structured_output_rejection),
|
||
# A chat-only model on an OpenAI-compatible relay rejects the profile's thinking-off encoding
|
||
# (top-level ``reasoning_effort: none``), and strict-schema gateways reject the generic
|
||
# ``extra_body.reasoning`` fallback outright (#109774); the caller only wanted "no thinking",
|
||
# so retry with every reasoning field omitted and let the route default apply (#112781).
|
||
# The endpoint refuses the *disable* rather than the field (Nous Portal gpt-6-astra: "Reasoning is
|
||
# mandatory ... cannot be disabled"): step the effort up to the floor and remember the route so
|
||
# the next thinking-off aux call starts there. Ordered before the strip so a floor that still
|
||
# 400s falls through to it.
|
||
(_is_reasoning_required_rejection, with_reasoning_floor,
|
||
"provider requires reasoning; retrying at the floor effort", remember_reasoning_floor),
|
||
(_is_reasoning_field_rejection, _without_reasoning_fields,
|
||
"provider rejected the reasoning field; retrying without it (route default applies)", None),
|
||
(lambda exc: max_tokens is not None and _is_max_tokens_rejection(exc, client), _without_max_tokens,
|
||
"provider rejected the output cap; retrying without it", None),
|
||
)
|
||
|
||
|
||
def _ladder_parameter_rungs(
|
||
first_err: Exception, route: _LadderRoute, kwargs: Dict[str, Any], max_tokens: Optional[int],
|
||
):
|
||
"""Parameter rungs: retry without temperature / structured-output format / reasoning field /
|
||
max_tokens. Rungs chain in whichever order the provider raises them (reasoning models reject
|
||
temperature AND max_tokens; a reasoning-strip retry can then trip temperature, #78273), each
|
||
field stripped at most once, so a request with N rejected fields recovers in N retries.
|
||
Returns ``(response, None, kwargs)`` or ``(None, narrowed_err, stripped_kwargs)``."""
|
||
client, task, tag = route.client, route.task, route.tag
|
||
rungs = list(_parameter_rungs(client, max_tokens))
|
||
while rungs:
|
||
hit = next((rung for rung in rungs
|
||
if rung[0](first_err) and rung[1](kwargs) is not None), None)
|
||
if hit is None:
|
||
break
|
||
rungs.remove(hit)
|
||
matches, strip, message, remember = hit
|
||
retry_kwargs = strip(kwargs)
|
||
logger.info("Auxiliary %s%s: %s: %s", task or "call", tag, message, first_err)
|
||
rejection = first_err
|
||
resp, first_err = yield from _rung(
|
||
_LadderStep("call", (client, retry_kwargs)), _param_rung_accepts)
|
||
if first_err is None:
|
||
if remember is not None:
|
||
# Same key _build_call_kwargs looks up (base_info or resolved_base_url), so the
|
||
# memory hits when the client exposes no base_url but the task resolved one.
|
||
remember(route.resolved_provider, route.base_info or route.resolved_base_url,
|
||
kwargs, rejection)
|
||
return resp, None, retry_kwargs
|
||
kwargs = retry_kwargs
|
||
return None, first_err, kwargs
|
||
|
||
|
||
def _refreshed_nous_step(route: _LadderRoute, kwargs: Dict[str, Any], message: str) -> Optional[_LadderStep]:
|
||
"""Rebuild the Nous client after a credential event; None when nothing refreshed."""
|
||
refreshed_client, refreshed_model = _refresh_nous_auxiliary_client(
|
||
cache_provider=route.resolved_provider or "nous", model=route.final_model,
|
||
lookup_model=route.resolved_model, lookup_task=route.task, async_mode=route.async_mode,
|
||
base_url=route.resolved_base_url, api_key=route.resolved_api_key,
|
||
api_mode=route.resolved_api_mode, main_runtime=route.main_runtime,
|
||
is_vision=(route.task == "vision"),
|
||
)
|
||
if refreshed_client is None:
|
||
return None
|
||
logger.info(message, route.task or "call", route.tag)
|
||
if refreshed_model and refreshed_model != kwargs.get("model"):
|
||
kwargs["model"] = refreshed_model
|
||
return _LadderStep("call", (refreshed_client, kwargs))
|
||
|
||
|
||
def _ladder_nous_rungs(
|
||
first_err: Exception, route: _LadderRoute, kwargs: Dict[str, Any], client_is_nous: bool,
|
||
):
|
||
"""Nous-only rungs: stale-model self-heal, paid-account refresh, 401 refresh.
|
||
Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
|
||
client, task, tag = route.client, route.task, route.tag
|
||
# A long-lived process can pin a Portal model since dropped from the catalog (every call
|
||
# 404s); force a fresh Portal fetch and retry once.
|
||
if _is_model_not_found_error(first_err) and client_is_nous:
|
||
healed_model = _refresh_nous_recommended_model(
|
||
vision=(task == "vision"), stale_model=kwargs.get("model"))
|
||
if healed_model and healed_model != kwargs.get("model"):
|
||
logger.warning("Auxiliary %s%s: model %r no longer in Nous catalog; "
|
||
"retrying with refreshed recommendation %r",
|
||
task or "call", tag, kwargs.get("model"), healed_model)
|
||
kwargs["model"] = healed_model
|
||
resp, first_err = yield from _rung(_LadderStep("call", (client, kwargs)), lambda exc: True)
|
||
if first_err is None:
|
||
return resp, None
|
||
# Auth refresh parity with the main agent.
|
||
if _is_payment_error(first_err) and client_is_nous and _nous_portal_account_has_fresh_paid_access():
|
||
step = _refreshed_nous_step(
|
||
route, kwargs,
|
||
"Auxiliary %s%s: refreshed Nous runtime credentials after paid account check, retrying")
|
||
if step is not None:
|
||
resp, first_err = yield from _rung(
|
||
step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
|
||
if first_err is None:
|
||
return resp, None
|
||
if _is_auth_error(first_err) and client_is_nous:
|
||
step = _refreshed_nous_step(
|
||
route, kwargs, "Auxiliary %s%s: refreshed Nous runtime credentials after 401, retrying")
|
||
if step is not None:
|
||
resp, first_err = yield from _rung(
|
||
step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
|
||
if first_err is None:
|
||
return resp, None
|
||
return None, first_err
|
||
|
||
|
||
def _ladder_credential_rungs(
|
||
first_err: Exception, route: _LadderRoute, kwargs: Dict[str, Any], client_is_nous: bool,
|
||
):
|
||
"""OAuth credential refresh + same-provider retry, then credential-pool rotation.
|
||
Returns ``(response, None)`` or ``(None, first_err)`` to fall through."""
|
||
client, task, tag, resolved_provider = route.client, route.task, route.tag, route.resolved_provider
|
||
auth_refresh_provider = _auth_refresh_provider_for_route(
|
||
resolved_provider, route.base_info, _effective_provider_for_client(client, ""))
|
||
if (_is_auth_error(first_err) and auth_refresh_provider not in {"auto", "", None}
|
||
and not client_is_nous):
|
||
refresh_kwargs = ({"failed_api_key": getattr(client, "api_key", "")}
|
||
if auth_refresh_provider == "anthropic" else {})
|
||
if _refresh_provider_credentials(auth_refresh_provider, **refresh_kwargs):
|
||
if auth_refresh_provider != _normalize_aux_provider(resolved_provider):
|
||
# The stale client is cached under the route label (e.g. "auto"), not the
|
||
# concrete backend we refreshed.
|
||
_evict_cached_clients(resolved_provider)
|
||
logger.info("Auxiliary %s%s: refreshed %s credentials after auth error, retrying",
|
||
task or "call", tag, auth_refresh_provider)
|
||
step = _LadderStep(
|
||
"retry_same_provider",
|
||
(auth_refresh_provider, route.resolved_model or route.final_model))
|
||
resp, first_err = yield from _rung(
|
||
step, lambda exc: _credential_rung_accepts(exc) or _is_connection_error(exc))
|
||
if first_err is None:
|
||
return resp, None
|
||
# ``first_err`` is now the retry's own failure, not the original auth error: the
|
||
# pool gate below and the ladder tail's eviction check both read this narrowed
|
||
# value. An unclaimed failure (e.g. a 500) re-raised out of ``_rung`` above
|
||
# instead, since the provider-fallback rung only acts on ``_FALLBACK_REASONS``.
|
||
pool_provider = _recoverable_pool_provider(resolved_provider, client, main_runtime=route.main_runtime)
|
||
# Capture the exact key used so recovery finds the right pool entry even if another
|
||
# process rotated the pool meanwhile (current() would be None).
|
||
_client_api_key = str(getattr(client, "api_key", "") or "")
|
||
# Gate on the narrowed error: a connection failure from the retry above arrives here
|
||
# unaccepted on purpose (a fresh key cannot fix an unreachable endpoint), so rotation
|
||
# is skipped and ``first_err`` is handed to the provider-fallback chain as-is.
|
||
if pool_provider and _credential_rung_accepts(first_err):
|
||
recovery_err = first_err
|
||
# Skip the extra retry for clear payment/quota errors — the endpoint won't accept
|
||
# another request with the same exhausted key.
|
||
if _is_rate_limit_error(first_err) and not _is_payment_error(first_err):
|
||
resp, recovery_err = yield from _rung(
|
||
_LadderStep("call", (client, kwargs)), _credential_rung_accepts)
|
||
if recovery_err is None:
|
||
return resp, None
|
||
if _recover_provider_pool(pool_provider, recovery_err, failed_api_key=_client_api_key):
|
||
logger.info("Auxiliary %s%s: recovered %s via credential-pool rotation after %s",
|
||
task or "call", tag, pool_provider, type(recovery_err).__name__)
|
||
try:
|
||
return (yield _LadderStep(
|
||
"retry_same_provider", (resolved_provider, route.resolved_model))), None
|
||
except Exception as retry2_err:
|
||
# Rotated key also hit a wall: mark it now so concurrent processes skip it,
|
||
# then fall through to the provider fallback.
|
||
if (_is_payment_error(retry2_err) or _is_auth_error(retry2_err)
|
||
or _is_rate_limit_error(retry2_err)):
|
||
_recover_provider_pool(pool_provider, retry2_err)
|
||
first_err = retry2_err
|
||
else:
|
||
raise
|
||
return None, first_err
|
||
|
||
|
||
def _next_fallback_after_quarantine(
|
||
task: Optional[str], resolved_provider: str, is_auto: bool, route: _LadderRoute,
|
||
failed_model: Optional[str], failure_scope: Any, *, task_chain_only: bool = False,
|
||
) -> Tuple[Optional[Any], Optional[str], str]:
|
||
"""Next candidate after a fallback entry was quarantined mid-request (dead credential or a
|
||
capacity error): remaining configured entries (task chain, then main chain on auto) before the
|
||
discovery chain. ``task_chain_only`` (explicit-provider auth error) stops at the task chain —
|
||
the user never opted that task into discovery or the main model."""
|
||
reason = "fallback candidate unavailable"
|
||
fb = _try_configured_fallback_chain(
|
||
task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
|
||
failed_base_url=route.base_info, failure_scope=failure_scope)
|
||
if task_chain_only:
|
||
return fb
|
||
if fb[0] is None and is_auto:
|
||
fb = _try_main_fallback_chain(
|
||
task, resolved_provider or "auto", reason=reason, failed_model=failed_model,
|
||
failed_base_url=route.base_info, failure_scope=failure_scope)
|
||
if fb[0] is None:
|
||
fb = _try_payment_fallback(
|
||
resolved_provider, task, reason=reason, failed_base_url=route.base_info,
|
||
failure_scope=failure_scope, main_runtime=route.main_runtime)
|
||
return fb
|
||
|
||
|
||
def _ladder_provider_fallback(first_err: Exception, route: _LadderRoute):
|
||
"""Last rung: other providers (per-task chain; then auto: main fallback chain + discovery
|
||
chain, explicit: main-agent-model net). Returns the response or None.
|
||
Capacity errors (payment/quota, connection, exhausted 429, model incompatible, malformed
|
||
response) bypass the explicit-provider gate — the provider cannot serve this request
|
||
regardless of user intent. Auth errors from an explicit provider may only use the task's
|
||
own configured fallback_chain; they never imply an unconfigured provider hop."""
|
||
task, tag, resolved_provider = route.task, route.tag, route.resolved_provider
|
||
# Respect explicit provider choice for transient errors (auth, request validation, etc.) but allow
|
||
# fallback when the provider clearly cannot serve the request due to capacity: payment/quota exhaustion
|
||
# and connection failures are capacity problems, not request constraints. See #26803: daily token quota
|
||
# (429 + "too many tokens per day") must fall back just like a 402 credit error.
|
||
# Rate limits are included: after retries are exhausted, a 429 means the provider is at capacity. See
|
||
# #52228. See #26803: daily token quota must fall back like a 402 credit error.
|
||
is_auto = resolved_provider in {"auto", "", None}
|
||
reason = next((label for predicate, label in _FALLBACK_REASONS if predicate(first_err)), None)
|
||
is_capacity_error = any(
|
||
predicate(first_err) for predicate, label in _FALLBACK_REASONS if label != "auth error")
|
||
task_chain = _get_auxiliary_task_config(task).get("fallback_chain") if task else None
|
||
has_task_fallback_chain = isinstance(task_chain, list) and bool(task_chain)
|
||
explicit_auth_with_task_chain = (
|
||
reason == "auth error" and not is_auto and has_task_fallback_chain
|
||
)
|
||
if reason is None or not (is_auto or is_capacity_error or explicit_auth_with_task_chain):
|
||
return None
|
||
if reason == "payment error":
|
||
# Mark the concrete backend (not the "auto" label) unhealthy so later aux calls skip
|
||
# it instead of paying another doomed RTT.
|
||
_mark_provider_unhealthy(
|
||
_recoverable_pool_provider(resolved_provider, route.client, main_runtime=route.main_runtime)
|
||
or resolved_provider, base_url=route.base_info)
|
||
if reason == "request timed out":
|
||
# WARNING, naming the endpoint, the budget and the knob: the only other trace of a slow
|
||
# local model is the fallback provider's complaint about a model it never had (#89445).
|
||
logger.warning("Auxiliary %s%s: request to %s timed out after %ss (raise auxiliary.%s.timeout "
|
||
"for slow or reasoning models) on %s, trying fallback",
|
||
task or "call", tag, route.base_info or resolved_provider, route.timeout,
|
||
task or "call", resolved_provider)
|
||
else:
|
||
logger.info("Auxiliary %s%s: %s on %s (%s), trying fallback",
|
||
task or "call", tag, reason, resolved_provider, first_err)
|
||
# Skip only the failed model for model-specific failures; 401/402 are provider-wide, so
|
||
# auth keeps skipping the credential surface, while billing is scoped to the endpoint:
|
||
# separate custom URLs can carry separate credentials (or no billing relationship at all).
|
||
_chain_failed_model = None if reason in ("auth error", "payment error") else route.final_model
|
||
from agent.backend_identity import FailureScope
|
||
_chain_failure_scope = (
|
||
FailureScope.ENDPOINT
|
||
if reason == "payment error" and _custom_health_base_url(resolved_provider, route.base_info)
|
||
else None
|
||
)
|
||
fb_client, fb_model, fb_label = _try_configured_fallback_chain(
|
||
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
|
||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||
if fb_client is None and is_auto:
|
||
fb_client, fb_model, fb_label = _try_main_fallback_chain(
|
||
task, resolved_provider or "auto", reason=reason, failed_model=_chain_failed_model,
|
||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||
if fb_client is None:
|
||
fb_client, fb_model, fb_label = _try_payment_fallback(
|
||
resolved_provider, task, reason=reason, failed_base_url=route.base_info,
|
||
failure_scope=_chain_failure_scope, main_runtime=route.main_runtime)
|
||
elif fb_client is None and not explicit_auth_with_task_chain:
|
||
fb_client, fb_model, fb_label = _try_main_agent_model_fallback(
|
||
resolved_provider, task, reason=reason, failed_model=_chain_failed_model,
|
||
failed_base_url=route.base_info, failure_scope=_chain_failure_scope)
|
||
# Ordered walk: a candidate that returns None was quarantined (dead credential or a capacity
|
||
# error such as a quota 429) and is now unhealthy, so re-walking the CONFIGURED chains first
|
||
# lands on the next entry, then discovery where the selection policy allows it (#106367).
|
||
# Bounded by construction: every pass quarantines its candidate and the walk stops as soon as
|
||
# re-selection hands back a lane already tried, so each lane is attempted at most once.
|
||
tried_lanes: set = set()
|
||
while fb_client is not None:
|
||
lane = (fb_label, fb_model, str(getattr(fb_client, "base_url", "") or ""))
|
||
if lane in tried_lanes:
|
||
break
|
||
tried_lanes.add(lane)
|
||
_record_route_info(route.route_info, _fallback_provider_from_label(fb_label), fb_model)
|
||
fb_resp = yield _LadderStep("fallback", (fb_client, fb_model, fb_label))
|
||
if fb_resp is not None:
|
||
return fb_resp
|
||
fb_client, fb_model, fb_label = _next_fallback_after_quarantine(
|
||
task, resolved_provider, is_auto, route, _chain_failed_model, _chain_failure_scope,
|
||
task_chain_only=explicit_auth_with_task_chain)
|
||
# All fallback layers exhausted — one user-visible warning, then re-raise.
|
||
logger.warning("Auxiliary %s%s: %s on %s and all fallbacks exhausted "
|
||
# All fallback layers exhausted — emit a single user-visible warning so the operator
|
||
# knows aux task is about to fail. (#26882) The error itself is re-raised below.
|
||
# (#26882)
|
||
"(fallback_chain + main agent model). Raising the primary error.",
|
||
task or "call", tag, reason, resolved_provider)
|
||
return None
|
||
|
||
|
||
def _aux_recovery_ladder(
|
||
first_err: Exception, *, client: Any, kwargs: Dict[str, Any], task: Optional[str],
|
||
async_mode: bool, base_info: str, resolved_provider: str, resolved_model: Optional[str],
|
||
resolved_base_url: Optional[str], resolved_api_key: Optional[str],
|
||
resolved_api_mode: Optional[str], final_model: Optional[str], max_tokens: Optional[int],
|
||
main_runtime: Optional[Dict[str, Any]], route_info: Optional[Dict[str, str]],
|
||
):
|
||
"""Ordered recovery rungs after the primary request failed (generator): parameter
|
||
strips → Nous heal/refresh → credential refresh/pool rotation → provider fallback.
|
||
Each rung returns a response, narrows ``first_err`` and falls through, or re-raises.
|
||
Raises the narrowed ``first_err`` when exhausted (after evicting a connection-poisoned client)."""
|
||
tag = " (async)" if async_mode else ""
|
||
route = _LadderRoute(
|
||
client, task, tag, async_mode, base_info, resolved_provider, resolved_model,
|
||
resolved_base_url, resolved_api_key, resolved_api_mode, final_model, main_runtime, route_info,
|
||
kwargs.get("timeout"))
|
||
resp, first_err, kwargs = yield from _ladder_parameter_rungs(first_err, route, kwargs, max_tokens)
|
||
if first_err is None:
|
||
return resp
|
||
client_is_nous = (resolved_provider == "nous"
|
||
or base_url_host_matches(base_info, "inference-api.nousresearch.com"))
|
||
resp, first_err = yield from _ladder_nous_rungs(first_err, route, kwargs, client_is_nous)
|
||
if first_err is None:
|
||
return resp
|
||
resp, first_err = yield from _ladder_credential_rungs(first_err, route, kwargs, client_is_nous)
|
||
if first_err is None:
|
||
return resp
|
||
resp = yield from _ladder_provider_fallback(first_err, route)
|
||
if resp is not None:
|
||
return resp
|
||
# Connection/timeout errors poison the cached client (closed transport, half-read
|
||
# stream); evict so the next aux call rebuilds a fresh one.
|
||
# Reached only when no fallback answered, so the next auxiliary call rebuilds a fresh
|
||
# client instead of reusing the dead one. ``first_err`` is the narrowed error from the
|
||
# rungs above, not necessarily the original one. See issue #23432.
|
||
# Mirror the sync path: drop poisoned clients on connection/timeout so the next aux call rebuilds. See
|
||
# issue #23432.
|
||
if _is_connection_error(first_err):
|
||
try:
|
||
_evict_cached_client_instance(client)
|
||
except Exception:
|
||
logger.debug("Auxiliary%s: cache eviction after connection error failed",
|
||
tag, exc_info=True)
|
||
# The narrowed error is the actionable one (e.g. a 404 "requires credits" from the
|
||
# retry after a healed 401), so surface it rather than the original.
|
||
raise first_err
|
||
|
||
|
||
def _drive_ladder(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
|
||
"""Run a ladder generator, feeding each step's result (or exception) back in."""
|
||
try:
|
||
step = next(ladder)
|
||
while True:
|
||
try:
|
||
result = perform(step)
|
||
except Exception as exc:
|
||
step = ladder.throw(exc)
|
||
else:
|
||
step = ladder.send(result)
|
||
except StopIteration as stop:
|
||
return stop.value
|
||
|
||
|
||
async def _drive_ladder_async(ladder, perform: Callable[[_LadderStep], Any]) -> Any:
|
||
"""Async twin of :func:`_drive_ladder` (``perform`` is awaited)."""
|
||
try:
|
||
step = next(ladder)
|
||
while True:
|
||
try:
|
||
result = await perform(step)
|
||
except Exception as exc:
|
||
step = ladder.throw(exc)
|
||
else:
|
||
step = ladder.send(result)
|
||
except StopIteration as stop:
|
||
return stop.value
|
||
|
||
|
||
def _elapsed_ms(started_at: float, now: Optional[float] = None) -> int:
|
||
"""Whole milliseconds since ``started_at`` (clamped at 0)."""
|
||
return max(0, int(((time.monotonic() if now is None else now) - started_at) * 1000))
|
||
|
||
|
||
def _stamp_latency_once(latency_info: Optional[Dict[str, int]], key: str, started_at: float) -> None:
|
||
"""Record ``key`` in ``latency_info`` the first time it fires."""
|
||
if latency_info is not None and key not in latency_info:
|
||
latency_info[key] = _elapsed_ms(started_at)
|
||
|
||
|
||
@_relay_auxiliary_call
|
||
def call_llm(
|
||
task: str = None, *, provider: str = None, model: str = None, base_url: str = None,
|
||
api_key: str = None, main_runtime: Optional[Dict[str, Any]] = None, messages: list,
|
||
temperature: Optional[float] = None, max_tokens: int = None, tools: list = None,
|
||
timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None,
|
||
extra_headers: Optional[Dict[str, str]] = None, api_mode: str = None, stream: bool = False,
|
||
stream_options: dict = None, route_info: Optional[Dict[str, str]] = None,
|
||
latency_info: Optional[Dict[str, int]] = None,
|
||
) -> Any:
|
||
"""Run an auxiliary LLM request, applying the configured task limit."""
|
||
queue_started_at = time.monotonic()
|
||
semaphore = _acquire_sync_aux_semaphore(task)
|
||
if semaphore is not None:
|
||
semaphore.acquire()
|
||
request_started_at = time.monotonic()
|
||
if latency_info is not None:
|
||
latency_info["queue_wait_ms"] = _elapsed_ms(queue_started_at, request_started_at)
|
||
prior_progress_hook = getattr(_aux_progress, "hook", None)
|
||
try:
|
||
with (
|
||
scoped_runtime_main(main_runtime),
|
||
aux_progress_hook(
|
||
prior_progress_hook
|
||
if callable(prior_progress_hook)
|
||
else ((lambda: None) if latency_info is not None else None)
|
||
),
|
||
_aux_thread_local_hook(_aux_dispatch, functools.partial(
|
||
_stamp_latency_once, latency_info, "provider_dispatch_ms", request_started_at)),
|
||
_aux_thread_local_hook(_aux_provider_response, functools.partial(
|
||
_stamp_latency_once, latency_info, "time_to_first_progress_ms", request_started_at)),
|
||
):
|
||
response = _call_llm_impl(
|
||
task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
|
||
main_runtime=main_runtime, messages=messages, temperature=temperature,
|
||
max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
|
||
reasoning_config=reasoning_config, extra_headers=extra_headers, api_mode=api_mode,
|
||
stream=stream, stream_options=stream_options, route_info=route_info,
|
||
)
|
||
if stream and semaphore is not None:
|
||
stream_semaphore = semaphore
|
||
semaphore = None
|
||
return _release_sync_semaphore_after_stream(response, stream_semaphore)
|
||
return response
|
||
finally:
|
||
if latency_info is not None:
|
||
latency_info["summary_generation_ms"] = _elapsed_ms(request_started_at)
|
||
if semaphore is not None:
|
||
semaphore.release()
|
||
|
||
|
||
def _release_sync_semaphore_after_stream(stream: Any, semaphore: threading.BoundedSemaphore):
|
||
"""Release a permit only after a streaming response is consumed or closed."""
|
||
try:
|
||
yield from stream
|
||
finally:
|
||
try:
|
||
close = getattr(stream, "close", None)
|
||
if callable(close):
|
||
close()
|
||
finally:
|
||
semaphore.release()
|
||
|
||
|
||
def _plan_aux_call(
|
||
task: Optional[str], *, async_mode: bool, provider: Optional[str], model: Optional[str],
|
||
base_url: Optional[str], api_key: Optional[str], main_runtime: Optional[Dict[str, Any]],
|
||
messages: list, temperature: Optional[float], max_tokens: Optional[int], tools: Optional[list],
|
||
timeout: Optional[float], extra_body: Optional[dict], reasoning_config: Optional[dict],
|
||
extra_headers: Optional[Dict[str, str]], api_mode: Optional[str],
|
||
route_info: Optional[Dict[str, str]],
|
||
) -> Tuple[_PreparedAuxRequest, Dict[str, Any], Dict[str, Any]]:
|
||
"""Shared head of both call impls: prepare the request and bundle the kwargs the recovery
|
||
drivers pass to ``_retry_same_provider_*`` / ``_call_fallback_candidate_*``. One immutable
|
||
runtime snapshot for keying/resolution/retries/fallbacks, so a concurrent /model switch
|
||
can't mix key and client from different runtimes."""
|
||
main_runtime = _normalize_main_runtime(main_runtime)
|
||
req = _prepare_aux_request(
|
||
task, provider=provider, model=model, base_url=base_url, api_key=api_key,
|
||
main_runtime=main_runtime, messages=messages, temperature=temperature,
|
||
max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
|
||
reasoning_config=reasoning_config, extra_headers=extra_headers,
|
||
api_mode=api_mode, route_info=route_info, async_mode=async_mode,
|
||
)
|
||
candidate_kwargs = dict(
|
||
task=task, messages=messages, temperature=temperature, max_tokens=max_tokens,
|
||
tools=tools, effective_timeout=req.effective_timeout,
|
||
effective_extra_body=req.effective_extra_body, reasoning_config=reasoning_config,
|
||
)
|
||
retry_kwargs = dict(
|
||
candidate_kwargs, resolved_base_url=req.resolved_base_url,
|
||
resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
|
||
main_runtime=main_runtime, final_model=req.final_model, extra_headers=extra_headers,
|
||
)
|
||
return req, retry_kwargs, candidate_kwargs
|
||
|
||
|
||
def _should_retry_same_provider(task: Optional[str], exc: Exception, tag: str) -> bool:
|
||
"""True when ``exc`` is a transient transport blip worth a same-provider retry; critical-path
|
||
tasks skip it on a full-budget timeout (``_should_skip_same_provider_retry``) and go straight
|
||
to fallback."""
|
||
if not _is_transient_transport_error(exc):
|
||
return False
|
||
if _should_skip_same_provider_retry(task, exc):
|
||
logger.info("Auxiliary %s%s: timeout on the critical path; "
|
||
"skipping same-provider retry and falling back: %s", task, tag, exc)
|
||
return False
|
||
return True
|
||
|
||
|
||
def _ladder_step_call(
|
||
step: _LadderStep, req: _PreparedAuxRequest, retry_kwargs: Dict[str, Any], candidate_kwargs: Dict[str, Any],
|
||
) -> Tuple[str, tuple, Dict[str, Any]]:
|
||
"""Resolve a ladder step into ``(kind, args, kwargs)`` for the sync/async performer."""
|
||
if step.kind == "call":
|
||
return "call", step.args, dict(provider=req.resolved_provider, api_mode=req.resolved_api_mode)
|
||
if step.kind == "retry_same_provider":
|
||
retry_provider, retry_model = step.args
|
||
return "retry", (), dict(retry_kwargs, resolved_provider=retry_provider, resolved_model=retry_model)
|
||
return "fallback", step.args, candidate_kwargs
|
||
|
||
|
||
def _start_recovery_ladder(
|
||
first_err: Exception, req: _PreparedAuxRequest, retry_kwargs: Dict[str, Any], *,
|
||
task: Optional[str], async_mode: bool, route_info: Optional[Dict[str, str]],
|
||
):
|
||
"""Build the recovery-ladder generator for a failed primary request."""
|
||
return _aux_recovery_ladder(
|
||
first_err, client=req.client, kwargs=req.kwargs, task=task, async_mode=async_mode,
|
||
base_info=req.base_info, resolved_provider=req.resolved_provider,
|
||
resolved_model=req.resolved_model, resolved_base_url=req.resolved_base_url,
|
||
resolved_api_key=req.resolved_api_key, resolved_api_mode=req.resolved_api_mode,
|
||
final_model=req.final_model, max_tokens=retry_kwargs["max_tokens"],
|
||
main_runtime=retry_kwargs["main_runtime"], route_info=route_info)
|
||
|
||
|
||
def _call_llm_impl(
|
||
task: str = None, *, provider: str = None, model: str = None, base_url: str = None,
|
||
api_key: str = None, main_runtime: Optional[Dict[str, Any]] = None, messages: list,
|
||
temperature: Optional[float] = None, max_tokens: int = None, tools: list = None,
|
||
timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None,
|
||
extra_headers: Optional[Dict[str, str]] = None, api_mode: str = None, stream: bool = False,
|
||
stream_options: dict = None, route_info: Optional[Dict[str, str]] = None,
|
||
) -> Any:
|
||
"""Centralized synchronous LLM call: resolve provider/model, auth, kwargs, fallbacks.
|
||
task: aux task whose provider:model comes from config (ignored if provider set); api_mode
|
||
overrides task config; timeout=None reads auxiliary.{task}.timeout; extra_headers override
|
||
client defaults. stream=True returns the raw SDK stream (caller consumes/falls back)
|
||
instead of a validated response. RuntimeError if no provider is configured."""
|
||
req, retry_kwargs, candidate_kwargs = _plan_aux_call(
|
||
task, async_mode=False, provider=provider, model=model, base_url=base_url,
|
||
api_key=api_key, main_runtime=main_runtime, messages=messages,
|
||
temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
|
||
extra_body=extra_body, reasoning_config=reasoning_config,
|
||
extra_headers=extra_headers, api_mode=api_mode, route_info=route_info,
|
||
)
|
||
client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
|
||
# Streaming path (MoA aggregator): return the raw SDK stream, skipping validation and
|
||
# the fallback chain (they assume a complete response); the caller owns reassembly/fallback.
|
||
if stream:
|
||
kwargs["stream"] = True
|
||
if stream_options:
|
||
kwargs["stream_options"] = stream_options
|
||
if task == "moa_aggregator" and isinstance(client, CodexAuxiliaryClient):
|
||
# Responses-shim clients consume the stream internally and return a completed
|
||
# object Relay's managed stream would iterate; the MoA facade wraps it as one chunk.
|
||
return client.chat.completions.create(**kwargs)
|
||
return _relay_sync_stream(client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode)
|
||
|
||
def _primary(**validate_kw: Any) -> Any:
|
||
# Retry on the same provider for a transient transport blip (connection reset / streaming-close /
|
||
# incomplete chunked read / 5xx / 408) before the except-chain below escalates to provider/model
|
||
# fallback. A dropped connection shouldn't abandon an otherwise-healthy provider — this especially
|
||
# matters for pinned auxiliary calls like MoA reference advisors, where "fallback to another
|
||
# provider" is not a meaningful recovery (the advisor is a specific model), so a transient blip that
|
||
# isn't retried simply loses that advisor for the turn (root of the run2 double-advisor "Connection
|
||
# error" collapse — a genuine upstream blip hitting both parallel advisors at once). Attempts are
|
||
# bounded and use exponential backoff. Count is configurable via auxiliary.transient_retries
|
||
# (default 2 retries → 3 total attempts); a second/third failure or any non-transient error falls
|
||
# through to ``first_err`` and the existing fallback handling unchanged. Unified home for the
|
||
# transient retry every auxiliary task shares. (PR #16587)
|
||
return _validate_llm_response(
|
||
_relay_sync_completion(
|
||
client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
|
||
create=lambda request: _create_with_progress(
|
||
client, request, task,
|
||
force_stream=_provider_requires_stream(
|
||
request_provider, req.base_info or req.resolved_base_url),
|
||
),
|
||
),
|
||
task, **validate_kw,
|
||
)
|
||
try:
|
||
# Bounded same-provider retry (exponential backoff, auxiliary.transient_retries) for
|
||
# transient blips before escalating to fallback — a dropped connection shouldn't
|
||
# abandon a healthy provider (matters for pinned MoA advisors).
|
||
try:
|
||
return _primary(provider=request_provider, base_url=req.base_info)
|
||
except Exception as transient_err:
|
||
if not _should_retry_same_provider(task, transient_err, ""):
|
||
raise
|
||
_max_transient_retries = _transient_retry_count()
|
||
_last_transient = transient_err
|
||
for _attempt in range(1, _max_transient_retries + 1):
|
||
_backoff = min(_TRANSIENT_RETRY_BACKOFF_BASE * (2.0 ** (_attempt - 1)), 8.0)
|
||
logger.info("Auxiliary %s: transient transport error (attempt %d/%d); "
|
||
"retrying same provider after %.1fs before fallback: %s",
|
||
task or "call", _attempt, _max_transient_retries, _backoff, _last_transient)
|
||
time.sleep(_backoff)
|
||
try:
|
||
return _primary()
|
||
except Exception as retry_transient:
|
||
if not _is_transient_transport_error(retry_transient):
|
||
raise
|
||
_last_transient = retry_transient
|
||
raise _last_transient
|
||
except Exception as first_err:
|
||
def _perform(step: _LadderStep) -> Any:
|
||
kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
|
||
if kind == "call":
|
||
return _validate_llm_response(_relay_sync_completion(*args, **kw), task)
|
||
if kind == "retry":
|
||
return _retry_same_provider_sync(**kw)
|
||
return _call_fallback_candidate_sync(*args, **kw)
|
||
return _drive_ladder(
|
||
_start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=False, route_info=route_info),
|
||
_perform)
|
||
|
||
|
||
def _coerce_llm_message(response):
|
||
"""Pull a message (dict, object, or str) out of a response-or-message value: dict-shaped
|
||
responses/bare messages (compression, proxies) and ChatCompletion objects; MagicMock
|
||
``reasoning_*`` attrs are deliberately not strings."""
|
||
if response is None or isinstance(response, str):
|
||
return response
|
||
if isinstance(response, dict):
|
||
if "choices" not in response:
|
||
return response
|
||
choices = response.get("choices") or []
|
||
else:
|
||
choices = getattr(response, "choices", None)
|
||
if not choices:
|
||
return response
|
||
return _message_field(choices[0], "message") if choices else None
|
||
|
||
|
||
def _message_field(msg, name):
|
||
return msg.get(name) if isinstance(msg, dict) else getattr(msg, name, None)
|
||
|
||
|
||
def extract_content_or_reasoning(response, *, max_reasoning_chars: int | None = None) -> str:
|
||
"""Extract content from an LLM response, falling back to reasoning fields.
|
||
Order: ``content`` (inline think blocks stripped) → ``reasoning``/``reasoning_content`` →
|
||
``reasoning_details`` (OpenRouter array). Accepts a response or bare message;
|
||
``max_reasoning_chars`` bounds a reasoning fallback so unbounded chain-of-thought can't
|
||
become the compaction summary. Returns ``""`` if nothing found."""
|
||
msg = _coerce_llm_message(response)
|
||
if msg is None:
|
||
return ""
|
||
if isinstance(msg, str):
|
||
return msg.strip()
|
||
raw = _message_field(msg, "content")
|
||
if not isinstance(raw, str):
|
||
raw = str(raw) if raw else ""
|
||
content = raw.strip()
|
||
if content:
|
||
# Same precompiled closed-pair patterns as strip_think_blocks.
|
||
from agent.agent_runtime_helpers import _REASONING_BLOCK_PATTERNS
|
||
cleaned = content
|
||
for pattern in _REASONING_BLOCK_PATTERNS:
|
||
cleaned = pattern.sub("", cleaned)
|
||
cleaned = cleaned.strip()
|
||
if cleaned:
|
||
return cleaned
|
||
# Content is empty or reasoning-only — try structured reasoning fields
|
||
reasoning_parts: list[str] = []
|
||
for field in ("reasoning", "reasoning_content"):
|
||
val = _message_field(msg, field)
|
||
if val and isinstance(val, str) and val.strip() and val not in reasoning_parts:
|
||
reasoning_parts.append(val.strip())
|
||
details = _message_field(msg, "reasoning_details")
|
||
if details and isinstance(details, list):
|
||
for detail in details:
|
||
if isinstance(detail, dict):
|
||
summary = detail.get("summary") or detail.get("content") or detail.get("text")
|
||
if summary and summary not in reasoning_parts:
|
||
reasoning_parts.append(summary.strip() if isinstance(summary, str) else str(summary))
|
||
if not reasoning_parts:
|
||
return ""
|
||
text = "\n\n".join(reasoning_parts)
|
||
if max_reasoning_chars is not None and len(text) > max_reasoning_chars:
|
||
logger.warning("fell back to reasoning fields (%d chars); truncating to %d",
|
||
len(text), max_reasoning_chars)
|
||
return text[:max_reasoning_chars]
|
||
return text
|
||
|
||
|
||
@_relay_auxiliary_call_async
|
||
async def async_call_llm(
|
||
task: str = None, *, provider: str = None, model: str = None, base_url: str = None,
|
||
api_key: str = None, main_runtime: Optional[Dict[str, Any]] = None, messages: list,
|
||
temperature: Optional[float] = None, max_tokens: int = None, tools: list = None,
|
||
timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None,
|
||
route_info: Optional[Dict[str, str]] = None,
|
||
) -> Any:
|
||
"""Run an asynchronous auxiliary LLM request under the configured limit."""
|
||
semaphore = _acquire_async_aux_semaphore(task)
|
||
if semaphore is not None:
|
||
await semaphore.acquire()
|
||
try:
|
||
with scoped_runtime_main(main_runtime):
|
||
return await _async_call_llm_impl(
|
||
task=task, provider=provider, model=model, base_url=base_url, api_key=api_key,
|
||
main_runtime=main_runtime, messages=messages, temperature=temperature,
|
||
max_tokens=max_tokens, tools=tools, timeout=timeout, extra_body=extra_body,
|
||
reasoning_config=reasoning_config, route_info=route_info,
|
||
)
|
||
finally:
|
||
if semaphore is not None:
|
||
semaphore.release()
|
||
|
||
|
||
async def _async_call_llm_impl(
|
||
task: str = None, *, provider: str = None, model: str = None, base_url: str = None,
|
||
api_key: str = None, main_runtime: Optional[Dict[str, Any]] = None, messages: list,
|
||
temperature: Optional[float] = None, max_tokens: int = None, tools: list = None,
|
||
timeout: float = None, extra_body: dict = None, reasoning_config: Optional[dict] = None,
|
||
route_info: Optional[Dict[str, str]] = None,
|
||
) -> Any:
|
||
"""Centralized asynchronous LLM call; see call_llm() for full documentation.
|
||
No per-request header / api_mode override on the async entry point."""
|
||
req, retry_kwargs, candidate_kwargs = _plan_aux_call(
|
||
task, async_mode=True, provider=provider, model=model, base_url=base_url,
|
||
api_key=api_key, main_runtime=main_runtime, messages=messages,
|
||
temperature=temperature, max_tokens=max_tokens, tools=tools, timeout=timeout,
|
||
extra_body=extra_body, reasoning_config=reasoning_config,
|
||
extra_headers=None, api_mode=None, route_info=route_info,
|
||
)
|
||
client, kwargs, request_provider = req.client, req.kwargs, req.request_provider
|
||
try:
|
||
# Retry ONCE on the same provider for a transient blip before fallback (see call_llm()).
|
||
# (PR #16587)
|
||
_force_stream_async = _provider_requires_stream(request_provider, req.base_info or req.resolved_base_url)
|
||
|
||
async def _acreate(_kwargs: Dict[str, Any]) -> Any:
|
||
return await _acreate_with_progress(client, _kwargs, task, force_stream=_force_stream_async)
|
||
|
||
async def _primary(**validate_kw: Any) -> Any:
|
||
return _validate_llm_response(
|
||
await _relay_async_completion(
|
||
client, kwargs, provider=request_provider, api_mode=req.resolved_api_mode,
|
||
create=_acreate),
|
||
task, **validate_kw)
|
||
try:
|
||
return await _primary(provider=request_provider, base_url=req.base_info)
|
||
except Exception as transient_err:
|
||
# The async Codex adapter wraps the sync stream via to_thread: same TimeoutError here.
|
||
if not _should_retry_same_provider(task, transient_err, " (async)"):
|
||
raise
|
||
logger.info("Auxiliary %s (async): transient transport error; retrying "
|
||
"once on the same provider before fallback: %s", task or "call", transient_err)
|
||
return await _primary()
|
||
except Exception as first_err:
|
||
async def _perform(step: _LadderStep) -> Any:
|
||
kind, args, kw = _ladder_step_call(step, req, retry_kwargs, candidate_kwargs)
|
||
if kind == "call":
|
||
return _validate_llm_response(await _relay_async_completion(*args, **kw), task)
|
||
if kind == "retry":
|
||
return await _retry_same_provider_async(**kw)
|
||
fb_client, fb_model, fb_label = args
|
||
fb_client, _ = _to_async_client(fb_client, fb_model or "", is_vision=(task == "vision"))
|
||
return await _call_fallback_candidate_async(fb_client, fb_model, fb_label, **kw)
|
||
return await _drive_ladder_async(
|
||
_start_recovery_ladder(first_err, req, retry_kwargs, task=task, async_mode=True, route_info=route_info),
|
||
_perform)
|
||
|
||
|
||
# ---- BEGIN PLUGIN-COMPAT (revert-scheduled; see COMPAT_MANIFEST.md) ----
|
||
# Names external plugins imported from this module before the Sep 2026 decomposition.
|
||
# Internal code MUST NOT use these (scripts/check_compat_pointers.py fails CI if it does).
|
||
# The whole block is removed by reverting the commit that added it.
|
||
from pathlib import Path # noqa: F401,E402
|
||
import copy # noqa: F401,E402
|
||
|
||
NOUS_EXTRA_BODY = _nous_extra_body()
|
||
|
||
def get_async_text_auxiliary_client(task: str = "", *, main_runtime: Optional[Dict[str, Any]] = None):
|
||
"""Return (async_client, model_slug) for async consumers.
|
||
|
||
For standard providers returns (AsyncOpenAI, model). For Codex returns
|
||
(AsyncCodexAuxiliaryClient, model) which wraps the Responses API.
|
||
Returns (None, None) when no provider is available.
|
||
"""
|
||
provider, model, base_url, api_key, api_mode = _resolve_task_provider_model(task or None)
|
||
return resolve_provider_client(
|
||
provider,
|
||
model=model,
|
||
async_mode=True,
|
||
explicit_base_url=base_url,
|
||
explicit_api_key=api_key,
|
||
api_mode=api_mode,
|
||
main_runtime=main_runtime,
|
||
)
|
||
# ---- END PLUGIN-COMPAT ----
|