From f6f6befbcfaa3d0811887b045ebcfdb6a849fff5 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:24:46 -0700 Subject: [PATCH] fix(fallback): a bare llama.cpp alias resolves only to the live managed server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the managed-endpoint rung for every caller that resolves a bare llamacpp / llama.cpp / llama-cpp alias outside the main ladder: - The rung moves from _resolve_custom_branch to the resolve_provider_client entry and to _get_cached_client, via agent/auxiliary_local_runtime.py. The `main_runtime is None` gate is gone: the cached auxiliary path always passes a normalized dict, so the rung never ran there, and a task pinned to llamacpp inherited a foreign main runtime's URL instead. - No running server now means (None, None) — the same answer the main ladder's _resolve_llamacpp_runtime gives — instead of continuing into _try_custom_endpoint / _resolve_api_key_provider, which posted the local GGUF slug (and, for compression, the conversation) to whichever cloud provider held a key (#119227) or to the primary's own endpoint. OPENAI_BASE_URL is not consulted either: the alias table entry (b7e0d715be0) exists for explicit base_url lanes, and the main ladder never reads it for a bare alias. - The cached client keys on the live endpoint (base_url + key), read at call time, so a server restarted on a new port/key gets a fresh client and a stopped one resolves to nothing instead of a stale client. - The endpoint is resolved with the calling profile's config (load_config() at call time), matching the main ladder; a configured provider entry named like the alias still wins. - Vision keeps the raw provider name for its final resolve so the alias is not flattened to bare `custom` first. - The aux "unavailable" message names the stopped local server instead of an invented LLAMACPP_API_KEY. Co-authored-by: Gille <4317663+helix4u@users.noreply.github.com> --- agent/auxiliary_client.py | 54 +++++++------------ agent/auxiliary_local_runtime.py | 35 ++++++++++++ agent/auxiliary_unavailable.py | 5 +- .../user-guide/features/fallback-providers.md | 2 + 4 files changed, 59 insertions(+), 37 deletions(-) create mode 100644 agent/auxiliary_local_runtime.py diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index aa7a65af77..20856e764f 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -5053,43 +5053,9 @@ def _resolve_xai_oauth_branch(req: _ResolveRequest) -> _ResolveResult: "OAuth token found (run: hermes model -> xAI Grok OAuth — SuperGrok / Premium+)") -_LLAMACPP_ALIASES = ("llamacpp", "llama.cpp", "llama-cpp") - - -def _managed_llamacpp_endpoint() -> Optional[Dict[str, Any]]: - """Live endpoint of the supervised local llama.cpp server, or None when it is not running. - - Best-effort: any failure (no state file, import error, server off) returns None so the caller - keeps its original fall-through instead of erroring an auxiliary/fallback call. - """ - try: - from hermes_cli.local_runtime.endpoint import resolve_llamacpp_endpoint - return resolve_llamacpp_endpoint(wait_for_boot_s=2.0) - except Exception: # noqa: BLE001 — endpoint resolution is best-effort - logger.debug("resolve_provider_client: managed llama.cpp endpoint unavailable", exc_info=True) - return None - - def _resolve_custom_branch(req: _ResolveRequest) -> _ResolveResult: """Custom endpoint (OPENAI_BASE_URL + OPENAI_API_KEY).""" provider, model, main_runtime = req.provider, req.model, req.main_runtime - # A llama.cpp alias with nothing explicit must resolve to the MANAGED local server — the same - # rung the main ladder takes (_resolve_llamacpp_runtime). Without it the fall-through below - # reached _try_custom_endpoint / _resolve_api_key_provider and handed back whichever cloud - # API-key provider happened to hold credentials, so the fallback path (try_activate_fallback - # resolves through THIS function) posted a local model slug to e.g. Gemini and got - # "unexpected model name format" (400) back, which then surfaced as a bogus - # "server rejected this request as too large" context rejection. Re-entering with the managed - # endpoint set keeps every downstream rule (alias /v1 tail, alias key scoping) in force, and - # the endpoint stays live: port and per-install key are re-read on every resolve. - if (req.original_provider in _LLAMACPP_ALIASES and not req.explicit_base_url - and req.main_runtime is None): - endpoint = _managed_llamacpp_endpoint() - if endpoint and str(endpoint.get("base_url") or "").strip(): - return _resolve_custom_branch(req._replace( - explicit_base_url=str(endpoint["base_url"]), - explicit_api_key=req.explicit_api_key or str(endpoint.get("api_key") or ""), - )) # wrap_base: base for the Anthropic-wrap decision. anthropic_messages must keep the raw # /anthropic base while the plain OpenAI client uses the /v1-rewritten custom_base (never # /anthropic/chat/completions). Empty means "use custom_base". @@ -5471,6 +5437,13 @@ def resolve_provider_client( if explicit_base_url and str(explicit_base_url).lower().startswith("moa://"): explicit_base_url = None explicit_api_key = None + from agent.auxiliary_local_runtime import bare_llamacpp_endpoint + local = bare_llamacpp_endpoint(original_provider, explicit_base_url, explicit_api_key) + if local is not None: + if not local[0]: + logger.warning("resolve_provider_client: %s requested but no local llama.cpp server is running", original_provider) + return None, None + explicit_base_url, explicit_api_key = local # Model for concrete providers: caller ``model`` → catalog default (empty for OAuth-gated providers whose # lists drift) → configured main model (MoA → aggregator), keeping OAuth aux tasks off the Step-2 fallback. # Excluded: ``auto`` (a stale main slug could pair with any picked provider) and Nous + vision (the @@ -5713,7 +5686,8 @@ def resolve_vision_provider_client( requested, resolved_model, resolved_base_url, resolved_api_key, resolved_api_mode = _resolve_task_provider_model( "vision", provider, model, base_url, api_key ) - requested = _normalize_vision_provider(requested) + # The raw name keeps a llama.cpp alias distinguishable from bare ``custom`` for the last resolve. + raw_requested, requested = requested, _normalize_vision_provider(requested) if resolved_base_url: provider_for_base_override = requested if requested and requested not in {"", "auto"} else "custom" client, final_model = resolve_provider_client( @@ -5738,7 +5712,7 @@ def resolve_vision_provider_client( return _finalize_vision_client(requested, client, final_model, resolved_model, async_mode) # Fallback: try without explicit base_url (old behavior) client, final_model = _get_cached_client( - requested, resolved_model, async_mode, api_mode=resolved_api_mode, main_runtime=runtime, is_vision=True, + raw_requested, resolved_model, async_mode, api_mode=resolved_api_mode, main_runtime=runtime, is_vision=True, ) return requested, client, (final_model if client is not None else None) @@ -6020,6 +5994,14 @@ def _get_cached_client( previously occurred in long-running gateways where recycled worker threads created unbounded entries (#10200). """ + # A bare llama.cpp alias keys on the live local endpoint: a restarted server (new port/key) + # must not be served the old client, and a stopped one resolves to nothing, not a cloud client. + from agent.auxiliary_local_runtime import bare_llamacpp_endpoint + local = bare_llamacpp_endpoint(provider, base_url, api_key) + if local is not None: + if not local[0]: + return None, None + base_url, api_key = local current_loop = _current_event_loop() if async_mode else None runtime = _normalize_main_runtime(main_runtime) cache_key = _client_cache_key( diff --git a/agent/auxiliary_local_runtime.py b/agent/auxiliary_local_runtime.py new file mode 100644 index 0000000000..6c5bf24208 --- /dev/null +++ b/agent/auxiliary_local_runtime.py @@ -0,0 +1,35 @@ +"""Managed llama.cpp endpoint for auxiliary and fallback client resolution. + +A bare ``llamacpp`` / ``llama.cpp`` / ``llama-cpp`` alias (no base_url, no configured provider of +that name) means "the local server Hermes manages for this profile" — exactly what the main ladder +resolves in ``hermes_cli.runtime_provider_custom._resolve_llamacpp_runtime``. With no server +running the answer is "unavailable", never the generic custom/API-key discovery: that handed the +local model slug to whatever cloud provider held a key (#119227) or to the primary's own endpoint. +""" + +from __future__ import annotations + +from typing import Any, Optional, Tuple + +from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES + + +def bare_llamacpp_endpoint(provider: Optional[str], base_url: Optional[str], + api_key: Any) -> Optional[Tuple[str, Any]]: + """``(base_url, api_key)`` of the profile's local llama.cpp server for a bare alias. + + ``base_url`` is ``""`` when nothing is serving. ``None`` when the request is not a bare alias: + an explicit base_url or a configured provider entry under the alias name resolves as before. + Read on every call so a restarted server's new port/key is picked up. + """ + alias = (provider or "").strip().lower() + if alias not in LLAMACPP_ALIASES or base_url: + return None + from hermes_cli.config import load_config + from hermes_cli.local_runtime.endpoint import resolve_llamacpp_endpoint + from hermes_cli.runtime_provider import _get_named_custom_provider + if _get_named_custom_provider(alias): + return None + endpoint = resolve_llamacpp_endpoint(config=load_config()) or {} + return (str(endpoint.get("base_url") or "").strip(), + api_key or endpoint.get("api_key") or "no-key-required") diff --git a/agent/auxiliary_unavailable.py b/agent/auxiliary_unavailable.py index b1c180d59e..639ace1c86 100644 --- a/agent/auxiliary_unavailable.py +++ b/agent/auxiliary_unavailable.py @@ -206,7 +206,10 @@ def missing_provider_credentials_message(provider_id: str) -> str: pconfig = PROVIDER_REGISTRY.get(provider_id) env_vars = tuple(getattr(pconfig, "api_key_env_vars", None) or ()) problem, remedy = "no API key was found", "" - if env_vars: + from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES + if provider_id in LLAMACPP_ALIASES: + problem, remedy = "no local llama.cpp server is running", "Start it in Settings → Providers → Local models" + elif env_vars: remedy = f"Set the {env_vars[0]} environment variable" elif pconfig is None: remedy = f"Set the {provider_id.upper().replace('-', '_')}_API_KEY environment variable" diff --git a/website/docs/user-guide/features/fallback-providers.md b/website/docs/user-guide/features/fallback-providers.md index 489f6a86be..32d698c020 100644 --- a/website/docs/user-guide/features/fallback-providers.md +++ b/website/docs/user-guide/features/fallback-providers.md @@ -103,6 +103,7 @@ OpenAI-compatible base URL continues to use the compatible client instead. | Tencent TokenPlan | `tencent-tokenplan` | `TOKENPLAN_API_KEY` | | Microsoft Foundry | `azure-foundry` | `AZURE_FOUNDRY_API_KEY` + `AZURE_FOUNDRY_BASE_URL` | | LM Studio (local) | `lmstudio` | `LM_API_KEY` (or none for local) + `LM_BASE_URL` | +| Local llama.cpp (managed) | `llamacpp` (aliases `llama.cpp`, `llama-cpp`) | The local model server Hermes manages (Desktop **Settings → Providers → Local models**), no `base_url` needed. If it is not running the entry is skipped — Hermes never sends the local model name to another provider | | Hugging Face | `huggingface` | `HF_TOKEN` | | Custom endpoint | `custom` | `base_url` + `key_env` (see below) | | Mixture of Agents preset | `moa` (`model` = preset name) | A configured MoA preset whose aggregator has credentials — the fallback runs the whole preset (references + aggregator), not the aggregator alone | @@ -319,6 +320,7 @@ These options apply to `auxiliary:`, `compression:`, and `fallback_providers:` e | `"codex"` | Force Codex OAuth | `hermes model` → ChatGPT or Codex Subscription | | `"main"` | Use whatever provider the main agent uses (auxiliary tasks only) | Active main provider configured | | `"anthropic"` | Force Anthropic native | `ANTHROPIC_API_KEY` or Claude Code credentials | +| `"llamacpp"` | Force the managed local llama.cpp server | Local model server running; when it is off, the local model name is never sent to another provider | ### Direct Endpoint Override