fix(agent): scope the provider-level answer_in_reasoning opt-in to the live entry

Gate finding (C11 r3, Low): the provider-level loop returned the first
entry on the base_url carrying an answer_in_reasoning bool, whatever
provider was live. Two providers: entries on one endpoint (LiteLLM/vLLM
router) leaked one's opt-in to the other, and an earlier sibling's true
overrode the live entry's explicit false (probe: vllm-nemo true + vllm-ds
false on one URL, live custom:vllm-ds -> True).

Match the live custom:<slug> via custom_provider_slug, the helper that
derives that identity from an entry. With no slug match (bare 'custom'),
trust only when every flagged sibling on the endpoint opts in.
This commit is contained in:
kshitijk4poor
2026-10-06 11:19:02 +05:30
committed by kshitij
parent b659f9a1d1
commit 14701564b4
2 changed files with 15 additions and 5 deletions

View File

@@ -57,10 +57,17 @@ def answer_in_reasoning_capability(agent: Any) -> bool:
return configured
# The provider-level ``capabilities:`` block, re-read on the live route like the per-model key
# so it holds on CLI/TUI (no constructor ``capabilities=``) and survives /model switches.
for entry in _entries_for_route(base_url, custom_providers, None):
configured = (entry.get("capabilities") or {}).get(_ANSWER_IN_REASONING_CAPABILITY)
if isinstance(configured, bool):
return configured
# Several entries may share one endpoint (LiteLLM/vLLM router): only the live ``custom:<slug>``
# entry counts; without one, every flagged sibling on the endpoint must opt in.
from hermes_cli.providers import custom_provider_slug
entries = list(_entries_for_route(base_url, custom_providers, None))
live = [e for e in entries
if custom_provider_slug(str(e.get("name") or ""), str(e.get("provider_key") or "")) == provider]
flags = {(e.get("capabilities") or {}).get(_ANSWER_IN_REASONING_CAPABILITY) for e in live or entries}
flags &= {True, False}
if flags:
return flags == {True}
if _NEMOTRON_PARSER_MODEL_MARKER not in model.lower():
return False

View File

@@ -27,11 +27,13 @@ sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
sys.modules.setdefault("fal_client", types.SimpleNamespace())
# Provider-level ``capabilities:`` opt-ins: the fixture's own route and a second custom provider.
# Provider-level ``capabilities:`` opt-ins: the fixture's own route and a second custom provider
# (plus an un-opted sibling on that provider's endpoint).
_OPTED_IN_PROVIDERS = [
{"name": "local", "base_url": "https://example.invalid/v1", "capabilities": {"answer_in_reasoning": True}},
{"name": "acme", "base_url": "https://llm.example.com/v1", "model": "acme/reasoner",
"capabilities": {"answer_in_reasoning": True}},
{"name": "acme-plain", "base_url": "https://llm.example.com/v1"}, # same endpoint, no opt-in
]
@@ -188,6 +190,7 @@ def test_length_cut_reasoning_is_not_promoted(tmp_path, monkeypatch):
("custom", "https://llm.example.com/v1", "acme/reasoner", "private thoughts that must not be shown", 1),
("custom", "https://llm.example.com/v1", "acme/after-model-switch", "private thoughts that must not be shown", 1),
("custom", "https://fallback.example.com/v1", "acme/reasoner", "the visible answer", 2),
("custom:acme-plain", "https://llm.example.com/v1", "acme/reasoner", "the visible answer", 2),
])
def test_reasoning_promotion_requires_a_trusted_route(tmp_path, monkeypatch, provider, base_url, model, final, calls):
"""Private reasoning on an untrusted route retries to the visible answer and never