fix(agent): scope the provider-level answer_in_reasoning opt-in to the live entry
Gate finding (C11 r3, Low): the provider-level loop returned the first entry on the base_url carrying an answer_in_reasoning bool, whatever provider was live. Two providers: entries on one endpoint (LiteLLM/vLLM router) leaked one's opt-in to the other, and an earlier sibling's true overrode the live entry's explicit false (probe: vllm-nemo true + vllm-ds false on one URL, live custom:vllm-ds -> True). Match the live custom:<slug> via custom_provider_slug, the helper that derives that identity from an entry. With no slug match (bare 'custom'), trust only when every flagged sibling on the endpoint opts in.
This commit is contained in:
@@ -57,10 +57,17 @@ def answer_in_reasoning_capability(agent: Any) -> bool:
|
||||
return configured
|
||||
# The provider-level ``capabilities:`` block, re-read on the live route like the per-model key
|
||||
# so it holds on CLI/TUI (no constructor ``capabilities=``) and survives /model switches.
|
||||
for entry in _entries_for_route(base_url, custom_providers, None):
|
||||
configured = (entry.get("capabilities") or {}).get(_ANSWER_IN_REASONING_CAPABILITY)
|
||||
if isinstance(configured, bool):
|
||||
return configured
|
||||
# Several entries may share one endpoint (LiteLLM/vLLM router): only the live ``custom:<slug>``
|
||||
# entry counts; without one, every flagged sibling on the endpoint must opt in.
|
||||
from hermes_cli.providers import custom_provider_slug
|
||||
|
||||
entries = list(_entries_for_route(base_url, custom_providers, None))
|
||||
live = [e for e in entries
|
||||
if custom_provider_slug(str(e.get("name") or ""), str(e.get("provider_key") or "")) == provider]
|
||||
flags = {(e.get("capabilities") or {}).get(_ANSWER_IN_REASONING_CAPABILITY) for e in live or entries}
|
||||
flags &= {True, False}
|
||||
if flags:
|
||||
return flags == {True}
|
||||
|
||||
if _NEMOTRON_PARSER_MODEL_MARKER not in model.lower():
|
||||
return False
|
||||
|
||||
@@ -27,11 +27,13 @@ sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
|
||||
sys.modules.setdefault("fal_client", types.SimpleNamespace())
|
||||
|
||||
|
||||
# Provider-level ``capabilities:`` opt-ins: the fixture's own route and a second custom provider.
|
||||
# Provider-level ``capabilities:`` opt-ins: the fixture's own route and a second custom provider
|
||||
# (plus an un-opted sibling on that provider's endpoint).
|
||||
_OPTED_IN_PROVIDERS = [
|
||||
{"name": "local", "base_url": "https://example.invalid/v1", "capabilities": {"answer_in_reasoning": True}},
|
||||
{"name": "acme", "base_url": "https://llm.example.com/v1", "model": "acme/reasoner",
|
||||
"capabilities": {"answer_in_reasoning": True}},
|
||||
{"name": "acme-plain", "base_url": "https://llm.example.com/v1"}, # same endpoint, no opt-in
|
||||
]
|
||||
|
||||
|
||||
@@ -188,6 +190,7 @@ def test_length_cut_reasoning_is_not_promoted(tmp_path, monkeypatch):
|
||||
("custom", "https://llm.example.com/v1", "acme/reasoner", "private thoughts that must not be shown", 1),
|
||||
("custom", "https://llm.example.com/v1", "acme/after-model-switch", "private thoughts that must not be shown", 1),
|
||||
("custom", "https://fallback.example.com/v1", "acme/reasoner", "the visible answer", 2),
|
||||
("custom:acme-plain", "https://llm.example.com/v1", "acme/reasoner", "the visible answer", 2),
|
||||
])
|
||||
def test_reasoning_promotion_requires_a_trusted_route(tmp_path, monkeypatch, provider, base_url, model, final, calls):
|
||||
"""Private reasoning on an untrusted route retries to the visible answer and never
|
||||
|
||||
Reference in New Issue
Block a user