d09dacf66a
Change-detectors, tautologies, source-reading tests, redundant duplicates, mock-echo tests and dead/unrunnable tests. Per-test rationale in the lane ledger (category + reason for every removal).
232 lines
8.0 KiB
Python
232 lines
8.0 KiB
Python
"""Regression tests for custom_providers per-model context_length resolution.
|
|
|
|
Covers the fix for #15779 — mid-session /model switch to a named custom
|
|
provider must honor ``custom_providers[].models.<id>.context_length`` the
|
|
same way startup already does.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
from hermes_cli.config import (
|
|
get_custom_provider_context_length,
|
|
get_custom_provider_model_capability,
|
|
)
|
|
|
|
|
|
class TestGetCustomProviderContextLength:
|
|
|
|
def test_trailing_slash_insensitive(self):
|
|
custom = [
|
|
{
|
|
"base_url": "https://example.invalid/v1/",
|
|
"models": {"m": {"context_length": 500_000}},
|
|
}
|
|
]
|
|
# config has trailing slash, runtime doesn't — must match
|
|
assert (
|
|
get_custom_provider_context_length(
|
|
"m", "https://example.invalid/v1", custom
|
|
)
|
|
== 500_000
|
|
)
|
|
# and the reverse
|
|
custom2 = [
|
|
{
|
|
"base_url": "https://example.invalid/v1",
|
|
"models": {"m": {"context_length": 500_000}},
|
|
}
|
|
]
|
|
assert (
|
|
get_custom_provider_context_length(
|
|
"m", "https://example.invalid/v1/", custom2
|
|
)
|
|
== 500_000
|
|
)
|
|
|
|
|
|
def test_empty_inputs_return_none(self):
|
|
assert get_custom_provider_context_length("", "http://x", [{"base_url": "http://x", "models": {"": {"context_length": 1}}}]) is None
|
|
assert get_custom_provider_context_length("m", "", [{"base_url": "", "models": {"m": {"context_length": 1}}}]) is None
|
|
assert get_custom_provider_context_length("m", "http://x", None) is None
|
|
assert get_custom_provider_context_length("m", "http://x", []) is None
|
|
|
|
|
|
class TestGetCustomProviderModelCapability:
|
|
def test_matches_exact_model_on_normalized_route(self):
|
|
custom = [
|
|
{
|
|
"base_url": "https://example.invalid/anthropic/",
|
|
"models": {"fable": {"prompt_caching": True}},
|
|
}
|
|
]
|
|
|
|
assert get_custom_provider_model_capability(
|
|
"fable",
|
|
"https://example.invalid/anthropic",
|
|
"prompt_caching",
|
|
custom,
|
|
) is True
|
|
assert get_custom_provider_model_capability(
|
|
"opus",
|
|
"https://example.invalid/anthropic",
|
|
"prompt_caching",
|
|
custom,
|
|
) is None
|
|
|
|
def test_false_is_preserved_and_non_boolean_is_ignored(self):
|
|
custom = [
|
|
{
|
|
"base_url": "https://example.invalid/anthropic",
|
|
"models": {
|
|
"disabled": {"prompt_caching": False},
|
|
"invalid": {"prompt_caching": "true"},
|
|
},
|
|
}
|
|
]
|
|
|
|
assert get_custom_provider_model_capability(
|
|
"disabled",
|
|
"https://example.invalid/anthropic",
|
|
"prompt_caching",
|
|
custom,
|
|
) is False
|
|
assert get_custom_provider_model_capability(
|
|
"invalid",
|
|
"https://example.invalid/anthropic",
|
|
"prompt_caching",
|
|
custom,
|
|
) is None
|
|
|
|
def test_capability_is_route_isolated(self):
|
|
"""A declaration for one route must not apply to another route.
|
|
|
|
Guards normalize_route_base_url matching: if the URL comparison ever
|
|
regresses to a model-only (or hostname-only) shortcut, this pins the
|
|
failure.
|
|
"""
|
|
custom = [
|
|
{
|
|
"base_url": "https://other.example.invalid/anthropic",
|
|
"models": {"fable": {"prompt_caching": True}},
|
|
}
|
|
]
|
|
|
|
assert get_custom_provider_model_capability(
|
|
"fable",
|
|
"https://example.invalid/anthropic",
|
|
"prompt_caching",
|
|
custom,
|
|
) is None
|
|
|
|
|
|
|
|
class TestGetModelContextLengthHonorsOverride:
|
|
"""agent.model_metadata.get_model_context_length must honor the
|
|
custom_providers override at step 0b — before any probe, cache hit,
|
|
or models.dev lookup can override it.
|
|
"""
|
|
|
|
def _mock_all_probes(self):
|
|
"""Context manager that disables every downstream resolution step."""
|
|
from agent import model_metadata as _mm
|
|
return [
|
|
patch.object(_mm, "get_cached_context_length", return_value=None),
|
|
patch.object(_mm, "fetch_endpoint_model_metadata", return_value={}),
|
|
patch.object(_mm, "fetch_model_metadata", return_value={}),
|
|
patch.object(_mm, "is_local_endpoint", return_value=False),
|
|
patch.object(_mm, "_is_known_provider_base_url", return_value=False),
|
|
]
|
|
|
|
def test_custom_providers_override_wins_over_default_fallback(self):
|
|
from agent.model_metadata import get_model_context_length
|
|
custom = [
|
|
{
|
|
"base_url": "https://example.invalid/v1",
|
|
"models": {"gpt-5.5": {"context_length": 1_050_000}},
|
|
}
|
|
]
|
|
patches = self._mock_all_probes()
|
|
for p in patches:
|
|
p.start()
|
|
try:
|
|
ctx = get_model_context_length(
|
|
"gpt-5.5",
|
|
base_url="https://example.invalid/v1",
|
|
provider="custom",
|
|
custom_providers=custom,
|
|
)
|
|
finally:
|
|
for p in patches:
|
|
p.stop()
|
|
assert ctx == 1_050_000
|
|
|
|
def test_explicit_config_context_length_still_wins(self):
|
|
"""Top-level model.context_length (step 0) outranks custom_providers (step 0b).
|
|
|
|
Users who set both should see the top-level value — that's the
|
|
documented precedence and matches the long-standing step-0 behavior.
|
|
"""
|
|
from agent.model_metadata import get_model_context_length
|
|
custom = [
|
|
{
|
|
"base_url": "https://example.invalid/v1",
|
|
"models": {"m": {"context_length": 1_050_000}},
|
|
}
|
|
]
|
|
ctx = get_model_context_length(
|
|
"m",
|
|
base_url="https://example.invalid/v1",
|
|
provider="custom",
|
|
config_context_length=500_000, # explicit top-level wins
|
|
custom_providers=custom,
|
|
)
|
|
assert ctx == 500_000
|
|
|
|
def test_no_override_falls_through_to_default(self):
|
|
"""With custom_providers=None and all probes disabled, resolver
|
|
returns DEFAULT_FALLBACK_CONTEXT (256K after the stepdown bump).
|
|
"""
|
|
from agent.model_metadata import get_model_context_length, DEFAULT_FALLBACK_CONTEXT
|
|
patches = self._mock_all_probes()
|
|
for p in patches:
|
|
p.start()
|
|
try:
|
|
ctx = get_model_context_length(
|
|
"unknown-model",
|
|
base_url="https://example.invalid/v1",
|
|
provider="custom",
|
|
custom_providers=None,
|
|
)
|
|
finally:
|
|
for p in patches:
|
|
p.stop()
|
|
assert ctx == DEFAULT_FALLBACK_CONTEXT
|
|
|
|
|
|
|
|
|
|
def test_override_honored_when_caller_passes_no_custom_providers(tmp_path, monkeypatch):
|
|
"""Step 0c must not be gated on the caller having loaded the route list: aux fallback screening,
|
|
CLI/TUI context estimators and gateway /status pass ``custom_providers=None`` (#69807)."""
|
|
from pathlib import Path
|
|
|
|
base_url, model, override = "https://cp-ctx-selfresolve.invalid/v1", "router/auto", 999_999
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
|
(tmp_path / "config.yaml").write_text(
|
|
"custom_providers:\n"
|
|
" - name: test-route\n"
|
|
f" base_url: {base_url}\n"
|
|
" key_env: TEST_FAKE_CONTEXT_ENV\n"
|
|
f" model: {model}\n"
|
|
" api_mode: chat_completions\n"
|
|
" models:\n"
|
|
f" {model}:\n"
|
|
f" context_length: {override}\n",
|
|
encoding="utf-8",
|
|
)
|
|
from agent.model_metadata import get_model_context_length
|
|
|
|
assert get_model_context_length(model, base_url=base_url, api_key="", provider="custom") == override
|