From 31e41eed347f62de312e59ccc822b30a635d4aba Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:48:58 -0700 Subject: [PATCH] fix(tests): runtime_provider no longer permanently captures a mocked load_config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test-pollution class: runtime_provider is usually imported lazily (inside switch_model's resolution path), so its first import in a pytest worker can happen while a test has hermes_cli.config.load_config patched. The module-level from-import then bound the MagicMock permanently — after the patch exited, every later caller in the process silently read the dead test's config. Live victim: MoA aggregator context-length resolution (resolve_runtime_provider -> AuthError 'Unknown provider custom:example'), making TestMoAContextLength::test_moa_custom_context_configures_compressor_threshold fail whenever it shared a process with TestLocalOllamaModelDiscovery::test_switch_model_on_current_ollama_custom_endpoint_keeps_base_url. Fix: load_config / get_compatible_custom_providers / normalize_extra_headers become late-bound delegates resolving hermes_cli.config attributes at call time. Both patch targets (config.load_config and runtime_provider.load_config) keep working. Regression tests pin the late-binding property and fail if the delegates revert to from-imports (sabotage-verified). --- hermes_cli/runtime_provider.py | 32 ++++++++-- .../test_runtime_provider_late_binding.py | 63 +++++++++++++++++++ 2 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 tests/hermes_cli/test_runtime_provider_late_binding.py diff --git a/hermes_cli/runtime_provider.py b/hermes_cli/runtime_provider.py index cff75aff13..582f5f6fcd 100644 --- a/hermes_cli/runtime_provider.py +++ b/hermes_cli/runtime_provider.py @@ -40,14 +40,36 @@ from hermes_cli.auth import ( is_actual_local_base_url, normalize_actual_base_url, ) -from hermes_cli.config import ( - get_compatible_custom_providers, - load_config, - normalize_extra_headers, -) +from hermes_cli import config as _config_mod from hermes_cli.providers import custom_provider_aliases, custom_provider_slug from hermes_constants import OPENROUTER_BASE_URL from hermes_cli.providers import is_official_openai_host + + +def load_config(): + """Late-bound delegate to :func:`hermes_cli.config.load_config`. + + Deliberately NOT a module-level ``from hermes_cli.config import + load_config``: this module is often imported lazily (inside functions), + so its first import can happen while a test has + ``hermes_cli.config.load_config`` patched — a from-import would then + bind the MagicMock *permanently*, poisoning every later caller in the + process (the mock's fixed config shadows the real one long after the + patch exits). Delegating at call time keeps both patch targets working: + patching ``hermes_cli.config.load_config`` OR + ``hermes_cli.runtime_provider.load_config`` behaves as expected. + """ + return _config_mod.load_config() + + +def get_compatible_custom_providers(config=None): + """Late-bound delegate — see :func:`load_config` for why.""" + return _config_mod.get_compatible_custom_providers(config) + + +def normalize_extra_headers(value): + """Late-bound delegate — see :func:`load_config` for why.""" + return _config_mod.normalize_extra_headers(value) from utils import base_url_host_matches, base_url_hostname, env_int diff --git a/tests/hermes_cli/test_runtime_provider_late_binding.py b/tests/hermes_cli/test_runtime_provider_late_binding.py new file mode 100644 index 0000000000..1212a34ca3 --- /dev/null +++ b/tests/hermes_cli/test_runtime_provider_late_binding.py @@ -0,0 +1,63 @@ +"""Regression: runtime_provider's config helpers must be late-bound. + +Test-pollution class found Aug 2026: ``hermes_cli.runtime_provider`` is +frequently imported lazily (inside functions like ``switch_model``'s +resolution path), so its first import in a process can happen while a test +has ``hermes_cli.config.load_config`` patched. With a module-level +from-import, that bound the MagicMock into ``runtime_provider.load_config`` +for the life of the process: after the patch exited, every later caller +(e.g. MoA aggregator context-length resolution reading ``providers:``) +silently got the long-dead test's config. Live pair that exposed it: + + tests/hermes_cli/test_models.py::TestLocalOllamaModelDiscovery:: + test_switch_model_on_current_ollama_custom_endpoint_keeps_base_url + tests/agent/test_model_metadata.py::TestMoAContextLength:: + test_moa_custom_context_configures_compressor_threshold + +The fix: runtime_provider exposes late-bound delegates that resolve +``hermes_cli.config`` attributes at call time. These tests pin exactly +that property — a patch on ``hermes_cli.config.`` must be visible +through ``runtime_provider.`` while active, and must fully release +when it exits (no permanent capture). They fail if anyone reverts the +delegates back to module-level from-imports. + +NOTE: deliberately no ``sys.modules`` pop/re-import here — a fresh +re-import under an active patch is itself a polluter (duplicate module +objects with diverging state). +""" + +from unittest.mock import patch + + +def test_load_config_patch_is_visible_and_releases(): + import hermes_cli.runtime_provider as rp + + sentinel = {"providers": {"only-the-mock-has-this": {"api": "http://x/v1"}}} + with patch("hermes_cli.config.load_config", return_value=sentinel): + # While the patch is active, the delegate must see the mock… + assert rp.load_config() is sentinel + + # …and once it exits, the same module must resolve the real function + # again instead of replaying the mock's config (the permanent-capture + # failure mode of a from-import binding). + result = rp.load_config() + assert result is not sentinel + assert "only-the-mock-has-this" not in (result.get("providers") or {}) + + +def test_sibling_delegates_are_late_bound_too(): + """get_compatible_custom_providers / normalize_extra_headers share the + same import shape and must share the same late-binding behavior.""" + import hermes_cli.runtime_provider as rp + + marker = [{"name": "marker", "base_url": "http://m/v1"}] + with patch( + "hermes_cli.config.get_compatible_custom_providers", return_value=marker + ): + assert rp.get_compatible_custom_providers({}) is marker + assert rp.get_compatible_custom_providers({}) is not marker + + hdrs = {"X-Marker": "1"} + with patch("hermes_cli.config.normalize_extra_headers", return_value=hdrs): + assert rp.normalize_extra_headers(None) is hdrs + assert rp.normalize_extra_headers(None) is not hdrs