diff --git a/gateway/run_adapters.py b/gateway/run_adapters.py index 1d9dad2410..16e16169ab 100644 --- a/gateway/run_adapters.py +++ b/gateway/run_adapters.py @@ -828,9 +828,11 @@ class GatewayAdapterLifecycleMixin: return # Unregistered plugin: keep it queued and re-arm a failed load (capped) for the next tick. from hermes_cli.plugins import get_plugin_manager - if info.get("load_rearms", 0) < _MAX_PLUGIN_LOAD_REARMS and get_plugin_manager().rearm_failed_platform( - platform.value): - info["load_rearms"] = info.get("load_rearms", 0) + 1 + rearms = info.get("load_rearms", 0) + # Off the loop: the re-arm takes the manager's discovery lock, which a sweep or deferred load holds. + if rearms < _MAX_PLUGIN_LOAD_REARMS and await self._run_in_executor_with_context( + get_plugin_manager().rearm_failed_platform, platform.value): + info["load_rearms"] = rearms + 1 backoff = self._bump_reconnect_backoff( platform, info, attempt, "adapter_unavailable", _adapter_unavailable_message(platform), ) diff --git a/hermes_cli/plugins_loader.py b/hermes_cli/plugins_loader.py index d82bbe9b66..bb71365f4c 100644 --- a/hermes_cli/plugins_loader.py +++ b/hermes_cli/plugins_loader.py @@ -44,6 +44,7 @@ _BARE_MODULE_SCOPE: Dict[str, str] = {} # bare module name -> owning scope_key _LOAD_TIMEOUT_SECS = 10.0 _MAX_LOAD_TIMEOUT_SECS = 600.0 _MAX_ABANDONED_LOADERS = 8 +_LOADER_THREAD_PREFIX = "plugin-load:" # names each deadline-bounded load worker; the re-arm guard matches it _ABANDONED_LOADERS: List[threading.Thread] = [] _ABANDONED_LOADERS_LOCK = threading.Lock() # PluginContexts of the outermost deadline-bounded load; set only inside its worker. @@ -135,7 +136,7 @@ def run_with_load_deadline(plugin_key: str, ctx: "PluginContext", fn: Callable[[ _IN_PLUGIN_LOAD.reset(token) worker = threading.Thread( - target=contextvars.copy_context().run, args=(_worker,), name=f"plugin-load:{plugin_key}", daemon=True, + target=contextvars.copy_context().run, args=(_worker,), name=f"{_LOADER_THREAD_PREFIX}{plugin_key}", daemon=True, ) worker.start() worker.join(timeout) @@ -282,7 +283,7 @@ class PluginLoaderMixin: failed, _get_disabled_plugins(), _get_enabled_plugins()).action not in ("defer", "load"): return False with _ABANDONED_LOADERS_LOCK: # its hung import is still running: another retry only leaks a thread - if any(t.is_alive() and t.name == f"plugin-load:{manifest_key(failed)}" for t in _ABANDONED_LOADERS): + if any(t.is_alive() and t.name == f"{_LOADER_THREAD_PREFIX}{manifest_key(failed)}" for t in _ABANDONED_LOADERS): return False logger.info("Re-arming failed platform plugin load: %s", platform_name) self._register_deferred_platform(failed)