Merge pull request #115831 from NousResearch/fix/boa-desktop-openai-codex-fallback

fix(desktop): open chats switch to a fallback provider added after they were opened (#95066, salvage #95139)

hermes_cli/cli_chat_turn_mixin.py

@@ -29,6 +29,21 @@ class CLIChatTurnMixin:
     # process exit code (see cli._run_single_query_mode) read this instead.
     _last_turn_result = None
 
+    def _sync_fallback_chain_with_config(self, agent) -> None:
+        """Adopt ``fallback_providers`` edits made while this chat is open (#95066) — the same
+        per-turn, fail-closed contract as the Desktop/TUI and messaging gateways: a torn config.yaml
+        keeps the last known-good chain instead of reading as "chain removed"."""
+        from cli import logger
+        try:
+            from gateway.run import GatewayRunner
+            from hermes_cli.config_effective import load_user_config_effective
+            from hermes_cli.fallback_config import get_fallback_chain
+            self._fallback_model = get_fallback_chain(load_user_config_effective(fail_closed=True))
+        except Exception as e:
+            logger.debug("fallback chain sync skipped (keeping current chain): %s", e)
+            return
+        GatewayRunner._apply_fallback_chain_to_agent(agent, self._fallback_model)
+
     def chat(self, message, images: list = None, voice_input: bool = False) -> Optional[str]:
         """Run one user turn; returns the agent's response, or None on error.
 
@@ -62,6 +77,7 @@ class CLIChatTurnMixin:
         agent = self.agent
         if agent is None:
             return None
+        self._sync_fallback_chain_with_config(agent)  # chain added after this chat opened reaches this turn
         message = self._chat_route_images(message, images)
 
         if isinstance(message, str) and not isinstance(message, TimelineNotification):

tests/hermes_cli/test_cli_fallback_chain_hot_reload.py

@@ -0,0 +1,56 @@
+"""An open classic-CLI chat adopts a ``fallback_providers`` chain added after it started (#95066).
+
+``HermesCLI`` holds one long-lived agent and read the chain once in ``__init__``; ``hermes fallback
+add`` from another terminal never reached the open chat. The turn loop (``HermesCLI.chat``) now
+re-reads the chain fail-closed, so a torn config.yaml keeps the last known-good chain.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+import cli
+from hermes_cli.config import get_config_path
+
+FALLBACK = [{"provider": "xai-oauth", "model": "grok-4.6"}]
+
+
+class _StopAfterSync(Exception):
+    pass
+
+
+def _chat_turn(monkeypatch, shell, config_text: str) -> None:
+    """Drive ``HermesCLI.chat`` past the fallback sync against ``config_text`` as the live config.yaml."""
+    path = get_config_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(config_text, encoding="utf-8")
+    monkeypatch.setattr(shell, "_ensure_runtime_credentials", lambda: True)
+    monkeypatch.setattr(shell, "_resolve_turn_agent_config",
+                        lambda message: {"signature": shell._active_agent_route_signature, "model": None, "runtime": None})
+    monkeypatch.setattr(shell, "_init_agent", lambda **kw: True)
+
+    def stop(message, images):
+        raise _StopAfterSync()
+
+    monkeypatch.setattr(shell, "_chat_route_images", stop)
+    with pytest.raises(_StopAfterSync):
+        shell.chat("hello")
+
+
+def test_chat_turn_adopts_chain_added_after_the_cli_opened_and_keeps_it_on_torn_config(monkeypatch):
+    shell = cli.HermesCLI(compact=True, max_turns=1)
+    shell.agent = SimpleNamespace(
+        _fallback_chain=[], _fallback_model=None, _fallback_index=0,
+        _fallback_activated=False, _rate_limited_until=0, _unavailable_fallback_keys=set(),
+    )
+
+    _chat_turn(monkeypatch, shell, "fallback_providers:\n  - provider: xai-oauth\n    model: grok-4.6\n")
+    assert shell.agent._fallback_chain == FALLBACK
+    assert shell.agent._fallback_model == FALLBACK[0]
+    assert shell._fallback_model == FALLBACK  # a rebuilt agent starts from the fresh chain too
+
+    # Torn mid-edit write: keep the last known-good chain rather than wiping it.
+    _chat_turn(monkeypatch, shell, "fallback_providers: [\n  - provider: {{{\n")
+    assert shell.agent._fallback_chain == FALLBACK

tests/tui_gateway/test_fallback_chain_hot_reload.py

@@ -0,0 +1,83 @@
+"""Desktop/TUI sessions must adopt a ``fallback_providers`` chain added after the chat was opened.
+
+Regression for #95066: ``_make_agent`` read the chain once, so a session born before ``hermes fallback
+add`` kept an empty ``_fallback_chain`` forever and a Codex ``usage_limit_reached`` 429 ended in a
+provider error instead of switching to the configured fallback.
+
+Both tests drive ``_prepare_turn_input`` (turn admission, the production entry) up to the sync's
+successor so the wiring — not just the helper — is pinned.
+"""
+
+from __future__ import annotations
+
+import time
+from types import SimpleNamespace
+
+import pytest
+
+from tui_gateway import server
+
+FALLBACK = [{"provider": "xai-oauth", "model": "grok-4.6"}]
+
+
+class _StopAfterSync(Exception):
+    pass
+
+
+def _session(chain=None):
+    agent = SimpleNamespace(
+        _fallback_chain=list(chain or []), _fallback_model=(chain or [None])[0], _fallback_index=0,
+        _fallback_activated=False, _rate_limited_until=0, _unavailable_fallback_keys=set(),
+    )
+    return {"agent": agent, "session_key": "session-95066"}, agent
+
+
+def _admit_turn(monkeypatch, tmp_path, session, config_text: str) -> None:
+    """Run turn admission against ``config_text`` as the live config.yaml; stop right after the sync."""
+    cfg_path = tmp_path / "config.yaml"
+    cfg_path.write_text(config_text, encoding="utf-8")
+    monkeypatch.setattr(server, "_active_config_path", lambda: cfg_path)
+    monkeypatch.setattr(server, "_profile_runtime_scope_tokens", lambda profile_home: None)
+    monkeypatch.setattr(server, "_set_session_context", lambda *a, **k: [])
+    monkeypatch.setattr(server, "_wire_callbacks", lambda sid: None)
+    for name in ("_apply_pending_model_switch", "_sync_agent_model_with_config", "_sync_agent_compression_with_config"):
+        monkeypatch.setattr(server, name, lambda sid, session: None)
+
+    def stop(sid, session):
+        raise _StopAfterSync()
+
+    monkeypatch.setattr(server, "_sync_bot_capabilities", stop)
+    st = server._TurnRun(agent=None, one_turn_restore=None, terminal_callback=None, receipt_committed=False)
+    with pytest.raises(_StopAfterSync):
+        server._prepare_turn_input("sid", session, st, "hello", [])
+
+
+def test_chain_added_after_open_reaches_the_live_agent_at_turn_admission(monkeypatch, tmp_path):
+    session, agent = _session()
+
+    _admit_turn(monkeypatch, tmp_path, session,
+                "fallback_providers:\n  - provider: xai-oauth\n    model: grok-4.6\n")
+
+    assert agent._fallback_chain == FALLBACK
+    assert agent._fallback_model == FALLBACK[0]
+    assert agent._fallback_index == 0
+
+
+def test_torn_config_keeps_the_last_known_good_chain_but_removal_still_applies(monkeypatch, tmp_path):
+    session, agent = _session(FALLBACK)
+
+    # Torn mid-edit write: an unparsable config.yaml must NOT read as "chain removed".
+    _admit_turn(monkeypatch, tmp_path, session, "fallback_providers: [\n  - provider: {{{\n")
+    assert agent._fallback_chain == FALLBACK
+    assert agent._fallback_model == FALLBACK[0]
+
+    # Control: a valid config with the chain removed clears it on the next turn.
+    _admit_turn(monkeypatch, tmp_path, session, "model:\n  provider: openai\n")
+    assert agent._fallback_chain == []
+    assert agent._fallback_model is None
+
+    # While a cooldown holds the agent on an activated fallback, the sync leaves the chain alone.
+    agent._fallback_chain, agent._fallback_activated = list(FALLBACK), True
+    agent._rate_limited_until = time.monotonic() + 600
+    _admit_turn(monkeypatch, tmp_path, session, "model:\n  provider: openai\n")
+    assert agent._fallback_chain == FALLBACK

tui_gateway/agent_callbacks.py

@@ -310,6 +310,30 @@ def _load_fallback_model():
     return get_fallback_chain(_load_cfg())
 
 
+def _sync_agent_fallback_with_config(sid: str, session: dict) -> None:
+    """Adopt ``fallback_providers`` edits into the cached agent at turn start.
+
+    Desktop/TUI chats keep one agent across turns, and ``_make_agent`` reads the chain once: a chat
+    opened before ``hermes fallback add`` kept an empty chain forever and a provider-quota 429 ended in
+    a provider error with a healthy fallback configured (#95066). Same per-turn contract the messaging
+    gateway applies to its cached agents (``GatewayRunner._refresh_fallback_model``): the config is
+    read fail-closed, so a torn/invalid config.yaml keeps the agent's last known-good chain instead of
+    ``_load_cfg()``'s fail-open ``{}`` reading as "chain removed" and wiping it. Never blocks the turn.
+    """
+    agent = session.get("agent")
+    if agent is None:
+        return
+    try:
+        from gateway.run import GatewayRunner
+        from hermes_cli.config_effective import load_user_config_effective
+        from hermes_cli.fallback_config import get_fallback_chain
+        chain = get_fallback_chain(load_user_config_effective(_active_config_path(), fail_closed=True))
+    except Exception as e:
+        logger.warning("fallback chain sync skipped for %s (keeping current chain): %s", sid, e)
+        return
+    GatewayRunner._apply_fallback_chain_to_agent(agent, chain)
+
+
 def _background_agent_kwargs(agent, task_id: str) -> dict:
     cfg = _load_cfg()
 

tui_gateway/prompt_turn.py

@@ -565,6 +565,7 @@ def _prepare_turn_input(sid: str, session: dict, st: _TurnRun, text: Any, images
         _apply_pending_model_switch(sid, session)
         _sync_agent_model_with_config(sid, session)
         _sync_agent_compression_with_config(sid, session)
+    _sync_agent_fallback_with_config(sid, session)  # chain added after the chat opened reaches this turn
     _sync_bot_capabilities(sid, session)  # Bot Chat: adopt Settings->Capabilities edits
     _adopt_out_of_band_turns(session)
     st.agent = agent = session["agent"]

website/docs/user-guide/features/fallback-providers.md

@@ -186,8 +186,9 @@ fallback_providers:
 
 | Context | Fallback Supported |
 |---------|-------------------|
-| CLI sessions (interactive and `hermes -z` one-shot) | ✔ (at startup when the primary's credentials/quota fail, and mid-session) |
+| CLI sessions (interactive and `hermes -z` one-shot) | ✔ (at startup when the primary's credentials/quota fail, mid-session, and a chain added or edited while a chat is open applies from its next turn) |
 | Messaging gateway (Telegram, Discord, etc.) | ✔ |
+| Desktop app / TUI chats | ✔ (a chain added or edited while a chat is open applies from its next turn) |
 | Subagent delegation | ✔ (`delegation.fallback_providers` when set; otherwise only unpinned children inherit the parent chain; `[]` disables) |
 | Cron jobs | ✔ (cron agents inherit configured fallback providers) |
 | Auxiliary tasks on `provider: auto` | ✔ (try per-task fallback, then the main fallback chain before built-in aux discovery) |