e63da95318
Two gaps from review of #113720's fix: 1. A profile logged in via auth.json (active_provider: nous) with no model.provider in config.yaml resolves as "auto". The ladder's OAuth rung swallowed the AuthError for "auto" and fell through to the keyless OpenRouter fallback, so the startup probe returned (False, None) and the first-run wizard ran anyway. The ladder now catches the AuthError in _ladder_rungs, still falls through for "auto", but stamps the swallowed error on a keyless fallback as `auth_error`; _probe_runtime_credentials returns it so the notice names the real failure. 2. The gate itself lived only in cli.py::_tui_print_startup and was untested at the seam (reverting cli.py left the suite green). It is now one mixin method, _maybe_offer_first_run_setup (tty check → probe → explain → offer), called from _tui_print_startup, and both tests drive that method with stdin.isatty patched True and _offer_first_run_setup asserting it is not called. Tests: the benched-credential test now covers the gate and the cooldown headline wording; the blank-install control is folded into the new auth.json-only test.
374 lines
14 KiB
Python
374 lines
14 KiB
Python
"""First-run onboarding routing for a completely unconfigured install.
|
|
|
|
Regression tests for the "keyless first run boots into a broken chat" bug:
|
|
a fresh install with zero providers accepted a message, spun for ~30s, then
|
|
failed with a provider-specific error ("Set OPENROUTER_API_KEY") the user
|
|
never chose, and never offered setup.
|
|
|
|
Covers:
|
|
- ``_runtime_credentials_ready()`` silent probe semantics
|
|
- ``_offer_first_run_setup()`` routing into the shared provider picker
|
|
- the provider-aware (non-OpenRouter-specific) empty-key error message
|
|
"""
|
|
|
|
import importlib
|
|
import os
|
|
import sys
|
|
import types
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.auth import AuthError
|
|
|
|
|
|
def _reset_modules(prefixes: tuple[str, ...]):
|
|
for name in list(sys.modules):
|
|
if any(name == p or name.startswith(p + ".") for p in prefixes):
|
|
sys.modules.pop(name, None)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_cli_and_tool_modules():
|
|
prefixes = ("tools", "cli", "run_agent")
|
|
original_modules = {
|
|
name: module
|
|
for name, module in sys.modules.items()
|
|
if any(name == p or name.startswith(p + ".") for p in prefixes)
|
|
}
|
|
try:
|
|
yield
|
|
finally:
|
|
_reset_modules(prefixes)
|
|
sys.modules.update(original_modules)
|
|
|
|
|
|
def _import_cli():
|
|
for name in list(sys.modules):
|
|
if name == "cli" or name == "run_agent" or name == "tools" or name.startswith("tools."):
|
|
sys.modules.pop(name, None)
|
|
if "firecrawl" not in sys.modules:
|
|
sys.modules["firecrawl"] = types.SimpleNamespace(Firecrawl=object)
|
|
return importlib.import_module("cli")
|
|
|
|
|
|
def _make_shell(cli, monkeypatch):
|
|
shell = cli.HermesCLI(compact=True, max_turns=1)
|
|
return shell
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _runtime_credentials_ready
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_credentials_ready_false_when_no_provider(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
def _raise(**kwargs):
|
|
raise AuthError("No inference provider configured.", code="no_provider_configured")
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell._runtime_credentials_ready() is False
|
|
|
|
|
|
def test_credentials_ready_false_on_empty_openrouter_key(monkeypatch):
|
|
"""The exact broken-chat state: provider resolves but api_key is empty."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "openrouter",
|
|
"api_key": "",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"source": "env/config",
|
|
},
|
|
)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell._runtime_credentials_ready() is False
|
|
|
|
|
|
def test_credentials_ready_true_with_key(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "openrouter",
|
|
"api_key": "sk-test",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"source": "env/config",
|
|
},
|
|
)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell._runtime_credentials_ready() is True
|
|
|
|
|
|
def test_credentials_ready_true_for_keyless_local_endpoint(monkeypatch):
|
|
"""ollama/llama.cpp-style custom endpoints need no key."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "custom",
|
|
"api_key": "",
|
|
"base_url": "http://localhost:11434/v1",
|
|
"source": "custom_provider",
|
|
},
|
|
)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell._runtime_credentials_ready() is True
|
|
|
|
|
|
def test_credentials_ready_true_for_callable_bearer_provider(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "azure-foundry",
|
|
"api_key": lambda: "tok",
|
|
"base_url": "https://foundry.example/v1",
|
|
"source": "entra",
|
|
},
|
|
)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell._runtime_credentials_ready() is True
|
|
|
|
|
|
def test_credentials_ready_never_prints(monkeypatch, capsys):
|
|
cli = _import_cli()
|
|
|
|
def _raise(**kwargs):
|
|
raise AuthError("No inference provider configured.", code="no_provider_configured")
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
capsys.readouterr() # drain construction output
|
|
shell._runtime_credentials_ready()
|
|
out = capsys.readouterr()
|
|
assert out.out == ""
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _offer_first_run_setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_offer_first_run_setup_routes_into_shared_picker(monkeypatch):
|
|
cli = _import_cli()
|
|
shell = _make_shell(cli, monkeypatch)
|
|
|
|
picker_calls = {"count": 0}
|
|
|
|
def _fake_picker():
|
|
picker_calls["count"] += 1
|
|
|
|
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _fake_picker)
|
|
monkeypatch.setattr("builtins.input", lambda *a, **k: "y")
|
|
# After the picker "runs", config has a provider and creds resolve.
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"model": {"provider": "nous", "default": "hermes-4-405b"}},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "nous",
|
|
"api_key": "portal-token",
|
|
"base_url": "https://inference-api.nousresearch.com/v1",
|
|
"source": "oauth",
|
|
},
|
|
)
|
|
|
|
assert shell._offer_first_run_setup() is True
|
|
assert picker_calls["count"] == 1
|
|
assert shell.requested_provider == "nous"
|
|
assert shell.model == "hermes-4-405b"
|
|
# Agent must be rebuilt with the new credentials on next use.
|
|
assert shell.agent is None
|
|
|
|
|
|
def test_offer_first_run_setup_re_resolves_reasoning_for_picked_model(monkeypatch):
|
|
"""The picker moves self.model; the CLI-level reasoning_config must follow it before the
|
|
lazily built agent inherits the launch model's effort."""
|
|
cli = _import_cli()
|
|
monkeypatch.setitem(cli.CLI_CONFIG, "agent", {
|
|
**cli.CLI_CONFIG.get("agent", {}), "reasoning_effort": "medium",
|
|
"reasoning_overrides": {"hermes-4-405b": "high"}})
|
|
shell = _make_shell(cli, monkeypatch)
|
|
assert shell.reasoning_config["effort"] == "medium"
|
|
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", lambda: None)
|
|
monkeypatch.setattr("builtins.input", lambda *a, **k: "y")
|
|
monkeypatch.setattr("hermes_cli.config.load_config",
|
|
lambda: {"model": {"provider": "nous", "default": "hermes-4-405b"}})
|
|
monkeypatch.setattr(shell, "_runtime_credentials_ready", lambda: True)
|
|
|
|
assert shell._offer_first_run_setup() is True
|
|
assert shell.model == "hermes-4-405b"
|
|
assert shell.reasoning_config["effort"] == "high"
|
|
|
|
|
|
def test_offer_first_run_setup_declined(monkeypatch):
|
|
cli = _import_cli()
|
|
shell = _make_shell(cli, monkeypatch)
|
|
|
|
def _fail_picker():
|
|
raise AssertionError("picker must not run when declined")
|
|
|
|
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _fail_picker)
|
|
monkeypatch.setattr("builtins.input", lambda *a, **k: "n")
|
|
assert shell._offer_first_run_setup() is False
|
|
|
|
|
|
def test_offer_first_run_setup_picker_cancel_is_graceful(monkeypatch):
|
|
cli = _import_cli()
|
|
shell = _make_shell(cli, monkeypatch)
|
|
|
|
def _cancel_picker():
|
|
raise KeyboardInterrupt()
|
|
|
|
monkeypatch.setattr("hermes_cli.main.select_provider_and_model", _cancel_picker)
|
|
monkeypatch.setattr("builtins.input", lambda *a, **k: "")
|
|
# Empty answer defaults to yes -> picker runs -> cancels -> False, no raise.
|
|
assert shell._offer_first_run_setup() is False
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Provider-aware empty-key error (replaces the OpenRouter-specific one)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_empty_key_error_names_actual_provider(monkeypatch, capsys):
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "fireworks",
|
|
"api_key": "",
|
|
"base_url": "https://api.fireworks.ai/inference/v1/extra",
|
|
"source": "env/config",
|
|
},
|
|
)
|
|
shell = _make_shell(cli, monkeypatch)
|
|
# A custom base_url would get the no-key placeholder; force the
|
|
# openrouter-shaped branch by pointing base_url at openrouter.
|
|
monkeypatch.setattr(
|
|
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
|
lambda **kw: {
|
|
"provider": "fireworks",
|
|
"api_key": "",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"source": "env/config",
|
|
},
|
|
)
|
|
capsys.readouterr()
|
|
assert shell._ensure_runtime_credentials() is False
|
|
out = capsys.readouterr().out
|
|
assert "fireworks" in out
|
|
assert "OPENROUTER_API_KEY" not in out
|
|
assert "hermes model" in out or "hermes setup" in out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configured-but-unusable credential: reason + cooldown, never the wizard (#113720)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _bench_nous_pool(monkeypatch, **entry_fields):
|
|
import time
|
|
from agent.credential_pool import STATUS_EXHAUSTED, CredentialPool, PooledCredential
|
|
|
|
benched = PooledCredential(id="e1", provider="nous", auth_type="oauth", access_token="x",
|
|
refresh_token="r", label="portal", source="manual:device_code",
|
|
priority=0, last_status=STATUS_EXHAUSTED, last_status_at=time.time() - 5,
|
|
**entry_fields)
|
|
pool = CredentialPool.__new__(CredentialPool)
|
|
monkeypatch.setattr(pool, "has_credentials", lambda: True, raising=False)
|
|
monkeypatch.setattr(pool, "has_available", lambda **kw: False, raising=False)
|
|
monkeypatch.setattr(pool, "next_available_at", lambda **kw: time.time() + 55, raising=False)
|
|
monkeypatch.setattr(pool, "entries", lambda: [benched], raising=False)
|
|
monkeypatch.setattr("agent.credential_pool.load_pool", lambda provider: pool)
|
|
|
|
|
|
def _forbid_wizard(monkeypatch, shell):
|
|
monkeypatch.setattr("hermes_cli.main.select_provider_and_model",
|
|
lambda: (_ for _ in ()).throw(AssertionError("wizard must not run")))
|
|
monkeypatch.setattr(shell, "_offer_first_run_setup",
|
|
lambda: (_ for _ in ()).throw(AssertionError("wizard must not be offered")))
|
|
|
|
|
|
def test_benched_credential_prints_cooldown_instead_of_wizard(monkeypatch, capsys):
|
|
"""A profile whose only credential is cooling down is not a blank install: the interactive
|
|
startup gate prints the cooldown (with why and how long) as the headline, without telling the
|
|
user to re-authenticate, and never offers the first-run wizard."""
|
|
cli = _import_cli()
|
|
shell = _make_shell(cli, monkeypatch)
|
|
shell.requested_provider = "nous"
|
|
|
|
def _raise(**kwargs):
|
|
raise AuthError("Hermes is not logged into Nous Portal.", provider="nous",
|
|
code="nous_auth_missing", relogin_required=True)
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _raise)
|
|
_bench_nous_pool(monkeypatch, last_error_code=429, last_error_reason="rate_limited")
|
|
_forbid_wizard(monkeypatch, shell)
|
|
monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
|
|
|
|
shell._maybe_offer_first_run_setup()
|
|
|
|
out = capsys.readouterr().out
|
|
assert "No inference provider is configured yet" not in out
|
|
headline = next(line for line in out.splitlines() if line.strip())
|
|
assert "cooling down after a rate-limit or quota response" in headline and "about 1m" in headline
|
|
assert "failed token refresh" not in out
|
|
assert "not logged into Nous Portal" in out
|
|
assert "re-authenticate" not in out and "hermes model" not in out
|
|
|
|
|
|
def test_auth_json_only_login_explains_instead_of_wizard(monkeypatch, capsys, tmp_path):
|
|
"""auth.json-only shape: logged into Nous but no ``model.provider`` (requested "auto"). The
|
|
ladder swallows the AuthError and falls through to a keyless OpenRouter fallback; the gate
|
|
must still explain the real failure rather than treat the profile as a blank install.
|
|
Control: the resolver's ``no_provider_configured`` still reaches the wizard."""
|
|
import dataclasses
|
|
|
|
import hermes_cli.runtime_provider as rp
|
|
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
(tmp_path / "config.yaml").write_text("model:\n default: some-model\n", encoding="utf-8")
|
|
for key in [k for k in os.environ if k.endswith("_API_KEY")]:
|
|
monkeypatch.delenv(key, raising=False)
|
|
|
|
def _nous_fail():
|
|
raise AuthError("Hermes is not logged into Nous Portal.", provider="nous",
|
|
code="nous_auth_missing", relogin_required=True)
|
|
|
|
monkeypatch.setattr(rp, "resolve_provider", lambda *a, **kw: "nous")
|
|
monkeypatch.setitem(rp._OAUTH_RUNTIME_PROVIDERS, "nous",
|
|
dataclasses.replace(rp._OAUTH_RUNTIME_PROVIDERS["nous"], resolve=_nous_fail))
|
|
|
|
cli = _import_cli()
|
|
shell = _make_shell(cli, monkeypatch)
|
|
shell.requested_provider = "auto"
|
|
shell._explicit_api_key = None
|
|
shell._explicit_base_url = None
|
|
_forbid_wizard(monkeypatch, shell)
|
|
monkeypatch.setattr(sys.stdin, "isatty", lambda: True)
|
|
|
|
shell._maybe_offer_first_run_setup()
|
|
out = capsys.readouterr().out
|
|
assert "not logged into Nous Portal" in out
|
|
assert "No inference provider is configured yet" not in out
|
|
|
|
offered = []
|
|
monkeypatch.setattr(shell, "_offer_first_run_setup", lambda: offered.append(True) or True)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", lambda **kw: (_ for _ in ()).throw(
|
|
AuthError("Hermes is not connected to any AI provider yet.", code="no_provider_configured")))
|
|
shell._maybe_offer_first_run_setup()
|
|
assert offered == [True]
|
|
assert "not logged into" not in capsys.readouterr().out
|