336b11584c
Every model reasons and intends that reasoning to be replayed, but the chat-completions wire echoed reasoning_content only to DeepSeek/Kimi/MiMo/ Ollama (plus a hardcoded Portal stealth model) and reasoning_details only to OpenRouter; turn_context popped `reasoning` everywhere, so Ollama's /v1 (reads only `reasoning`) never saw any. - message_sanitization owns one route->carrier table (_REASONING_CARRIER_ROUTES): strict schemas (Mistral, Groq, Cerebras, SambaNova) get nothing, Fireworks reasoning_content only, OpenRouter-format gateways (OpenRouter, Kilo, Vercel AI Gateway, Nous Portal) all three, Novita/MiniMax /v1/TokenHub + reasoning_details, local/Ollama/Upstage + reasoning, everyone else reasoning_content. - The " " pad stays limited to the must-echo tier (DeepSeek, Kimi, MiMo); Ollama leaves it (no echo requirement) and the stealth hardcode is gone. - Reactive recovery: a 4xx unknown-field rejection that NAMES a reasoning key this request carried is recorded per (provider, host, model) and the retry re-shapes api_messages without it. Generic upstream errors never count; a recorded key falls through, so no loop. History is untouched. - Nous Portal reads reasoning_details again: the #118182 "replay budget" does not exist in the Portal and does not reproduce live. - Gemini thought_signature survives on Gemini hosts/providers even when the model id is an alias. - Vendor keep-reasoning flags (docs-only): GLM thinking.clear_thinking=false, DashScope preserve_thinking / clear_thinking, Kimi K2.x thinking.keep=all. - stale_thinking_reaches_wire follows the same table, so compaction trigger and tail walks agree with what is sent.
269 lines
10 KiB
Python
269 lines
10 KiB
Python
"""Unit tests for the Z.AI / GLM provider profile's reasoning wiring.
|
|
|
|
Z.AI's GLM-4.5-and-later chat models default to thinking-mode ON when the
|
|
request omits ``thinking``. Before the profile emitted the parameter,
|
|
``reasoning_config = {"enabled": False}`` was a silent no-op on the direct
|
|
Z.AI route — users who turned thinking off kept burning thinking tokens on
|
|
every turn (the desktop "thinking reverts to medium" report).
|
|
|
|
GLM-5.2 additionally exposes a native ``reasoning_effort`` knob with two
|
|
enabled levels (high / max) on the OpenAI-compatible ``/api/paas/v4``
|
|
endpoint; the Hermes effort scale is collapsed onto those.
|
|
|
|
These tests pin the profile's wire-shape contract so Z.AI requests stay
|
|
correctly shaped without going live.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def zai_profile():
|
|
"""Resolve the registered Z.AI profile through the real discovery path."""
|
|
# ``model_tools`` triggers plugin discovery on import, which is what
|
|
# registers the Z.AI profile in the global provider registry.
|
|
import model_tools
|
|
import providers
|
|
|
|
profile = providers.get_provider_profile("zai")
|
|
assert profile is not None, "zai provider profile must be registered"
|
|
return profile
|
|
|
|
|
|
class TestZaiThinkingWireShape:
|
|
"""``build_api_kwargs_extras`` produces Z.AI's exact wire format."""
|
|
|
|
def test_no_preference_omits_thinking(self, zai_profile):
|
|
"""No reasoning_config → omit ``thinking`` so the server default
|
|
applies (matches prior behavior for users with no preference)."""
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config=None, model="glm-5"
|
|
)
|
|
assert extra_body == {}
|
|
assert top_level == {}
|
|
|
|
def test_enabled_sends_enabled_marker(self, zai_profile):
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "medium"}, model="glm-5"
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {}
|
|
|
|
def test_explicitly_disabled_sends_disabled_marker(self, zai_profile):
|
|
"""``reasoning_config.enabled=False`` → ``thinking.type=disabled``.
|
|
|
|
The crucial bit is that the parameter is *sent* at all — GLM defaults
|
|
to thinking-on when ``thinking`` is absent, so an unsent disable
|
|
burns thinking tokens forever.
|
|
"""
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": False}, model="glm-5"
|
|
)
|
|
assert extra_body == {"thinking": {"type": "disabled"}}
|
|
assert top_level == {}
|
|
|
|
|
|
class TestZaiGLM52ReasoningEffort:
|
|
"""GLM-5.2's native ``reasoning_effort`` knob (two enabled levels)."""
|
|
|
|
def test_high_maps_to_high(self, zai_profile):
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "high"},
|
|
model="glm-5.2",
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {"reasoning_effort": "high"}
|
|
|
|
@pytest.mark.parametrize("effort", ["low", "medium", "minimal"])
|
|
def test_lower_efforts_clamp_up_to_high(self, zai_profile, effort):
|
|
"""GLM-5.2's minimum thinking level is high — lower Hermes levels
|
|
clamp onto it."""
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": effort},
|
|
model="glm-5.2",
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {"reasoning_effort": "high"}
|
|
|
|
@pytest.mark.parametrize("effort", ["xhigh", "max"])
|
|
def test_strong_efforts_map_to_max(self, zai_profile, effort):
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": effort},
|
|
model="glm-5.2",
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {"reasoning_effort": "max"}
|
|
|
|
def test_disabled_sends_no_effort(self, zai_profile):
|
|
"""Disabled reasoning still sends the thinking-off marker but never
|
|
an effort level."""
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": False, "effort": "high"},
|
|
model="glm-5.2",
|
|
)
|
|
assert extra_body == {"thinking": {"type": "disabled"}}
|
|
assert top_level == {}
|
|
|
|
@pytest.mark.parametrize(
|
|
"model",
|
|
[
|
|
"z-ai/glm-5.2",
|
|
"glm-5-2",
|
|
"glm-5p2",
|
|
"accounts/fireworks/models/glm-5p2",
|
|
"zai-org-glm-5-2",
|
|
],
|
|
)
|
|
def test_alias_spellings_recognized(self, zai_profile, model):
|
|
_, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "max"},
|
|
model=model,
|
|
)
|
|
assert top_level == {"reasoning_effort": "max"}
|
|
|
|
@pytest.mark.parametrize(
|
|
"model",
|
|
["glm-5.1", "glm-5", "glm-4.7", "glm-4-9b", "", None],
|
|
)
|
|
def test_non_glm_5_2_models_get_no_effort(self, zai_profile, model):
|
|
_, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "high"},
|
|
model=model,
|
|
)
|
|
assert top_level == {}
|
|
|
|
|
|
class TestZaiGLM53ReasoningEffort:
|
|
"""GLM-5.3's graded low/medium/high/max effort scale (issue #91789).
|
|
|
|
Verified live on api.z.ai/api/coding/paas/v4: all four levels accepted
|
|
with monotonic reasoning-token scaling. Unlike 5.2, low and medium must
|
|
reach the wire instead of clamping up to high.
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
("effort", "expected"),
|
|
[
|
|
("low", "low"),
|
|
("medium", "medium"),
|
|
("high", "high"),
|
|
("max", "max"),
|
|
("xhigh", "max"),
|
|
("minimal", "low"),
|
|
],
|
|
)
|
|
def test_graded_efforts_pass_through(self, zai_profile, effort, expected):
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": effort},
|
|
model="glm-5.3",
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {"reasoning_effort": expected}
|
|
|
|
@pytest.mark.parametrize(
|
|
"model",
|
|
["z-ai/glm-5.3", "glm-5-3", "glm-5p3", "zai-org-glm-5-3"],
|
|
)
|
|
def test_alias_spellings_get_graded_scale(self, zai_profile, model):
|
|
_, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "low"},
|
|
model=model,
|
|
)
|
|
assert top_level == {"reasoning_effort": "low"}
|
|
|
|
def test_glm_5_2_still_clamps_low_to_high(self, zai_profile):
|
|
"""The 5.3 widening must not leak into 5.2's two-level wire."""
|
|
_, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": True, "effort": "low"},
|
|
model="glm-5.2",
|
|
)
|
|
assert top_level == {"reasoning_effort": "high"}
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"model",
|
|
[
|
|
"glm-5.3",
|
|
"glm-5.3-flash",
|
|
"z-ai/glm-5.3",
|
|
"z-ai/glm-5.3-flash",
|
|
"glm-5-3",
|
|
"glm-5p3",
|
|
],
|
|
)
|
|
def test_disabled_becomes_enabled_plus_low(self, zai_profile, model):
|
|
"""#85890 / #96373 — 5.3 rejects thinking.type=disabled (HTTP 400 / 1210)."""
|
|
extra_body, top_level = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": False},
|
|
model=model,
|
|
)
|
|
assert extra_body == {"thinking": {"type": "enabled", "clear_thinking": False}}
|
|
assert top_level == {"reasoning_effort": "low"}
|
|
|
|
|
|
|
|
class TestZaiModelGating:
|
|
"""GLM 4.5+ get thinking; earlier GLM models are left untouched."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"model",
|
|
[
|
|
"glm-4.5",
|
|
"glm-4.5-air",
|
|
"glm-4.5-flash",
|
|
"glm-4.6",
|
|
"glm-5",
|
|
"glm-5.2",
|
|
"GLM-5", # case-insensitive
|
|
],
|
|
)
|
|
def test_thinking_capable_models_emit_thinking(self, zai_profile, model):
|
|
extra_body, _ = zai_profile.build_api_kwargs_extras(
|
|
reasoning_config={"enabled": False}, model=model
|
|
)
|
|
assert extra_body == {"thinking": {"type": "disabled"}}
|
|
|
|
|
|
class TestZaiFullKwargsIntegration:
|
|
"""End-to-end: the transport's full kwargs carry the reasoning wiring."""
|
|
|
|
|
|
def test_glm_5_2_effort_reaches_top_level(self, zai_profile):
|
|
from agent.transports.chat_completions import ChatCompletionsTransport
|
|
|
|
kwargs = ChatCompletionsTransport().build_kwargs(
|
|
model="glm-5.2",
|
|
messages=[{"role": "user", "content": "ping"}],
|
|
tools=None,
|
|
provider_profile=zai_profile,
|
|
reasoning_config={"enabled": True, "effort": "max"},
|
|
base_url="https://api.z.ai/api/paas/v4",
|
|
provider_name="zai",
|
|
)
|
|
assert kwargs["reasoning_effort"] == "max"
|
|
assert kwargs["extra_body"]["thinking"] == {"type": "enabled", "clear_thinking": False}
|
|
|
|
@pytest.mark.parametrize("model,reasoning,thinking,effort", [
|
|
("glm-5.3-flash", {"enabled": False}, "enabled", "low"),
|
|
("glm-5.2", {"enabled": False}, "disabled", None),
|
|
("glm-5.3", None, None, None),
|
|
])
|
|
def test_glm_thinking_preference_reaches_transport(self, zai_profile, model, reasoning, thinking, effort):
|
|
from agent.transports.chat_completions import ChatCompletionsTransport
|
|
|
|
kwargs = ChatCompletionsTransport().build_kwargs(
|
|
model=model,
|
|
messages=[{"role": "user", "content": "ping"}],
|
|
tools=None,
|
|
provider_profile=zai_profile,
|
|
reasoning_config=reasoning,
|
|
base_url="https://open.bigmodel.cn/api/paas/v4",
|
|
provider_name="zai",
|
|
)
|
|
assert kwargs.get("reasoning_effort") == effort
|
|
expected = None if thinking is None else (
|
|
{"type": thinking, "clear_thinking": False} if thinking == "enabled" else {"type": thinking})
|
|
assert kwargs.get("extra_body", {}).get("thinking") == expected
|