1e866e34bf
The pre-first-token window used the env stale base, so an explicit per-provider stale timeout was ignored (900 configured, window capped at 180). It now goes through the main loop's _cloud_stale_timeout_for with the attempt's routed provider, so the explicit setting wins exactly as it does for the main loop.
883 lines
35 KiB
Python
883 lines
35 KiB
Python
"""Tests for the auxiliary forward-progress streaming layer.
|
|
|
|
Slow summary models must not be punished like hung ones (#see PR): when a
|
|
forward-progress hook is installed (context compression), the primary
|
|
auxiliary call streams and ticks the hook only for substantive payloads, so
|
|
outer watchdogs (gateway session hygiene) can extend their deadline on
|
|
liveness. Without a hook, behavior is byte-for-byte the old non-streaming call.
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent.auxiliary_client import (
|
|
_AnthropicCompletionsAdapter,
|
|
_ChatStreamAccumulator,
|
|
_CodexCompletionsAdapter,
|
|
_acreate_with_progress,
|
|
_acreate_with_stream,
|
|
_aggregate_chat_stream,
|
|
_aggregate_chat_stream_async,
|
|
_anthropic_event_has_content,
|
|
_aux_dispatch,
|
|
_aux_thread_local_hook,
|
|
_create_with_progress,
|
|
_create_with_progress_once,
|
|
_notify_aux_progress,
|
|
_provider_requires_stream,
|
|
aux_progress_hook,
|
|
)
|
|
from agent.codex_runtime import _codex_event_has_content
|
|
from agent.conversation_compression import CompressionCommitFence
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _chunk(content=None, reasoning=None, reasoning_details=None,
|
|
finish_reason=None, usage=None, tool_calls=None, model="m1",
|
|
chunk_id="c1"):
|
|
delta = SimpleNamespace(
|
|
content=content,
|
|
reasoning=reasoning,
|
|
reasoning_content=None,
|
|
reasoning_details=reasoning_details,
|
|
tool_calls=tool_calls,
|
|
)
|
|
choice = SimpleNamespace(delta=delta, finish_reason=finish_reason)
|
|
return SimpleNamespace(
|
|
id=chunk_id, model=model, choices=[choice], usage=usage,
|
|
)
|
|
|
|
|
|
class _FakeClient:
|
|
"""OpenAI-shaped client whose create() returns a canned value or stream."""
|
|
|
|
def __init__(self, response=None, stream_chunks=None, stream_error=None):
|
|
self.calls = []
|
|
self._response = response
|
|
self._stream_chunks = stream_chunks
|
|
self._stream_error = stream_error
|
|
completions = SimpleNamespace(create=self._create)
|
|
self.chat = SimpleNamespace(completions=completions)
|
|
|
|
def _create(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
if kwargs.get("stream"):
|
|
if self._stream_error is not None:
|
|
raise self._stream_error
|
|
return iter(self._stream_chunks or [])
|
|
return self._response
|
|
|
|
|
|
_COMPLETE = SimpleNamespace(
|
|
id="r1", model="m1", object="chat.completion",
|
|
choices=[SimpleNamespace(
|
|
index=0,
|
|
message=SimpleNamespace(role="assistant", content="non-streamed"),
|
|
finish_reason="stop",
|
|
)],
|
|
usage=None,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# aux_progress_hook plumbing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestAuxProgressHook:
|
|
def test_hook_installed_and_restored(self):
|
|
ticks = []
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
_notify_aux_progress()
|
|
_notify_aux_progress() # outside — must not tick
|
|
assert ticks == [1]
|
|
|
|
|
|
|
|
def test_hook_is_thread_local(self):
|
|
ticks = []
|
|
seen_in_thread = []
|
|
|
|
def _other_thread():
|
|
# No hook installed on this thread.
|
|
_notify_aux_progress()
|
|
seen_in_thread.append(len(ticks))
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
t = threading.Thread(target=_other_thread)
|
|
t.start()
|
|
t.join()
|
|
assert seen_in_thread == [0]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _create_with_progress
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCreateWithProgress:
|
|
|
|
def test_hook_upgrades_to_streaming_and_ticks_only_for_payload(self):
|
|
empty_tool_call = SimpleNamespace(
|
|
index=0,
|
|
id=None,
|
|
function=SimpleNamespace(name=None, arguments=""),
|
|
)
|
|
chunks = [
|
|
SimpleNamespace(id=None, model=None, choices=[], usage=None),
|
|
_chunk(content="", reasoning="", tool_calls=[empty_tool_call]),
|
|
_chunk(reasoning="thinking..."),
|
|
_chunk(content="Hello "),
|
|
_chunk(content="world", finish_reason="stop",
|
|
usage=SimpleNamespace(prompt_tokens=5, completion_tokens=2,
|
|
total_tokens=7)),
|
|
]
|
|
client = _FakeClient(stream_chunks=chunks)
|
|
ticks = []
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
result = _create_with_progress(
|
|
client, {"model": "m1", "messages": [], "timeout": 30},
|
|
)
|
|
assert client.calls[0]["stream"] is True
|
|
assert result.choices[0].message.content == "Hello world"
|
|
assert result.choices[0].message.reasoning == "thinking..."
|
|
assert result.choices[0].finish_reason == "stop"
|
|
assert result.usage.total_tokens == 7
|
|
# 1 tick per substantive chunk (reasoning, "Hello ", "world"); the
|
|
# dispatch itself is not progress (#114938).
|
|
assert ticks == [1, 1, 1]
|
|
|
|
def test_reasoning_only_in_model_extra_is_captured_and_counts_as_progress(self):
|
|
# Non-SDK delta objects (proxies, relays) may carry reasoning only in ``model_extra``;
|
|
# the accumulator must read it like the main streaming path does (#56516).
|
|
chunk = _chunk(finish_reason="stop")
|
|
chunk.choices[0].delta.model_extra = {"reasoning_content": "thinking..."}
|
|
client = _FakeClient(stream_chunks=[chunk])
|
|
ticks = []
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
result = _create_with_progress(client, {"model": "m1", "messages": [], "timeout": 30})
|
|
assert result.choices[0].message.reasoning == "thinking..."
|
|
assert ticks == [1] # the reasoning chunk only; dispatch is not progress (#114938)
|
|
|
|
def test_completed_response_ticks_only_terminal_signals(self):
|
|
calls = []
|
|
|
|
def _create(**kwargs):
|
|
calls.append(kwargs)
|
|
return _COMPLETE
|
|
|
|
client = SimpleNamespace(
|
|
chat=SimpleNamespace(completions=SimpleNamespace(create=_create))
|
|
)
|
|
ticks = []
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
result = _create_with_progress(client, {"model": "m1", "messages": []})
|
|
|
|
assert calls[0]["stream"] is True
|
|
assert result is _COMPLETE
|
|
# A completed response object carries the full summary payload: one
|
|
# terminal tick, and no dispatch tick (#114938).
|
|
assert ticks == [1]
|
|
|
|
def test_streaming_rejected_falls_back_to_plain_call(self):
|
|
client = _FakeClient(
|
|
response=_COMPLETE,
|
|
stream_error=RuntimeError("stream is not supported by this model"),
|
|
)
|
|
with aux_progress_hook(lambda: None):
|
|
result = _create_with_progress(
|
|
client, {"model": "m1", "messages": []},
|
|
)
|
|
assert result is _COMPLETE
|
|
# streamed attempt + non-streaming fallback
|
|
assert len(client.calls) == 2
|
|
assert client.calls[0].get("stream") is True
|
|
assert "stream" not in client.calls[1]
|
|
|
|
def test_dispatch_only_auth_failure_does_not_tick_progress(self):
|
|
"""Bug pin (#114938): a dispatch that dies before any payload (401) must
|
|
not reset the compression inactivity fence. Dispatch telemetry still
|
|
fires, but progress_observed stays False."""
|
|
fence = CompressionCommitFence()
|
|
dispatches = []
|
|
|
|
class _AuthError(Exception):
|
|
status_code = 401
|
|
|
|
client = _FakeClient(stream_error=_AuthError("unauthorized"))
|
|
with (
|
|
aux_progress_hook(fence.touch_progress),
|
|
_aux_thread_local_hook(_aux_dispatch, lambda: dispatches.append(1)),
|
|
pytest.raises(_AuthError),
|
|
):
|
|
_create_with_progress_once(
|
|
client, {"model": "m1", "messages": [], "timeout": 30},
|
|
)
|
|
assert dispatches == [1] # dispatch telemetry preserved
|
|
assert fence.progress_observed is False
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _aggregate_chat_stream
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestAggregateChatStream:
|
|
def test_tool_call_deltas_are_reassembled(self):
|
|
tc0 = SimpleNamespace(
|
|
index=0, id="call_1",
|
|
function=SimpleNamespace(name="do_thing", arguments='{"a"'),
|
|
)
|
|
tc1 = SimpleNamespace(
|
|
index=0, id=None,
|
|
function=SimpleNamespace(name=None, arguments=': 1}'),
|
|
)
|
|
chunks = [
|
|
_chunk(tool_calls=[tc0]),
|
|
_chunk(tool_calls=[tc1], finish_reason="tool_calls"),
|
|
]
|
|
result = _aggregate_chat_stream(iter(chunks))
|
|
tool_calls = result.choices[0].message.tool_calls
|
|
assert len(tool_calls) == 1
|
|
assert tool_calls[0].id == "call_1"
|
|
assert tool_calls[0].function.name == "do_thing"
|
|
assert tool_calls[0].function.arguments == '{"a": 1}'
|
|
assert result.choices[0].finish_reason == "tool_calls"
|
|
|
|
|
|
def test_stream_close_is_called(self):
|
|
closed = []
|
|
|
|
class _Stream:
|
|
def __iter__(self):
|
|
return iter([_chunk(content="ok", finish_reason="stop")])
|
|
|
|
def close(self):
|
|
closed.append(True)
|
|
|
|
result = _aggregate_chat_stream(_Stream())
|
|
assert result.choices[0].message.content == "ok"
|
|
assert closed == [True]
|
|
|
|
@staticmethod
|
|
def _sse_server(gates: dict):
|
|
"""Loopback HTTP/1.1 keepalive SSE endpoint; the request's ``model`` picks the script."""
|
|
import http.server
|
|
import json
|
|
|
|
def frame(handler, payload):
|
|
raw = (payload if isinstance(payload, str) else f"data: {json.dumps(payload)}\n\n").encode()
|
|
handler.wfile.write(b"%x\r\n%s\r\n" % (len(raw), raw))
|
|
handler.wfile.flush()
|
|
|
|
def chunk(content=None, finish=None, usage=None):
|
|
return {"id": "c1", "object": "chat.completion.chunk", "created": 0, "model": "m",
|
|
"choices": [{"index": 0, "delta": {"content": content} if content else {},
|
|
"finish_reason": finish}], **({"usage": usage} if usage else {})}
|
|
|
|
class _Handler(http.server.BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
script = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)))["model"]
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.send_header("Transfer-Encoding", "chunked")
|
|
self.end_headers()
|
|
self.wfile.flush()
|
|
try:
|
|
if script in ("openai/o3", "slow"): # silent reasoning before the first token
|
|
time.sleep(2.0)
|
|
if script == "b":
|
|
gates["b_started"].set()
|
|
frame(self, chunk(content="partial"))
|
|
if script == "usage": # terminal chunk with billed usage, then a stalled teardown
|
|
frame(self, chunk(finish="stop", usage={
|
|
"prompt_tokens": 100, "completion_tokens": 9, "total_tokens": 109}))
|
|
gates.get(script, gates["release"]).wait(30)
|
|
if script != "a": # "a" ends its body at EOF (no [DONE]), freeing the keepalive socket
|
|
frame(self, "data: [DONE]\n\n")
|
|
self.wfile.write(b"0\r\n\r\n")
|
|
self.wfile.flush()
|
|
except OSError:
|
|
pass
|
|
|
|
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
|
server.daemon_threads = True
|
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
return server
|
|
|
|
@pytest.fixture
|
|
def sse(self, monkeypatch):
|
|
"""(gates, call(model, task)) against the real OpenAI SDK + loopback transport. The default
|
|
window is 3s and ``auxiliary.compression.no_progress_timeout`` is 0.4s."""
|
|
import openai
|
|
|
|
gates = {name: threading.Event() for name in ("release", "a", "b", "b_started", "usage")}
|
|
server = self._sse_server(gates)
|
|
monkeypatch.setattr("agent.auxiliary_client._AUX_STREAM_NO_PROGRESS_TIMEOUT_SECONDS", 3.0)
|
|
monkeypatch.setattr("agent.auxiliary_client._get_task_no_progress_timeout",
|
|
lambda task: 0.4 if task == "compression" else None)
|
|
# Loopback stands in for a remote endpoint: the first-token window applies to non-local ones.
|
|
monkeypatch.setattr("agent.model_metadata.is_local_endpoint", lambda _url: False)
|
|
client = openai.OpenAI(api_key="k", base_url=f"http://127.0.0.1:{server.server_port}/v1", max_retries=0)
|
|
|
|
def call(model, task, *, via_relay_seam=False, provider=None):
|
|
from agent.auxiliary_client import _relay_aux_call_scope, _relay_sync_completion
|
|
|
|
request = {"model": model, "messages": [], "timeout": 20.0}
|
|
with aux_progress_hook(lambda: None):
|
|
if via_relay_seam: # a recovery rung / credential retry: the default relay callback
|
|
with _relay_aux_call_scope((task,), {}):
|
|
return _relay_sync_completion(client, request, provider=provider)
|
|
return _create_with_progress(client, request, task)
|
|
|
|
try:
|
|
yield gates, call
|
|
finally:
|
|
for gate in gates.values():
|
|
gate.set()
|
|
server.shutdown()
|
|
client.close()
|
|
|
|
@pytest.mark.parametrize("via_relay_seam", [False, True], ids=["primary", "recovery_rung"])
|
|
def test_silent_stream_fails_fast_through_the_real_transport(self, sse, via_relay_seam):
|
|
"""#100501: an endpoint that sends one chunk then goes silent fails at the task's no-progress
|
|
window as a timeout (so retry/fallback run), not at the request read timeout. Recovery rungs
|
|
go through the default relay callback and must keep the task's window."""
|
|
from agent.auxiliary_client import _should_skip_same_provider_retry
|
|
|
|
_gates, call = sse
|
|
started = time.monotonic()
|
|
with pytest.raises(TimeoutError) as excinfo:
|
|
call("m", "compression", via_relay_seam=via_relay_seam)
|
|
assert time.monotonic() - started < 5.0
|
|
assert "stalled: no new output for 0.4s" in str(excinfo.value)
|
|
assert _should_skip_same_provider_retry("compression", excinfo.value)
|
|
|
|
@pytest.mark.parametrize("case", ["reasoning_before_first_token", "provider_stale_timeout_before_first_token",
|
|
"stall_after_terminal_usage", "late_timer_after_pool_reuse"])
|
|
def test_watchdog_only_cuts_its_own_unfinished_stream(self, sse, case, monkeypatch):
|
|
"""The no-progress watchdog must not cut (a) a model thinking silently before its first token
|
|
for 5x the inter-chunk window (main-loop reasoning floor, or the routed provider's explicit
|
|
``providers.<id>.stale_timeout_seconds`` over a shorter env stale timeout), (b) a response
|
|
whose terminal chunk and billed usage already arrived, nor (c) a later request that reused
|
|
the keepalive connection after a stalled attempt's timer had already woken."""
|
|
gates, call = sse
|
|
if case == "reasoning_before_first_token":
|
|
gates["release"].set() # the stream ends right after its first chunk
|
|
assert call("openai/o3", "compression").choices[0].message.content == "partial"
|
|
return
|
|
if case == "provider_stale_timeout_before_first_token":
|
|
gates["release"].set()
|
|
monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.5")
|
|
monkeypatch.setattr("hermes_cli.config.load_config_readonly",
|
|
lambda: {"providers": {"openrouter": {"stale_timeout_seconds": 900}}})
|
|
result = call("slow", "compression", via_relay_seam=True, provider="openrouter")
|
|
assert result.choices[0].message.content == "partial"
|
|
return
|
|
if case == "stall_after_terminal_usage":
|
|
result = call("usage", "compression")
|
|
assert result.choices[0].message.content == "partial"
|
|
assert (result.usage.prompt_tokens, result.usage.completion_tokens) == (100, 9)
|
|
return
|
|
# Hold the stalled attempt's timer at the moment it shuts a socket down; meanwhile let
|
|
# attempt A end at EOF (its connection may return to the pool) and start B on the client.
|
|
paused, resume, outcome = threading.Event(), threading.Event(), {}
|
|
|
|
def tracer(frame, event, _arg):
|
|
if (event == "call" and frame.f_code.co_name == "_shutdown_socket"
|
|
and not paused.is_set()):
|
|
paused.set()
|
|
resume.wait(3)
|
|
|
|
def run(name, model, task):
|
|
try:
|
|
outcome[name] = call(model, task)
|
|
except Exception as exc:
|
|
outcome[name] = exc
|
|
|
|
threading.settrace(tracer)
|
|
try:
|
|
attempt_a = threading.Thread(target=run, args=("a", "a", "compression"))
|
|
attempt_a.start()
|
|
assert paused.wait(3)
|
|
gates["a"].set()
|
|
attempt_a.join(0.3)
|
|
attempt_b = threading.Thread(target=run, args=("b", "b", None))
|
|
attempt_b.start()
|
|
assert gates["b_started"].wait(3)
|
|
resume.set()
|
|
time.sleep(0.2)
|
|
gates["b"].set()
|
|
attempt_b.join(5)
|
|
attempt_a.join(5)
|
|
finally:
|
|
threading.settrace(None)
|
|
resume.set()
|
|
assert isinstance(outcome["a"], TimeoutError)
|
|
assert getattr(outcome["b"], "choices", None), outcome["b"]
|
|
assert outcome["b"].choices[0].message.content == "partial"
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Content-bearing progress classification
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestContentBearingProgress:
|
|
@pytest.mark.parametrize(
|
|
"event",
|
|
[
|
|
SimpleNamespace(type="response.created"),
|
|
SimpleNamespace(type="response.output_item.added", item={"type": "message"}),
|
|
SimpleNamespace(type="response.content_part.added", part={"type": "output_text"}),
|
|
SimpleNamespace(type="response.output_text.delta", delta=""),
|
|
SimpleNamespace(type="response.reasoning_summary_text.delta", delta=""),
|
|
SimpleNamespace(type="response.function_call_arguments.delta", delta=""),
|
|
SimpleNamespace(
|
|
type="response.output_item.added",
|
|
item=SimpleNamespace(type="function_call"),
|
|
),
|
|
{"type": "response.output_text.delta", "delta": ""},
|
|
],
|
|
)
|
|
def test_codex_empty_and_structural_events_are_not_progress(self, event):
|
|
assert _codex_event_has_content(event) is False
|
|
|
|
@pytest.mark.parametrize(
|
|
"event",
|
|
[
|
|
SimpleNamespace(type="response.output_text.delta", delta="token"),
|
|
SimpleNamespace(type="response.reasoning_summary_text.delta", delta="thought"),
|
|
SimpleNamespace(type="response.function_call_arguments.delta", delta='{"x"'),
|
|
SimpleNamespace(
|
|
type="response.output_item.added",
|
|
item=SimpleNamespace(
|
|
type="function_call",
|
|
id="item_1",
|
|
call_id="call_1",
|
|
name="lookup",
|
|
),
|
|
),
|
|
{"type": "response.output_text.delta", "delta": "token"},
|
|
],
|
|
)
|
|
def test_codex_nonempty_deltas_are_progress(self, event):
|
|
assert _codex_event_has_content(event) is True
|
|
|
|
@pytest.mark.parametrize(
|
|
("delta", "expected"),
|
|
[
|
|
(SimpleNamespace(type="text_delta", text="", thinking=None), False),
|
|
(SimpleNamespace(type="text_delta", text="token", thinking=None), True),
|
|
(SimpleNamespace(type="thinking_delta", text=None, thinking="thought"), True),
|
|
(SimpleNamespace(type="input_json_delta", partial_json=""), False),
|
|
(SimpleNamespace(type="input_json_delta", partial_json='{"x"'), True),
|
|
# Signed-thinking / citation payloads the transport emits
|
|
# (relay_llm.py signature_delta + citations_delta).
|
|
(SimpleNamespace(type="signature_delta", signature="sig-1"), True),
|
|
(SimpleNamespace(type="signature_delta", signature=""), False),
|
|
(SimpleNamespace(type="citations_delta", citation={"cited": 1}), True),
|
|
],
|
|
)
|
|
def test_anthropic_requires_nonempty_delta_payload(self, delta, expected):
|
|
event = SimpleNamespace(type="content_block_delta", delta=delta)
|
|
assert _anthropic_event_has_content(event) is expected
|
|
|
|
@pytest.mark.parametrize(
|
|
("block", "expected"),
|
|
[
|
|
(SimpleNamespace(type="text"), False),
|
|
(SimpleNamespace(type="tool_use", id=None, name=None), False),
|
|
(SimpleNamespace(type="tool_use", id="toolu_1", name="lookup"), True),
|
|
],
|
|
)
|
|
def test_anthropic_tool_start_requires_identity(self, block, expected):
|
|
event = SimpleNamespace(type="content_block_start", content_block=block)
|
|
assert _anthropic_event_has_content(event) is expected
|
|
|
|
def test_chat_stream_empty_tool_scaffolding_is_not_progress(self):
|
|
ticks = []
|
|
empty_tool_call = SimpleNamespace(
|
|
index=0,
|
|
id=None,
|
|
function=SimpleNamespace(name=None, arguments=""),
|
|
)
|
|
accumulator = _ChatStreamAccumulator()
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
accumulator.feed(_chunk(tool_calls=[empty_tool_call]))
|
|
|
|
assert ticks == []
|
|
|
|
@pytest.mark.parametrize(
|
|
"tool_call",
|
|
[
|
|
SimpleNamespace(index=0, id="call_1", function=None),
|
|
SimpleNamespace(
|
|
index=0,
|
|
id=None,
|
|
function=SimpleNamespace(name="lookup", arguments=""),
|
|
),
|
|
SimpleNamespace(
|
|
index=0,
|
|
id=None,
|
|
function=SimpleNamespace(name=None, arguments='{"q"'),
|
|
),
|
|
],
|
|
)
|
|
def test_chat_stream_substantive_tool_fragments_are_progress(self, tool_call):
|
|
ticks = []
|
|
accumulator = _ChatStreamAccumulator()
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
accumulator.feed(_chunk(tool_calls=[tool_call]))
|
|
|
|
assert ticks == [1]
|
|
|
|
def test_openrouter_reasoning_details_keep_compression_alive(self):
|
|
"""Reasoning-only streams must refresh the compression idle fence."""
|
|
ticks = []
|
|
accumulator = _ChatStreamAccumulator()
|
|
detail = {"type": "reasoning.summary", "summary": "working..."}
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
accumulator.feed(_chunk(reasoning_details=[detail]))
|
|
|
|
result = accumulator.finish()
|
|
assert ticks == [1]
|
|
assert result.choices[0].message.reasoning_details == [detail]
|
|
|
|
def test_structural_reasoning_details_are_not_progress(self):
|
|
ticks = []
|
|
accumulator = _ChatStreamAccumulator()
|
|
|
|
with aux_progress_hook(lambda: ticks.append(1)):
|
|
accumulator.feed(
|
|
_chunk(
|
|
reasoning_details=[
|
|
{"type": "reasoning.encrypted", "signature": "sig"}
|
|
]
|
|
)
|
|
)
|
|
|
|
assert ticks == []
|
|
|
|
def test_codex_adapter_updates_fence_only_for_substantive_events(self):
|
|
events = [
|
|
SimpleNamespace(type="response.created"),
|
|
SimpleNamespace(type="response.output_text.delta", delta=""),
|
|
SimpleNamespace(type="response.output_text.delta", delta="token"),
|
|
SimpleNamespace(
|
|
type="response.output_item.added",
|
|
item=SimpleNamespace(
|
|
type="function_call", id="item_1", call_id="call_1", name="lookup"
|
|
),
|
|
),
|
|
]
|
|
real_client = SimpleNamespace(
|
|
base_url="https://chatgpt.com/backend-api/codex",
|
|
responses=SimpleNamespace(create=lambda **_kwargs: iter(events)),
|
|
)
|
|
adapter = _CodexCompletionsAdapter(real_client, "gpt-5.6-sol")
|
|
fence = CompressionCommitFence()
|
|
touches = []
|
|
|
|
def _touch():
|
|
touches.append(1)
|
|
fence.touch_progress()
|
|
|
|
def _consume(stream, *, model, on_event):
|
|
del model
|
|
for event in stream:
|
|
on_event(event)
|
|
return SimpleNamespace(output=[], usage=None)
|
|
|
|
with (
|
|
patch("agent.codex_runtime._consume_codex_event_stream", _consume),
|
|
aux_progress_hook(_touch),
|
|
):
|
|
adapter.create(messages=[{"role": "user", "content": "summarize"}])
|
|
|
|
assert touches == [1, 1]
|
|
|
|
def test_anthropic_adapter_updates_fence_only_for_substantive_events(self):
|
|
events = [
|
|
SimpleNamespace(type="ping"),
|
|
SimpleNamespace(
|
|
type="content_block_delta",
|
|
delta=SimpleNamespace(type="text_delta", text=""),
|
|
),
|
|
SimpleNamespace(
|
|
type="content_block_delta",
|
|
delta=SimpleNamespace(type="input_json_delta", partial_json='{"q"'),
|
|
),
|
|
SimpleNamespace(
|
|
type="content_block_start",
|
|
content_block=SimpleNamespace(
|
|
type="tool_use", id="toolu_1", name="lookup"
|
|
),
|
|
),
|
|
]
|
|
adapter = _AnthropicCompletionsAdapter(
|
|
MagicMock(), "claude-sonnet-4-6", is_oauth=False
|
|
)
|
|
fence = CompressionCommitFence()
|
|
touches = []
|
|
|
|
def _touch():
|
|
touches.append(1)
|
|
fence.touch_progress()
|
|
|
|
def _create_message(*_args, **kwargs):
|
|
for event in events:
|
|
kwargs["on_stream_event"](event)
|
|
raise RuntimeError("stop after callback verification")
|
|
|
|
with (
|
|
patch(
|
|
"agent.anthropic_adapter.build_anthropic_kwargs",
|
|
return_value={"model": "claude-sonnet-4-6", "messages": []},
|
|
),
|
|
patch(
|
|
"agent.anthropic_adapter.create_anthropic_message",
|
|
side_effect=_create_message,
|
|
),
|
|
aux_progress_hook(_touch),
|
|
pytest.raises(RuntimeError, match="stop after callback verification"),
|
|
):
|
|
adapter.create(messages=[{"role": "user", "content": "summarize"}])
|
|
|
|
assert touches == [1, 1]
|
|
|
|
def test_keepalive_chunks_do_not_reset_the_compression_fence(self, monkeypatch):
|
|
"""End-to-end bug pin (#96707): content-free frames must not refresh
|
|
CompressionCommitFence._last_progress.
|
|
|
|
The waiter in conversation_compression charges its idle budget from
|
|
seconds_since_progress(); before the fix, every keepalive chunk fed
|
|
through _ChatStreamAccumulator ticked the fence, so a stalled
|
|
summary stream never hit the inactivity timeout."""
|
|
now = [10.0]
|
|
monkeypatch.setattr("agent.conversation_compression.time.monotonic", lambda: now[0])
|
|
fence = CompressionCommitFence()
|
|
now[0] = 20.0
|
|
accumulator = _ChatStreamAccumulator()
|
|
keepalive = SimpleNamespace(id=None, model=None, choices=[], usage=None)
|
|
empty_role_chunk = _chunk(content="", reasoning="")
|
|
|
|
with aux_progress_hook(fence.touch_progress):
|
|
for _ in range(5):
|
|
accumulator.feed(keepalive)
|
|
accumulator.feed(empty_role_chunk)
|
|
# No substantive payload arrived: the fence must have stayed stale.
|
|
assert fence.seconds_since_progress() > 0.0
|
|
|
|
with aux_progress_hook(fence.touch_progress):
|
|
accumulator.feed(_chunk(content="token"))
|
|
assert fence.seconds_since_progress() < 0.05
|
|
|
|
def test_content_free_frames_still_record_ttfp_timing(self):
|
|
"""The fast-lane telemetry contract (#96945/#96963) survives the
|
|
#96707 gating: the provider-response (time_to_first_progress_ms)
|
|
hook must fire on the FIRST frame of any kind (transport liveness),
|
|
not only on the first token."""
|
|
from agent.auxiliary_client import (
|
|
_aux_provider_response,
|
|
_aux_thread_local_hook,
|
|
)
|
|
|
|
responses: list = []
|
|
keepalive = SimpleNamespace(id=None, model=None, choices=[], usage=None)
|
|
accumulator = _ChatStreamAccumulator()
|
|
|
|
with (
|
|
_aux_thread_local_hook(_aux_provider_response, lambda: responses.append("response")),
|
|
aux_progress_hook(lambda: None),
|
|
):
|
|
accumulator.feed(keepalive)
|
|
|
|
assert responses, "content-free first frame must still record TTFP"
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Ceiling arithmetic
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CompressionCommitFence progress surface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stream-only providers (credit @kudi88, PR #60686)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestProviderRequiresStream:
|
|
|
|
def test_normal_endpoints_are_not(self):
|
|
assert _provider_requires_stream(
|
|
"openrouter", "https://openrouter.ai/api/v1"
|
|
) is False
|
|
assert _provider_requires_stream("auto", None) is False
|
|
assert _provider_requires_stream("auto", "") is False
|
|
|
|
def test_config_marker_matches_custom_endpoint(self):
|
|
with patch(
|
|
"hermes_cli.config.load_config",
|
|
return_value={"auxiliary": {"stream_only_base_urls": ["my-proxy.example.com"]}},
|
|
):
|
|
assert _provider_requires_stream(
|
|
"custom", "https://my-proxy.example.com/v1"
|
|
) is True
|
|
assert _provider_requires_stream(
|
|
"custom", "https://other.example.com/v1"
|
|
) is False
|
|
|
|
|
|
|
|
class TestForceStream:
|
|
def test_force_stream_streams_without_a_hook(self):
|
|
chunks = [_chunk(content="hi", finish_reason="stop")]
|
|
client = _FakeClient(stream_chunks=chunks)
|
|
# NO aux_progress_hook installed — force_stream alone must stream.
|
|
result = _create_with_progress(
|
|
client, {"model": "m1", "messages": []}, force_stream=True,
|
|
)
|
|
assert client.calls[0]["stream"] is True
|
|
assert result.choices[0].message.content == "hi"
|
|
|
|
def test_force_stream_does_not_retry_nonstreaming_on_failure(self):
|
|
client = _FakeClient(
|
|
response=_COMPLETE,
|
|
stream_error=RuntimeError("HTTP 400 bad request"),
|
|
)
|
|
with pytest.raises(RuntimeError, match="bad request"):
|
|
_create_with_progress(
|
|
client, {"model": "m1", "messages": []}, force_stream=True,
|
|
)
|
|
# No silent non-streaming retry — the provider rejects those anyway.
|
|
assert len(client.calls) == 1
|
|
|
|
|
|
class TestAsyncStreamAggregation:
|
|
@pytest.mark.asyncio
|
|
async def test_async_stream_is_consumed_with_async_for(self):
|
|
# The sweeper review of PR #60686 flagged that awaiting create() and
|
|
# then iterating synchronously raises — the async contract is
|
|
# ``async for``. Verify the async aggregator consumes a real async
|
|
# iterator and preserves tool-call deltas.
|
|
tc0 = SimpleNamespace(
|
|
index=0, id="call_9",
|
|
function=SimpleNamespace(name="lookup", arguments='{"q":'),
|
|
)
|
|
tc1 = SimpleNamespace(
|
|
index=0, id=None,
|
|
function=SimpleNamespace(name=None, arguments='"x"}'),
|
|
)
|
|
raw_chunks = [
|
|
_chunk(content="part1 "),
|
|
_chunk(tool_calls=[tc0]),
|
|
_chunk(tool_calls=[tc1], content="part2", finish_reason="tool_calls"),
|
|
]
|
|
|
|
class _AsyncStream:
|
|
def __init__(self, items):
|
|
self._items = list(items)
|
|
self.closed = False
|
|
|
|
def __aiter__(self):
|
|
return self
|
|
|
|
async def __anext__(self):
|
|
if not self._items:
|
|
raise StopAsyncIteration
|
|
return self._items.pop(0)
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
stream = _AsyncStream(raw_chunks)
|
|
result = await _aggregate_chat_stream_async(stream)
|
|
msg = result.choices[0].message
|
|
assert msg.content == "part1 part2"
|
|
assert msg.tool_calls[0].function.name == "lookup"
|
|
assert msg.tool_calls[0].function.arguments == '{"q":"x"}'
|
|
assert result.choices[0].finish_reason == "tool_calls"
|
|
assert stream.closed is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_acreate_with_stream_passes_stream_kwargs(self):
|
|
calls = []
|
|
|
|
class _AsyncStream:
|
|
def __init__(self, items):
|
|
self._items = list(items)
|
|
|
|
def __aiter__(self):
|
|
return self
|
|
|
|
async def __anext__(self):
|
|
if not self._items:
|
|
raise StopAsyncIteration
|
|
return self._items.pop(0)
|
|
|
|
class _AsyncClient:
|
|
def __init__(self):
|
|
completions = SimpleNamespace(create=self._create)
|
|
self.chat = SimpleNamespace(completions=completions)
|
|
|
|
async def _create(self, **kwargs):
|
|
calls.append(kwargs)
|
|
return _AsyncStream([_chunk(content="ok", finish_reason="stop")])
|
|
|
|
result = await _acreate_with_stream(
|
|
_AsyncClient(), {"model": "m1", "messages": [], "timeout": 30},
|
|
)
|
|
assert calls[0]["stream"] is True
|
|
assert result.choices[0].message.content == "ok"
|
|
@pytest.mark.asyncio
|
|
async def test_async_dispatch_only_auth_failure_does_not_tick_progress(self):
|
|
"""Async twin of the #114938 pin: a 401 dispatch must not reset the
|
|
compression inactivity fence."""
|
|
fence = CompressionCommitFence()
|
|
dispatches = []
|
|
|
|
class _AuthError(Exception):
|
|
status_code = 401
|
|
|
|
class _AsyncClient:
|
|
def __init__(self):
|
|
completions = SimpleNamespace(create=self._create)
|
|
self.chat = SimpleNamespace(completions=completions)
|
|
|
|
async def _create(self, **kwargs):
|
|
raise _AuthError("unauthorized")
|
|
|
|
with (
|
|
aux_progress_hook(fence.touch_progress),
|
|
_aux_thread_local_hook(_aux_dispatch, lambda: dispatches.append(1)),
|
|
pytest.raises(_AuthError),
|
|
):
|
|
await _acreate_with_progress(
|
|
_AsyncClient(), {"model": "m1", "messages": [], "timeout": 30},
|
|
)
|
|
assert dispatches == [1]
|
|
assert fence.progress_observed is False
|