fix(agent): collapse orphan continuation trail on retry exhaustion (#119001)

Ports #119027::finalize_continuation_partial: when a stream drop entered the
continuation path and the next request exhausted retries before a token, the
_length_continuation_fragment/_nudge rows were persisted as-is, so resume
replayed a dangling synthetic user nudge. Collapse them into one assistant row
before persistence and feed the collapsed text to the #119081 partial-retention
path (the fragment rows are gone by then).

Co-authored-by: fangliquan <fangliquan@qq.com>
This commit is contained in:
kshitijk4poor
2026-09-24 17:00:24 +05:30
committed by kshitij
parent e947f60689
commit bfeafa2d68
4 changed files with 96 additions and 51 deletions

View File

@@ -22,7 +22,7 @@ from agent.turn_recovery import (
_NONRETRYABLE_LABELS, abort_turn_on_interrupt, compute_error_backoff, interruptible_backoff_sleep, _NONRETRYABLE_LABELS, abort_turn_on_interrupt, compute_error_backoff, interruptible_backoff_sleep,
log_api_error_attempt, log_api_error_attempt,
max_retries_exhausted_result, nonretryable_client_error_result, recover_after_classification, max_retries_exhausted_result, nonretryable_client_error_result, recover_after_classification,
recover_before_classification, route_classified_error, recover_before_classification, route_classified_error, settle_delivered_partial,
) )
logger = logging.getLogger("agent.conversation_loop") logger = logging.getLogger("agent.conversation_loop")
@@ -55,6 +55,7 @@ def handle_api_error(
conversation_history: Any, approx_tokens: Any, retry_count: Any, max_retries: Any, conversation_history: Any, approx_tokens: Any, retry_count: Any, max_retries: Any,
compression_attempts: Any, max_compression_attempts: Any, api_call_count: Any, compression_attempts: Any, max_compression_attempts: Any, api_call_count: Any,
api_request_id: Any, api_start_time: Any, effective_task_id: Any, turn_id: Any, api_request_id: Any, api_start_time: Any, effective_task_id: Any, turn_id: Any,
current_turn_user_idx: Any = None,
) -> ApiErrorVerdict: ) -> ApiErrorVerdict:
"""Recover from ``api_error`` in the original order. Every fallback activation must leave """Recover from ``api_error`` in the original order. Every fallback activation must leave
the retry loop with ``restart_with_rebuilt_messages`` armed (``"break"``) so the pre-API the retry loop with ``restart_with_rebuilt_messages`` armed (``"break"``) so the pre-API
@@ -212,7 +213,7 @@ def handle_api_error(
api_messages=api_messages, api_kwargs=api_kwargs, active_system_prompt=active_system_prompt, api_messages=api_messages, api_kwargs=api_kwargs, active_system_prompt=active_system_prompt,
conversation_history=conversation_history, approx_tokens=approx_tokens, conversation_history=conversation_history, approx_tokens=approx_tokens,
retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts, retry_count=retry_count, max_retries=max_retries, compression_attempts=compression_attempts,
api_call_count=api_call_count, api_call_count=api_call_count, current_turn_user_idx=current_turn_user_idx,
) )
active_system_prompt = _ue.active_system_prompt active_system_prompt = _ue.active_system_prompt
retry_count = _ue.retry_count retry_count = _ue.retry_count
@@ -255,6 +256,7 @@ def settle_unrecovered_error(
_provider: Any, _base: Any, _model: Any, messages: Any, api_messages: Any, api_kwargs: Any, _provider: Any, _base: Any, _model: Any, messages: Any, api_messages: Any, api_kwargs: Any,
active_system_prompt: Any, conversation_history: Any, approx_tokens: Any, retry_count: Any, active_system_prompt: Any, conversation_history: Any, approx_tokens: Any, retry_count: Any,
max_retries: Any, compression_attempts: Any, api_call_count: Any, error_context: Any = None, max_retries: Any, compression_attempts: Any, api_call_count: Any, error_context: Any = None,
current_turn_user_idx: Any = None,
) -> UnrecoveredErrorVerdict: ) -> UnrecoveredErrorVerdict:
"""Decide the fate of an API error that every recovery chain declined: local validation / """Decide the fate of an API error that every recovery chain declined: local validation /
non-retryable client errors (Copilot stale-credential self-heal first, then fallback, then a non-retryable client errors (Copilot stale-credential self-heal first, then fallback, then a
@@ -342,11 +344,13 @@ def settle_unrecovered_error(
active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry) active_system_prompt = _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry)
retry_count = compression_attempts = 0 retry_count = compression_attempts = 0
return _verdict("break") return _verdict("break")
# Terminal from here: collapse the continuation trail once, before the persist.
_delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
return _verdict("return", nonretryable_client_error_result( return _verdict("return", nonretryable_client_error_result(
agent, api_error, classified, status_code=status_code, api_kwargs=api_kwargs, agent, api_error, classified, status_code=status_code, api_kwargs=api_kwargs,
api_messages=api_messages, messages=messages, conversation_history=conversation_history, api_messages=api_messages, messages=messages, conversation_history=conversation_history,
api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider, api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
base_url=_base, model=_model, base_url=_base, model=_model, delivered=_delivered,
)) ))
if retry_count >= max_retries: if retry_count >= max_retries:
@@ -383,12 +387,13 @@ def settle_unrecovered_error(
if _ladder["action"] == "continue": if _ladder["action"] == "continue":
retry_count = 0 retry_count = 0
return _verdict(_ladder["action"], _ladder.get("result")) return _verdict(_ladder["action"], _ladder.get("result"))
_delivered = settle_delivered_partial(agent, messages, current_turn_user_idx)
return _verdict("return", max_retries_exhausted_result( return _verdict("return", max_retries_exhausted_result(
agent, api_error, classified, max_retries=max_retries, is_rate_limited=is_rate_limited, agent, api_error, classified, max_retries=max_retries, is_rate_limited=is_rate_limited,
error_msg=error_msg, api_kwargs=api_kwargs, api_messages=api_messages, error_msg=error_msg, api_kwargs=api_kwargs, api_messages=api_messages,
messages=messages, conversation_history=conversation_history, messages=messages, conversation_history=conversation_history,
api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider, api_call_count=api_call_count, approx_tokens=approx_tokens, provider=_provider,
base_url=_base, model=_model, base_url=_base, model=_model, delivered=_delivered,
)) ))
wait_time = compute_error_backoff( wait_time = compute_error_backoff(

View File

@@ -773,37 +773,26 @@ def _failed_turn_result(final_response: str, messages: Any, api_call_count: int,
} }
def _recover_delivered_partial_text(agent: Any, messages: Any) -> str: def settle_delivered_partial(agent: Any, messages: Any, current_turn_user_idx: Any) -> str:
"""Visible assistant text already delivered this turn ("" when none). """Visible text already delivered this turn ("" when none), collapsing any continuation
trail first so the terminal persist never keeps a dangling synthetic nudge (#119001).
``build_api_request`` resets ``_current_streamed_assistant_text`` on every ``build_api_request`` resets ``_current_streamed_assistant_text`` per attempt, so after a
attempt, so after a mid-stream death + continuation + pre-stream 429 the mid-stream death + continuation + pre-stream error the live accumulator is empty and the
live accumulator is empty and the only record is the fragment rows (now collapsed into one assistant row) are the only record.
``_length_continuation_fragment`` rows the truncation path appended (#119001).
""" """
try: from agent.turn_truncation import collapse_continuation_trail
_live = getattr(agent, "_current_streamed_assistant_text", "") or "" collapsed = collapse_continuation_trail(
except Exception: agent, messages, current_turn_user_idx, finish_reason="error",
_live = "" )
if isinstance(_live, str) and _live.strip(): live = getattr(agent, "_current_streamed_assistant_text", "")
return _live.strip() if isinstance(live, str) and live.strip():
_parts = [ from agent.agent_runtime_helpers import strip_think_blocks
m["content"].strip() for m in messages or () return strip_think_blocks(agent, live).strip() or collapsed
if isinstance(m, dict) and m.get("_length_continuation_fragment") return collapsed
and isinstance(m.get("content"), str) and m["content"].strip()
]
if not _parts:
return ""
# Same glue as _join_truncated_parts: newline where two parts would stick.
_joined = ""
for _part in _parts:
if _joined and not _joined[-1].isspace() and not _part[0].isspace():
_joined += "\n"
_joined += _part
return _joined.strip()
def _with_delivered_partial(final_response: str, error_summary: str, agent: Any, messages: Any) -> tuple: def _with_delivered_partial(final_response: str, error_summary: str, delivered: str) -> tuple:
"""Prepend delivered partial text to a terminal error body ("" unchanged). """Prepend delivered partial text to a terminal error body ("" unchanged).
Returns ``(final_response, keep_partial)``; callers set ``result["partial"]`` Returns ``(final_response, keep_partial)``; callers set ``result["partial"]``
@@ -811,8 +800,8 @@ def _with_delivered_partial(final_response: str, error_summary: str, agent: Any,
retain the bubble instead of clearing it. ``final_response`` must stay retain the bubble instead of clearing it. ``final_response`` must stay
distinct from ``error`` — that inequality is the retention contract. distinct from ``error`` — that inequality is the retention contract.
""" """
_delivered = _recover_delivered_partial_text(agent, messages) _delivered = (delivered or "").strip()
if not _delivered or _delivered.strip() == (error_summary or "").strip(): if not _delivered or _delivered == (error_summary or "").strip():
return final_response, False return final_response, False
return f"{_delivered}\n\n{final_response}", True return f"{_delivered}\n\n{final_response}", True
@@ -984,6 +973,7 @@ def nonretryable_client_error_result(
agent: Any, api_error: Exception, classified: Any, *, status_code: Optional[int], agent: Any, api_error: Exception, classified: Any, *, status_code: Optional[int],
api_kwargs: Any, api_messages: Any, messages: List[Dict[str, Any]], conversation_history: Any, api_kwargs: Any, api_messages: Any, messages: List[Dict[str, Any]], conversation_history: Any,
api_call_count: int, approx_tokens: int, provider: Any, base_url: Any, model: Any, api_call_count: int, approx_tokens: int, provider: Any, base_url: Any, model: Any,
delivered: str = "",
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Terminal path for a non-retryable 4xx once fallback is exhausted: debug dump, flush """Terminal path for a non-retryable 4xx once fallback is exhausted: debug dump, flush
the retry trace, print auth / billing / content-policy / TLS guidance, persist (skipped the retry trace, print auth / billing / content-policy / TLS guidance, persist (skipped
@@ -1091,7 +1081,7 @@ def nonretryable_client_error_result(
# (agent/error_surface.py) reads a rejected OAuth token as a retryable # (agent/error_surface.py) reads a rejected OAuth token as a retryable
# "Provider error" and offers Retry instead of a re-login. # "Provider error" and offers Retry instead of a re-login.
_final_response, _keep_partial = _with_delivered_partial( _final_response, _keep_partial = _with_delivered_partial(
_final_response, _nonretryable_summary, agent, messages, _final_response, _nonretryable_summary, delivered,
) )
result = _failed_turn_result(_final_response, messages, api_call_count, _nonretryable_summary) result = _failed_turn_result(_final_response, messages, api_call_count, _nonretryable_summary)
result.update({ result.update({
@@ -1118,7 +1108,7 @@ def max_retries_exhausted_result(
agent: Any, api_error: Exception, classified: Any, *, max_retries: int, is_rate_limited: bool, agent: Any, api_error: Exception, classified: Any, *, max_retries: int, is_rate_limited: bool,
error_msg: str, api_kwargs: Any, api_messages: Any, messages: List[Dict[str, Any]], error_msg: str, api_kwargs: Any, api_messages: Any, messages: List[Dict[str, Any]],
conversation_history: Any, api_call_count: int, approx_tokens: int, provider: Any, conversation_history: Any, api_call_count: int, approx_tokens: int, provider: Any,
base_url: Any, model: Any, base_url: Any, model: Any, delivered: str = "",
) -> Dict[str, Any]: ) -> Dict[str, Any]:
"""Terminal path once retries, transport recovery and fallback all failed: flush the """Terminal path once retries, transport recovery and fallback all failed: flush the
trace, emit the billing / rate-limit / generic status, print stream-drop or thinking-timeout trace, emit the billing / rate-limit / generic status, print stream-drop or thinking-timeout
@@ -1243,7 +1233,7 @@ def max_retries_exhausted_result(
# shown, so keep it as the reply (marked failed) instead of an error-only # shown, so keep it as the reply (marked failed) instead of an error-only
# turn — the gateway flags ``partial`` and surfaces retain the bubble. # turn — the gateway flags ``partial`` and surfaces retain the bubble.
_final_response, _keep_partial = _with_delivered_partial( _final_response, _keep_partial = _with_delivered_partial(
_final_response, _final_summary, agent, messages, _final_response, _final_summary, delivered,
) )
if _keep_partial: if _keep_partial:
result["final_response"] = _final_response result["final_response"] = _final_response

View File

@@ -42,6 +42,49 @@ _CONTEXT_OVERFLOW_PARTIAL_FINAL = (
"chats are reset automatically)." "chats are reset automatically)."
) )
def collapse_continuation_trail(
agent: Any, messages: List[Dict[str, Any]], current_turn_user_idx: Any, *,
finish_reason: str, parts: Optional[List[str]] = None,
) -> str:
"""Drop this turn's ``_length_continuation_fragment``/``_nudge`` rows and append one
assistant row holding the joined, think-stripped partial; returns that text ("" none).
``parts=None`` (retry exhaustion, #119001): the text comes from the fragment rows and
nothing happens without a valid turn index or a trail — an unanswered synthetic nudge
must never be persisted, and an earlier turn's rows must never be read. Explicit
``parts`` (the continuation ceiling) always appends, scanning from 0 without an index.
"""
idx = current_turn_user_idx
valid_idx = isinstance(idx, int) and idx >= 0
if parts is None and not (valid_idx and idx < len(messages)):
return ""
turn_start = idx + 1 if valid_idx else 0
fragment_parts: List[str] = []
retained: List[Any] = []
found_trail = False
for message in messages[turn_start:]:
if isinstance(message, dict) and (
message.get("_length_continuation_fragment") or message.get("_length_continuation_nudge")
):
found_trail = True
content = message.get("content")
if message.get("_length_continuation_fragment") and isinstance(content, str) and content:
fragment_parts.append(content)
continue
retained.append(message)
if parts is None and not found_trail:
return ""
messages[turn_start:] = retained
from agent.conversation_loop import _join_truncated_parts
partial = agent._strip_think_blocks(
_join_truncated_parts(fragment_parts if parts is None else parts)
).strip()
if partial:
append_message(messages, {"role": "assistant", "content": partial, "finish_reason": finish_reason})
agent._session_messages = messages
return partial
_THINKING_EXHAUSTED = ( _THINKING_EXHAUSTED = (
"💭 Reasoning exhausted the output token budget — no visible response was produced.", "💭 Reasoning exhausted the output token budget — no visible response was produced.",
"⚠️ **Thinking Budget Exhausted**\n\nThe model used all its output tokens on reasoning " "⚠️ **Thinking Budget Exhausted**\n\nThe model used all its output tokens on reasoning "
@@ -288,7 +331,11 @@ def _continue_text(st: _Trunc, _retry: TurnRetryState, assistant_message: Any) -
_retry.restart_with_length_continuation = True _retry.restart_with_length_continuation = True
return st.done("break") return st.done("break")
partial_response = agent._strip_think_blocks(_join_truncated_parts(st.truncated_response_parts)).strip() # Unanswered continue nudges made every later turn re-truncate: drop the trail.
partial_response = collapse_continuation_trail(
agent, messages, st.current_turn_user_idx, finish_reason="length",
parts=st.truncated_response_parts,
)
# The one-shot reasoning-off override must not leak into the next turn. # The one-shot reasoning-off override must not leak into the next turn.
agent._ephemeral_reasoning_off = False agent._ephemeral_reasoning_off = False
agent._vprint( agent._vprint(
@@ -299,20 +346,6 @@ def _continue_text(st: _Trunc, _retry: TurnRetryState, assistant_message: Any) -
else "no visible text was produced."), else "no visible text was produced."),
force=True, diagnostic=True, force=True, diagnostic=True,
) )
# Unanswered continue nudges made every later turn re-truncate: drop the trail.
idx = st.current_turn_user_idx
_turn_start = idx + 1 if isinstance(idx, int) and idx >= 0 else 0
messages[_turn_start:] = [
m for m in messages[_turn_start:]
if not (isinstance(m, dict) and (
m.get("_length_continuation_fragment") or m.get("_length_continuation_nudge")
))
]
if partial_response:
append_message(messages, {
"role": "assistant", "content": partial_response, "finish_reason": "length"
})
agent._session_messages = messages
if filled is not None: if filled is not None:
notice = _WINDOW_FILLED.format(prompt=filled[0], ctx=filled[1]) notice = _WINDOW_FILLED.format(prompt=filled[0], ctx=filled[1])
return st.end_turn( return st.end_turn(

View File

@@ -121,3 +121,20 @@ def test_nonretryable_terminal_keeps_delivered_partial():
assert result.get("partial") is True assert result.get("partial") is True
assert PARTIAL in result["final_response"] assert PARTIAL in result["final_response"]
assert result["final_response"].strip() != str(result["error"]).strip() assert result["final_response"].strip() != str(result["error"]).strip()
def test_exhausted_429_collapses_continuation_trail_into_one_assistant_row():
messages = _messages_with_fragment()
error = _Http(429, "HTTP 429: RequestBurstTooFast — slow down traffic growth")
classified = classify_api_error(error, provider="openrouter", model="m")
result = max_retries_exhausted_result(
_Agent(), error, classified, max_retries=3, is_rate_limited=True,
error_msg=str(error).lower(), api_kwargs=None, api_messages=[], messages=messages,
conversation_history=None, api_call_count=3, approx_tokens=10, provider="openrouter",
base_url="https://openrouter.ai/api/v1", model="m", current_turn_user_idx=0,
)
# No dangling synthetic nudge: the turn persists as user -> one assistant row.
assert [m["role"] for m in messages] == ["user", "assistant"]
assert messages[1]["content"] == PARTIAL
assert not any(m.get("_length_continuation_nudge") for m in messages)
assert result.get("partial") is True and PARTIAL in result["final_response"]