fix(agent): gate stop-path repetition abort on runaway shape
Use is_runaway_repetition on completed replies so asked-for repetition with distinct lines is delivered; stamp the truncated/retryable failure verdict, log as diagnostic, and derive both repetition copies from one builder.
This commit is contained in:
@@ -40,6 +40,11 @@ _MAX_ANCHOR_MATCHES = 8
|
||||
# scan, but every line is distinct; a loop re-emits the same line(s).
|
||||
_RUNAWAY_DISTINCT_LINE_RATIO = 0.5
|
||||
|
||||
# The finish_reason="stop" path discards a COMPLETED answer, so it only aborts at runaway scale:
|
||||
# real stop-path loops (#100716) run 80k-350k chars, while asked-for repeats ("say X 50 times",
|
||||
# identical table rows, templated YAML) stay in the low KB and must be delivered.
|
||||
STOP_PATH_MIN_CHARS = 16_000
|
||||
|
||||
|
||||
def is_repetition_dominated(text: str) -> bool:
|
||||
"""True when a contiguous run of at least five exact repetitions covers at least half
|
||||
|
||||
@@ -12,20 +12,17 @@ import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from agent.message_metadata import append_message
|
||||
from agent.repetition_guard import is_repetition_dominated
|
||||
from agent.repetition_guard import STOP_PATH_MIN_CHARS, is_runaway_repetition
|
||||
from agent.turn_failure_copy import stamp_failure
|
||||
from agent.turn_empty_response import recover_empty_response
|
||||
from agent.turn_stop_gates import apply_stop_gates
|
||||
from agent.turn_truncation import partial_result
|
||||
from agent.turn_truncation import partial_result, repetition_copy
|
||||
|
||||
_REPETITION_STOPPED = (
|
||||
"🔁 Response dominated by repeated text — stopping before delivery.",
|
||||
"⚠️ **Response Stopped — Repetition Detected**\n\nThe model fell into a repetition loop while "
|
||||
"writing this response, so the repeated output was discarded.\n\n"
|
||||
"→ Switch to a different model with `/model`\n"
|
||||
"→ Or resend your message (your conversation history is preserved)",
|
||||
"Model output entered a repetition loop; refusing to return a degenerate response.",
|
||||
_REPETITION_STOPPED = repetition_copy(
|
||||
"before delivery",
|
||||
"so the repeated output was discarded.",
|
||||
"; refusing to return a",
|
||||
)
|
||||
|
||||
logger = logging.getLogger("agent.conversation_loop")
|
||||
|
||||
# Ephemeral retry scaffolding rows popped before the final answer becomes durable.
|
||||
@@ -62,7 +59,7 @@ def finish_text_response(
|
||||
_preflight_compression_blocked: Any, codex_ack_continuations: Any,
|
||||
truncated_response_parts: Any, length_continue_retries: Any,
|
||||
_pending_verification_response: Any, _pending_verification_response_previewed: Any,
|
||||
effective_task_id: Any = None,
|
||||
effective_task_id: Any,
|
||||
) -> FinalResponseVerdict:
|
||||
"""Finish (or defer) a text-only assistant response in the original guard order. Every
|
||||
continuation path sets ``final_response = None`` so an acknowledgment never suppresses
|
||||
@@ -258,12 +255,20 @@ def finish_text_response(
|
||||
# A provider may end a degenerate loop normally with finish_reason="stop" instead of
|
||||
# exhausting its output cap (#100716). Check every completed visible text response before
|
||||
# any verify/kanban interim emission or durable transcript write.
|
||||
if final_response and is_repetition_dominated(final_response):
|
||||
# Runaway scale and shape only: a completed answer the user asked to be repetitive is
|
||||
# delivered, unlike a length-truncated fragment that burned the whole budget.
|
||||
if (
|
||||
final_response
|
||||
and len(final_response) >= STOP_PATH_MIN_CHARS
|
||||
and is_runaway_repetition(final_response)
|
||||
):
|
||||
line, user_response, error = _REPETITION_STOPPED
|
||||
agent._vprint(f"{agent.log_prefix}{line}", force=True)
|
||||
agent._vprint(f"{agent.log_prefix}{line}", force=True, diagnostic=True)
|
||||
agent._cleanup_task_resources(effective_task_id)
|
||||
agent._persist_session(messages, conversation_history)
|
||||
return _verdict("return", partial_result(messages, api_call_count, user_response, error))
|
||||
return _verdict("return", stamp_failure(
|
||||
partial_result(messages, api_call_count, user_response, error), "truncated", True,
|
||||
))
|
||||
|
||||
final_msg = agent._build_assistant_message(assistant_message, finish_reason)
|
||||
if _promoted:
|
||||
|
||||
@@ -51,14 +51,23 @@ _THINKING_EXHAUSTED = (
|
||||
"Model used all output tokens on reasoning with none left "
|
||||
"for the response. Try lowering reasoning effort or increasing max_tokens.",
|
||||
)
|
||||
_REPETITION_DOMINATED = (
|
||||
"🔁 Response dominated by repeated text — stopping instead of continuing a degenerate response.",
|
||||
"⚠️ **Response Stopped — Repetition Detected**\n\nThe model fell into a repetition loop while "
|
||||
"writing this response, so continuing would only produce more repeated text. The partial response "
|
||||
"was discarded.\n\n→ Switch to a different model with `/model`\n"
|
||||
"→ Or resend your message (your conversation history is preserved)",
|
||||
"Model output entered a repetition loop and was truncated mid-loop; refusing to continue a "
|
||||
"degenerate response.",
|
||||
|
||||
def repetition_copy(stopping: str, outcome: str, refusal: str) -> Tuple[str, str, str]:
|
||||
"""(log line, user copy, error) for a repetition-dominated abort; only the clauses naming
|
||||
where the turn stopped differ between the length path and the stop path."""
|
||||
return (
|
||||
f"🔁 Response dominated by repeated text — stopping {stopping}.",
|
||||
"⚠️ **Response Stopped — Repetition Detected**\n\nThe model fell into a repetition loop while "
|
||||
f"writing this response, {outcome}\n\n→ Switch to a different model with `/model`\n"
|
||||
"→ Or resend your message (your conversation history is preserved)",
|
||||
f"Model output entered a repetition loop{refusal} degenerate response.",
|
||||
)
|
||||
|
||||
|
||||
_REPETITION_DOMINATED = repetition_copy(
|
||||
"instead of continuing a degenerate response",
|
||||
"so continuing would only produce more repeated text. The partial response was discarded.",
|
||||
" and was truncated mid-loop; refusing to continue a",
|
||||
)
|
||||
_CEILING_NO_TEXT = (
|
||||
"⚠️ **No visible answer was produced.** The model hit its output-token limit on every "
|
||||
|
||||
@@ -12,6 +12,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.repetition_guard import STOP_PATH_MIN_CHARS
|
||||
from hermes_constants import FINISH_REASON_LENGTH, PARTIAL_STREAM_STUB_ID
|
||||
|
||||
# The exact sentence from the #86581 incident.
|
||||
@@ -62,10 +63,6 @@ def _response(
|
||||
)
|
||||
|
||||
|
||||
def _stub(content):
|
||||
return _response(content)
|
||||
|
||||
|
||||
def _run(agent, message):
|
||||
with (
|
||||
patch.object(agent, "_persist_session"),
|
||||
@@ -78,7 +75,7 @@ def _run(agent, message):
|
||||
class TestContinuationRepetitionGuard:
|
||||
def test_repetition_dominated_truncation_aborts(self, loop_agent):
|
||||
echo = _INCIDENT_ECHO * 2000
|
||||
loop_agent.client.chat.completions.create.side_effect = [_stub(echo)]
|
||||
loop_agent.client.chat.completions.create.side_effect = [_response(echo)]
|
||||
|
||||
result = _run(loop_agent, "write me a long report")
|
||||
|
||||
@@ -93,33 +90,45 @@ class TestContinuationRepetitionGuard:
|
||||
# Exactly one API call — no continuation was attempted.
|
||||
assert loop_agent.client.chat.completions.create.call_count == 1
|
||||
|
||||
def test_repetition_dominated_stop_response_aborts(self, loop_agent):
|
||||
paragraph = (
|
||||
"A long paragraph that should never be delivered hundreds of times "
|
||||
"when a model enters a repetition loop.\n"
|
||||
"The second line makes this a multiline repeating unit.\n"
|
||||
)
|
||||
echo = paragraph * 500
|
||||
@pytest.mark.parametrize("requested_repeat", [False, True], ids=["loop", "repeat-on-request"])
|
||||
def test_repetition_dominated_stop_response_aborts(self, loop_agent, requested_repeat):
|
||||
if requested_repeat:
|
||||
# Asked-for repetition (identical lines, ~3.6k chars) is below runaway scale: a
|
||||
# completed answer must be delivered, not discarded.
|
||||
echo = "Hello world, this is a sentence the user asked me to repeat many times.\n" * 50
|
||||
else:
|
||||
paragraph = (
|
||||
"A long paragraph that should never be delivered hundreds of times "
|
||||
"when a model enters a repetition loop.\n"
|
||||
"The second line makes this a multiline repeating unit.\n"
|
||||
)
|
||||
echo = paragraph * 500
|
||||
assert len(echo) >= STOP_PATH_MIN_CHARS
|
||||
loop_agent.client.chat.completions.create.side_effect = [
|
||||
_response(echo, finish_reason="stop", response_id="completed-response")
|
||||
]
|
||||
|
||||
result = _run(loop_agent, "write me a long report")
|
||||
|
||||
assert loop_agent.client.chat.completions.create.call_count == 1
|
||||
if requested_repeat:
|
||||
assert result["completed"] is True
|
||||
assert result["final_response"] == echo.strip()
|
||||
return
|
||||
assert result["completed"] is False
|
||||
assert result["partial"] is True
|
||||
assert (result["failure_reason"], result["failure_retryable"]) == ("truncated", True)
|
||||
assert "Repetition" in (result["final_response"] or "")
|
||||
assert not any(
|
||||
isinstance(m, dict) and m.get("content") == echo
|
||||
for m in result["messages"]
|
||||
)
|
||||
assert loop_agent.client.chat.completions.create.call_count == 1
|
||||
|
||||
def test_legit_truncation_still_continues(self, loop_agent):
|
||||
# Ordinary short truncated fragments still get continuation retries.
|
||||
loop_agent.client.chat.completions.create.side_effect = [
|
||||
_stub("part one "), _stub("part two "),
|
||||
_stub("part three "), _stub("part four."),
|
||||
_response("part one "), _response("part two "),
|
||||
_response("part three "), _response("part four."),
|
||||
]
|
||||
|
||||
result = _run(loop_agent, "write me a long report")
|
||||
|
||||
Reference in New Issue
Block a user