diff --git a/agent/repetition_guard.py b/agent/repetition_guard.py index 0729e5742a..e1088f35c9 100644 --- a/agent/repetition_guard.py +++ b/agent/repetition_guard.py @@ -40,6 +40,11 @@ _MAX_ANCHOR_MATCHES = 8 # scan, but every line is distinct; a loop re-emits the same line(s). _RUNAWAY_DISTINCT_LINE_RATIO = 0.5 +# The finish_reason="stop" path discards a COMPLETED answer, so it only aborts at runaway scale: +# real stop-path loops (#100716) run 80k-350k chars, while asked-for repeats ("say X 50 times", +# identical table rows, templated YAML) stay in the low KB and must be delivered. +STOP_PATH_MIN_CHARS = 16_000 + def is_repetition_dominated(text: str) -> bool: """True when a contiguous run of at least five exact repetitions covers at least half diff --git a/agent/turn_final_response.py b/agent/turn_final_response.py index 11ebe06a44..6ebc40503d 100644 --- a/agent/turn_final_response.py +++ b/agent/turn_final_response.py @@ -12,20 +12,17 @@ import logging from typing import Any, Dict, Optional from agent.message_metadata import append_message -from agent.repetition_guard import is_repetition_dominated +from agent.repetition_guard import STOP_PATH_MIN_CHARS, is_runaway_repetition +from agent.turn_failure_copy import stamp_failure from agent.turn_empty_response import recover_empty_response from agent.turn_stop_gates import apply_stop_gates -from agent.turn_truncation import partial_result +from agent.turn_truncation import partial_result, repetition_copy -_REPETITION_STOPPED = ( - "šŸ” Response dominated by repeated text ā€” stopping before delivery.", - "āš ļø **Response Stopped ā€” Repetition Detected**\n\nThe model fell into a repetition loop while " - "writing this response, so the repeated output was discarded.\n\n" - "ā†’ Switch to a different model with `/model`\n" - "ā†’ Or resend your message (your conversation history is preserved)", - "Model output entered a repetition loop; refusing to return a degenerate response.", +_REPETITION_STOPPED = repetition_copy( + "before delivery", + "so the repeated output was discarded.", + "; refusing to return a", ) - logger = logging.getLogger("agent.conversation_loop") # Ephemeral retry scaffolding rows popped before the final answer becomes durable. @@ -62,7 +59,7 @@ def finish_text_response( _preflight_compression_blocked: Any, codex_ack_continuations: Any, truncated_response_parts: Any, length_continue_retries: Any, _pending_verification_response: Any, _pending_verification_response_previewed: Any, - effective_task_id: Any = None, + effective_task_id: Any, ) -> FinalResponseVerdict: """Finish (or defer) a text-only assistant response in the original guard order. Every continuation path sets ``final_response = None`` so an acknowledgment never suppresses @@ -258,12 +255,20 @@ def finish_text_response( # A provider may end a degenerate loop normally with finish_reason="stop" instead of # exhausting its output cap (#100716). Check every completed visible text response before # any verify/kanban interim emission or durable transcript write. - if final_response and is_repetition_dominated(final_response): + # Runaway scale and shape only: a completed answer the user asked to be repetitive is + # delivered, unlike a length-truncated fragment that burned the whole budget. + if ( + final_response + and len(final_response) >= STOP_PATH_MIN_CHARS + and is_runaway_repetition(final_response) + ): line, user_response, error = _REPETITION_STOPPED - agent._vprint(f"{agent.log_prefix}{line}", force=True) + agent._vprint(f"{agent.log_prefix}{line}", force=True, diagnostic=True) agent._cleanup_task_resources(effective_task_id) agent._persist_session(messages, conversation_history) - return _verdict("return", partial_result(messages, api_call_count, user_response, error)) + return _verdict("return", stamp_failure( + partial_result(messages, api_call_count, user_response, error), "truncated", True, + )) final_msg = agent._build_assistant_message(assistant_message, finish_reason) if _promoted: diff --git a/agent/turn_truncation.py b/agent/turn_truncation.py index fbecb43bf3..7a1dc6487c 100644 --- a/agent/turn_truncation.py +++ b/agent/turn_truncation.py @@ -51,14 +51,23 @@ _THINKING_EXHAUSTED = ( "Model used all output tokens on reasoning with none left " "for the response. Try lowering reasoning effort or increasing max_tokens.", ) -_REPETITION_DOMINATED = ( - "šŸ” Response dominated by repeated text ā€” stopping instead of continuing a degenerate response.", - "āš ļø **Response Stopped ā€” Repetition Detected**\n\nThe model fell into a repetition loop while " - "writing this response, so continuing would only produce more repeated text. The partial response " - "was discarded.\n\nā†’ Switch to a different model with `/model`\n" - "ā†’ Or resend your message (your conversation history is preserved)", - "Model output entered a repetition loop and was truncated mid-loop; refusing to continue a " - "degenerate response.", + +def repetition_copy(stopping: str, outcome: str, refusal: str) -> Tuple[str, str, str]: + """(log line, user copy, error) for a repetition-dominated abort; only the clauses naming + where the turn stopped differ between the length path and the stop path.""" + return ( + f"šŸ” Response dominated by repeated text ā€” stopping {stopping}.", + "āš ļø **Response Stopped ā€” Repetition Detected**\n\nThe model fell into a repetition loop while " + f"writing this response, {outcome}\n\nā†’ Switch to a different model with `/model`\n" + "ā†’ Or resend your message (your conversation history is preserved)", + f"Model output entered a repetition loop{refusal} degenerate response.", + ) + + +_REPETITION_DOMINATED = repetition_copy( + "instead of continuing a degenerate response", + "so continuing would only produce more repeated text. The partial response was discarded.", + " and was truncated mid-loop; refusing to continue a", ) _CEILING_NO_TEXT = ( "āš ļø **No visible answer was produced.** The model hit its output-token limit on every " diff --git a/tests/agent/test_continuation_repetition_guard.py b/tests/agent/test_continuation_repetition_guard.py index 6a2983f11d..e1fac16035 100644 --- a/tests/agent/test_continuation_repetition_guard.py +++ b/tests/agent/test_continuation_repetition_guard.py @@ -12,6 +12,7 @@ from unittest.mock import MagicMock, patch import pytest +from agent.repetition_guard import STOP_PATH_MIN_CHARS from hermes_constants import FINISH_REASON_LENGTH, PARTIAL_STREAM_STUB_ID # The exact sentence from the #86581 incident. @@ -62,10 +63,6 @@ def _response( ) -def _stub(content): - return _response(content) - - def _run(agent, message): with ( patch.object(agent, "_persist_session"), @@ -78,7 +75,7 @@ def _run(agent, message): class TestContinuationRepetitionGuard: def test_repetition_dominated_truncation_aborts(self, loop_agent): echo = _INCIDENT_ECHO * 2000 - loop_agent.client.chat.completions.create.side_effect = [_stub(echo)] + loop_agent.client.chat.completions.create.side_effect = [_response(echo)] result = _run(loop_agent, "write me a long report") @@ -93,33 +90,45 @@ class TestContinuationRepetitionGuard: # Exactly one API call ā€” no continuation was attempted. assert loop_agent.client.chat.completions.create.call_count == 1 - def test_repetition_dominated_stop_response_aborts(self, loop_agent): - paragraph = ( - "A long paragraph that should never be delivered hundreds of times " - "when a model enters a repetition loop.\n" - "The second line makes this a multiline repeating unit.\n" - ) - echo = paragraph * 500 + @pytest.mark.parametrize("requested_repeat", [False, True], ids=["loop", "repeat-on-request"]) + def test_repetition_dominated_stop_response_aborts(self, loop_agent, requested_repeat): + if requested_repeat: + # Asked-for repetition (identical lines, ~3.6k chars) is below runaway scale: a + # completed answer must be delivered, not discarded. + echo = "Hello world, this is a sentence the user asked me to repeat many times.\n" * 50 + else: + paragraph = ( + "A long paragraph that should never be delivered hundreds of times " + "when a model enters a repetition loop.\n" + "The second line makes this a multiline repeating unit.\n" + ) + echo = paragraph * 500 + assert len(echo) >= STOP_PATH_MIN_CHARS loop_agent.client.chat.completions.create.side_effect = [ _response(echo, finish_reason="stop", response_id="completed-response") ] result = _run(loop_agent, "write me a long report") + assert loop_agent.client.chat.completions.create.call_count == 1 + if requested_repeat: + assert result["completed"] is True + assert result["final_response"] == echo.strip() + return assert result["completed"] is False assert result["partial"] is True + assert (result["failure_reason"], result["failure_retryable"]) == ("truncated", True) assert "Repetition" in (result["final_response"] or "") assert not any( isinstance(m, dict) and m.get("content") == echo for m in result["messages"] ) - assert loop_agent.client.chat.completions.create.call_count == 1 def test_legit_truncation_still_continues(self, loop_agent): # Ordinary short truncated fragments still get continuation retries. loop_agent.client.chat.completions.create.side_effect = [ - _stub("part one "), _stub("part two "), - _stub("part three "), _stub("part four."), + _response("part one "), _response("part two "), + _response("part three "), _response("part four."), ] result = _run(loop_agent, "write me a long report")