hermes-agent/gateway/stream_consumer_fallback.py

"""Fallback delivery for GatewayStreamConsumer: continuation sends after edits
stop working, chunking, cursor cleanup, commentary and silence retraction."""

from __future__ import annotations

import asyncio
import contextlib
import logging
from typing import Any, Callable, Optional

from gateway.platforms.base import BasePlatformAdapter as _BasePlatformAdapter
from gateway.stream_consumer_fences import ensure_closed_code_fences

logger = logging.getLogger("gateway.stream_consumer")


class StreamFallbackMixin:
    """Non-streaming delivery paths used once progressive edits fail or the turn ends oddly."""

    async def _send_new_chunk(self, text: str, reply_to_id: Optional[str], *,
                              final: bool = False) -> Optional[str]:
        """Send a new chunk threaded to ``reply_to_id``; returns the new message_id."""
        text = self._clean_for_display(text)
        if not text.strip():
            return reply_to_id
        try:
            result = await self.adapter.send(
                chat_id=self.chat_id, content=text, reply_to=reply_to_id,
                metadata=self._metadata_for_send(final=final, expect_edits=not final))
            if not (result.success and result.message_id):
                self._edit_supported = False
                return reply_to_id
            self._message_id = str(result.message_id)
            self._track_preview_ids_from_result(result)
            self._already_sent = True
            self._last_sent_text = text
            self._notify_new_message()
            return str(result.message_id)
        except Exception as e:
            logger.error("Stream send chunk error: %s", e)
            return reply_to_id

    def _visible_prefix(self) -> str:
        """Return the visible text already shown in the streamed message."""
        prefix = self._last_sent_text or ""
        if self.cfg.cursor and prefix.endswith(self.cfg.cursor):
            prefix = prefix[:-len(self.cfg.cursor)]
        return self._clean_for_display(prefix)

    def _continuation_text(self, final_text: str) -> str:
        """Return only the part of final_text the user has not already seen."""
        prefix = self._fallback_prefix or self._visible_prefix()
        if prefix and final_text.startswith(prefix):
            return final_text[len(prefix):].lstrip()
        return final_text

    @staticmethod
    def _split_text_chunks(text: str, limit: int, len_fn: "Callable[[str], int]" = len,
                           ) -> list[str]:
        """Split text for fallback sends: newline-preferred, fence-balanced across chunks."""
        from gateway.platforms.helpers import split_text_fence_aware
        return split_text_fence_aware(text, limit, len_fn, prefer_paragraphs=False,
                                      balance_fences=True)

    def _truncate_for_stream(self, text: str, limit: int, len_fn: "Callable[[str], int]",
                             ) -> list[str]:
        """Split via the adapter's canonical truncate_message (platform-specific rules);
        non-base test doubles / legacy adapters keep the two-argument call shape."""
        truncate = getattr(self.adapter, "truncate_message", None)
        if not callable(truncate):
            return self._split_text_chunks(text, limit, len_fn)
        if isinstance(self.adapter, _BasePlatformAdapter):
            chunks = truncate(text, limit, len_fn=len_fn)
        else:
            chunks = truncate(text, limit)
        if not isinstance(chunks, (list, tuple)) or not all(isinstance(c, str) for c in chunks):
            return self._split_text_chunks(text, limit, len_fn)
        return list(chunks)

    async def _send_fallback_final(self, text: str) -> None:
        """Send the final continuation after streaming edits stop working (one flood retry
        per chunk)."""
        if getattr(self, "_egress_declined", False):
            # The connector refused this destination earlier in the run. The
            # whole point of this path is to deliver the unseen tail as a NEW
            # message, which is exactly the re-addressing the egress guard
            # exists to stop.
            logger.warning(
                "suppressing the fallback continuation: the connector already "
                "declined this destination for this run"
            )
            return
        # Balance fences BEFORE computing the continuation so the closing fence
        # reaches the user even when only the tail is delivered.
        final_text = ensure_closed_code_fences(self._clean_for_display(text))
        continuation = self._continuation_text(final_text)
        self._fallback_final_send = False
        if not continuation.strip():
            continuation = await self._fallback_when_nothing_unseen(final_text)
            if continuation is None:
                return

        _len_fn, raw_limit = self._fallback_len_budget()
        chunks = self._split_text_chunks(continuation, max(500, raw_limit - 100), len_fn=_len_fn)

        stale_message_id = self._message_id  # partial message to clean up
        last_message_id: Optional[str] = None
        last_successful_chunk = ""
        sent_any_chunk = False
        for chunk in chunks:
            result = await self._send_with_flood_retry(
                content=chunk, retry_log="Flood control on fallback send, retrying in %.1fs")
            if not result or not result.success:
                # Partial continuation landed: do NOT set _final_response_sent (the
                # gateway must still deliver the full answer); _already_sent only
                # prevents a duplicate of the partial.  Nothing landed: let the
                # gateway final send try once more.
                self._already_sent = sent_any_chunk
                self._message_id = last_message_id
                self._last_sent_text = last_successful_chunk
                self._fallback_prefix = ""
                return
            sent_any_chunk = True
            last_successful_chunk = chunk
            last_message_id = result.message_id or last_message_id
            self._notify_new_message()

        # Best-effort delete of the frozen partial — ONLY when the FULL final was
        # re-sent.  If only the missing tail went out, the partial IS the head of
        # the answer ("sent only the second half" symptom).
        if (stale_message_id and stale_message_id != last_message_id
                and not self._fallback_preserve_partial_messages and continuation == final_text):
            await self._delete_previews([stale_message_id], label="Fallback partial",
                                        skip_sentinel=False)

        self._message_id = last_message_id
        self._already_sent = True
        # Recorder substitutes the unsplit ledger on a split turn.
        self._mark_final_delivered(record=final_text)
        self._last_sent_text = chunks[-1]
        self._fallback_prefix = ""
        self._fallback_preserve_partial_messages = False

    async def _fallback_when_nothing_unseen(self, final_text: str) -> Optional[str]:
        """Fallback entered but the visible prefix already covers ``final_text``: returns the
        continuation to send (the whole final when the prefix is from a *previous* segment)
        or None when the turn is settled here."""
        visible = self._visible_prefix()
        # Telegram clients can lose (part of) a streamed preview after a failed
        # final edit, so opt-in adapters commit a fresh final send.
        if (final_text.strip() and final_text == visible
                and getattr(self.adapter, "RESEND_FINAL_ON_EMPTY_STREAM_FALLBACK", False) is True):
            delivery = await self._send_empty_fallback_final(final_text)
            if delivery == "delivered":
                return None
            self._already_sent = True
            self._fallback_prefix = ""
            self._fallback_preserve_partial_messages = False
            # "ambiguous" (timeout: Telegram may have accepted) and "preview" (flood:
            # the complete ACKed preview is authoritative) keep dup suppression;
            # "failed" lets the gateway perform its normal final send.
            self._final_content_delivered = delivery in {"ambiguous", "preview"}
            if delivery == "preview":
                # This branch is only reached when the ACKed preview already shows the complete final text
                # (final_text == _visible_prefix()), so record it as the turn-final payload: the gateway's
                # reconciliation then confirms delivery instead of re-sending a second bubble next to the
                # never-deleted preview (#71047 Problem B).
                self._record_turn_final_payload(final_text)
            elif delivery == "ambiguous":
                self._delivery_ambiguous = True
            else:
                self._final_response_sent = False
            return None
        # The prefix may be from a *previous* segment (before a tool boundary),
        # wrongly reading as "already shown" — send final_text as-is.
        if final_text.strip() and final_text != visible:
            return final_text
        # Best-effort strip of a cursor left stuck by the edit failure.
        if (self._message_id and self._last_sent_text and self.cfg.cursor
                and self._last_sent_text.endswith(self.cfg.cursor)):
            clean_text = self._last_sent_text[:-len(self.cfg.cursor)]
            with contextlib.suppress(Exception):
                result = await self._edit_message(message_id=self._message_id, content=clean_text)
                if result.success:
                    self._last_sent_text = clean_text
        self._already_sent = True
        # Recorder substitutes the full ledger on a split turn.
        self._mark_final_delivered(record=final_text)
        return None

    def _fallback_len_budget(self) -> "tuple[Callable[[str], int], int]":
        """(len_fn, raw_limit) for fallback chunking — per-chat cap/unit on base adapters."""
        raw_limit = getattr(self.adapter, "MAX_MESSAGE_LENGTH", 4096)
        _len_fn: "Callable[[str], int]" = len
        if isinstance(self.adapter, _BasePlatformAdapter):
            _len_fn = self.adapter.message_len_fn
            try:  # per-chat cap/unit (relay adapter fronting N platforms)
                raw_limit = self.adapter.max_message_length_for_chat(self.chat_id)
                _len_fn = self.adapter.message_len_fn_for_chat(self.chat_id)
            except Exception as e:
                logger.debug("per-chat limit resolution failed: %s", e)
        return _len_fn, raw_limit

    async def _send_with_flood_retry(self, *, content: str, retry_log: str, reply_to=None):
        """adapter.send(final metadata) with ONE bounded flood retry; returns the last
        SendResult.  Exceptions propagate (callers decide whether a raise is "ambiguous")."""
        kwargs = dict(chat_id=self.chat_id, content=content,
                      metadata=self._metadata_for_send(final=True))
        if reply_to is not None:
            kwargs["reply_to"] = reply_to
        result = None
        for attempt in range(2):
            result = await self.adapter.send(**kwargs)
            if getattr(result, "success", False):
                break
            retry_delay = self._fallback_flood_retry_delay(result)
            if attempt or retry_delay is None:
                break  # non-flood error, long flood wait, or second failure
            logger.debug(retry_log, retry_delay)
            await asyncio.sleep(retry_delay)
        return result

    async def _send_empty_fallback_final(self, final_text: str) -> str:
        """Commit a completed answer after Telegram finalization fails: "delivered", "failed"
        (gateway may retry), "ambiguous" (a timeout may have landed) or "preview" (flood
        control; the complete preview is authoritative)."""
        # Segment-scoped only: never delete an earlier finalized preamble.
        stale_ids = self._stale_preview_ids(segment_only=True)
        try:
            result = await self._send_with_flood_retry(
                content=final_text, reply_to=self._initial_reply_to_id,
                retry_log="Flood control on empty fallback final send; retrying in %.1fs")
        except Exception as exc:
            logger.debug("Empty fallback final send failed: %s", exc)
            return "ambiguous" if self._send_failure_may_have_delivered(exc) else "failed"
        if not getattr(result, "success", False):
            if self._is_flood_error(result):
                return "preview"
            return "ambiguous" if self._send_failure_may_have_delivered(result) else "failed"

        new_message_id = getattr(result, "message_id", None)
        # Telegram reports delete failure by returning False; the flood window that
        # broke the finalize can reject this too — one bounded retry.
        await self._delete_previews(stale_ids, skip=new_message_id, label="Empty fallback",
                                    retry_on_false=True)
        self._segment_preview_message_ids = set()
        self._message_id = new_message_id or "__no_edit__"
        self._already_sent = True
        self._mark_final_delivered()
        # Record VERBATIM, not via _record_turn_final_payload: the sealed previews
        # were just deleted, so the ledger (still holding sealed heads) would claim
        # delivery for text this path removed.
        self._delivered_final_text = self._display_payload(final_text)
        self._last_sent_text = final_text
        self._fallback_prefix = ""
        self._fallback_preserve_partial_messages = False
        self._notify_new_message()
        return "delivered"

    @staticmethod
    def _send_failure_may_have_delivered(result_or_exc: Any) -> bool:
        """Return True for timeout failures where retrying may duplicate."""
        if getattr(result_or_exc, "retryable", None) is True:
            return False
        error = str(getattr(result_or_exc, "error", None) or result_or_exc).lower()
        name = result_or_exc.__class__.__name__.lower()
        return "timeout" in error or "timed out" in error or "timeout" in name

    def _fallback_flood_retry_delay(self, result: Any) -> float | None:
        """Return a bounded retry delay for a fallback send, if safe to retry."""
        if not self._is_flood_error(result):
            return None
        try:
            delay = float(getattr(result, "retry_after", None) or 3.0)
        except (TypeError, ValueError):
            delay = 3.0
        if delay > self._max_fallback_flood_retry_seconds:
            logger.debug("Flood control requests %.1fs; leaving final delivery to the gateway",
                         delay)
            return None
        return max(0.0, delay)

    def _is_flood_error(self, result) -> bool:
        """Check if a SendResult failure is due to flood control / rate limiting."""
        err_lower = (getattr(result, "error", "") or "").lower()
        return "flood" in err_lower or "retry after" in err_lower or "rate" in err_lower

    async def _flush_segment_tail_on_edit_failure(self) -> None:
        """Before a segment reset, send the unseen tail as a new message (and best-effort
        strip the stuck cursor from the partial)."""
        if not self._fallback_final_send:
            await self._try_strip_cursor()
        visible = self._fallback_prefix or self._visible_prefix()
        tail = self._accumulated
        if visible and tail.startswith(visible):
            tail = tail[len(visible):].lstrip()
        tail = self._clean_for_display(tail)
        if not tail.strip():
            return
        try:
            # Interim: must never seal a native stream (see _send_commentary).
            _md = dict(self.metadata) if self.metadata else {}
            _md["_interim_send"] = True
            result = await self.adapter.send(chat_id=self.chat_id, content=tail, metadata=_md)
            if result.success:
                self._already_sent = True
        except Exception as e:
            logger.error("Segment-break tail flush error: %s", e)

    async def _try_strip_cursor(self) -> None:
        """Best-effort edit removing a stuck cursor when entering fallback mode."""
        prefix = self._visible_prefix()
        if not self._has_real_preview() or not prefix.strip():
            return
        with contextlib.suppress(Exception):  # never block the fallback path
            result = await self._edit_message(message_id=self._message_id, content=prefix)
            if getattr(result, "success", False):
                self._last_sent_text = prefix

    async def _send_commentary(self, text: str) -> bool:
        """Send a completed interim assistant commentary message."""
        text = self._clean_for_display(text)
        if not text.strip():
            return False
        try:
            # Interim: a stream-is-the-message adapter's seal-interception must not
            # turn this into draft(final=true), which would seal the live stream
            # with interim text and orphan the true final.
            _md = self._metadata_for_send(final=False) or {}
            _md["_interim_send"] = True
            # reply_to only for reply-anchored threading; Discord/Telegram use
            # thread_id metadata and reply_to on every commentary is spam.
            _plat = getattr(getattr(self.adapter, "platform", None), "value", None)
            _platform_name = str(_plat or getattr(self.adapter, "name", "")).lower()
            _needs_reply_anchor = _platform_name in ("buzz", "slack", "mattermost", "feishu")
            result = await self.adapter.send(
                chat_id=self.chat_id, content=text,
                reply_to=self._initial_reply_to_id if _needs_reply_anchor else None, metadata=_md)
            # Do NOT set _already_sent: commentary is interim, and the flag would
            # suppress the real final after multiple tool calls.
            if result.success:
                self._notify_new_message()
                # Lets run.py confirm whether an interim send carried the final.
                # Record the exact delivered text so run.py can confirm whether an interim "preview"
                # actually carried the final response, vs. unrelated commentary delivered during a session
                # split (#14238).
                self._delivered_commentary_texts.append(text)
            return result.success
        except Exception as e:
            logger.error("Commentary send error: %s", e)
            return False

    def _raw_message_limit(self) -> int:
        """Per-chat length budget (``message_len_fn`` units) before overflow splits; rich
        adapters may raise it via ``streaming_overflow_limit`` so a reply that fits one rich
        message isn't fragmented at the edit limit."""
        base = getattr(self.adapter, "MAX_MESSAGE_LENGTH", 4096)
        # isinstance gate keeps MagicMock adapters (mock attrs, not ints) on base.
        if isinstance(self.adapter, _BasePlatformAdapter):
            try:
                base = self.adapter.max_message_length_for_chat(self.chat_id)
            except Exception as e:
                logger.debug("max_message_length_for_chat failed: %s", e)
            try:
                cap = self.adapter.streaming_overflow_limit()
            except Exception as e:
                logger.debug("streaming_overflow_limit check failed: %s", e)
                cap = None
            if isinstance(cap, int) and cap > base:
                return cap
        return base

    # Fresh send carried exactly ``text`` — record it so the gateway can reconcile the flag against the
    # completed response (#71643/#95382 content-vs-flag contract).
    async def _suppress_silence_marker(self) -> None:
        """Retract any streamed preview when the final reply is a bare silence marker.  Flags
        stay False: the gateway's whole-response filter turns the marker into "" so no
        fallback send happens either."""
        # A native-stream bubble isn't a deletable message — close an open one
        # (e.g. from an eager re-seed) with an empty finalize so it doesn't hang.
        if self._native_stream_opened:
            await self._close_empty_native_bubble("Silence-marker native stream close failed: %s")

        await self._delete_previews(self._stale_preview_ids(), label="Silence-marker")
        self._preview_message_ids = set()
        self._message_id = None
        self._accumulated = self._stream_ledger = self._last_sent_text = ""
        self._already_sent = False
        self._clear_turn_final_flags()
        logger.info("Suppressed streamed intentional-silence marker (chat=%s)", self.chat_id)