0e805041d5
The dispatch guard now uses _COMPRESSION_MARKER_ARTIFACT_RE, so the comment on _COMPRESSION_MARKER_RE no longer describes its job: only renderer tests use it to assert a complete first sentence. Say that, and keep the editability reasoning on the artifact regex alone. The count fragment was typed three times; a single _MARKER_COUNT_RE keeps the matchers derived rather than re-typed, as the module docstring promises. No behaviour change.
87 lines
4.1 KiB
Python
87 lines
4.1 KiB
Python
"""The model-visible compression/elision marker and its matcher.
|
|
|
|
Tool-call arguments are no longer rewritten by the compressor, but legacy sessions may still
|
|
carry this marker inside replayed tool-call arguments. Dependency-free leaf shared by the
|
|
elision renderers (``agent.context_compressor``) and the dispatch-boundary detector
|
|
(``agent.tool_dispatch_helpers``), so the matcher is derived from the template instead of
|
|
re-typing its wording.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
# #83714 — this text lands inside the model's OWN replayed tool call, so it must not read like
|
|
# something the model would write itself: the bare "...[truncated]" it replaced was imitated into
|
|
# new calls and written to disk. Non-prose delimiters, an explicit "not original content"
|
|
# disclaimer, and per-instance counts keep a copied marker visibly wrong; the counts also make a
|
|
# verbatim copy stale, which is why the marker must never be re-applied.
|
|
_COMPRESSION_MARKER_PREFIX = "⟪HERMES-CONTEXT-COMPRESSION:"
|
|
_COMPRESSION_MARKER_TEMPLATE = (
|
|
_COMPRESSION_MARKER_PREFIX
|
|
+ " {omitted:,} of {total:,} chars omitted here by Hermes's context compressor. "
|
|
"This is NOT part of the original tool call and must never be reproduced in new "
|
|
"output — always write full, untruncated content.⟫"
|
|
)
|
|
|
|
# A rendered omitted/total count inside a minted marker.
|
|
_MARKER_COUNT_RE = r"\d[\d,]*"
|
|
|
|
# Full rendered first sentence of a minted marker; renderer tests assert this shape
|
|
# (the dispatch guard uses _COMPRESSION_MARKER_ARTIFACT_RE below).
|
|
_COMPRESSION_MARKER_RE = re.compile(
|
|
re.escape(_COMPRESSION_MARKER_TEMPLATE.split(". ", 1)[0] + ".")
|
|
.replace(re.escape("{omitted:,}"), _MARKER_COUNT_RE)
|
|
.replace(re.escape("{total:,}"), _MARKER_COUNT_RE)
|
|
)
|
|
|
|
# A marker cut by a later boundary may never reach the fixed sentence above. The
|
|
# dispatch guard therefore recognizes a minted marker as soon as its first numeric
|
|
# count is present. Requiring that rendered count keeps the bare prefix/template
|
|
# editable in source and documentation.
|
|
_COMPRESSION_MARKER_ARTIFACT_RE = re.compile(
|
|
re.escape(_COMPRESSION_MARKER_PREFIX) + r"\s+" + _MARKER_COUNT_RE
|
|
)
|
|
|
|
# #121548 — every OTHER model-visible elision (turn text, summaries, skill bodies, diagnostics)
|
|
# mints the args marker's first sentence only (so the dispatch-boundary artifact
|
|
# matcher catches a copy from any renderer) and fits small caps like clarify's 199.
|
|
_ELISION_MARKER_TEMPLATE = _COMPRESSION_MARKER_TEMPLATE.split(". ", 1)[0] + ".⟫"
|
|
|
|
|
|
def _elision_marker(omitted: int, total: int) -> str:
|
|
"""Render the non-imitable elision marker with per-instance byte counts."""
|
|
return _ELISION_MARKER_TEMPLATE.format(omitted=omitted, total=total)
|
|
|
|
|
|
# Widest marker for any text under ~1 TB: callers with a small leftover budget skip the item
|
|
# when the budget cannot hold the marker plus content, instead of emitting a marker-only line.
|
|
ELISION_MARKER_MAX_LEN = len(_elision_marker(omitted=10**12 - 1, total=10**12 - 1))
|
|
|
|
|
|
def elide(text: str, limit: int) -> str:
|
|
"""Cap ``text`` at ``limit`` chars, marking the elision with ``_elision_marker``.
|
|
|
|
Every model-visible renderer must truncate through here (#121548). ``text`` is
|
|
returned unchanged when it already fits. Otherwise the result is ``head + marker``
|
|
with accurate omitted/total counts and never exceeds ``limit`` (sized against the
|
|
widest rendering the counts can take), unless ``limit`` cannot hold the marker at
|
|
all, in which case the marker alone is returned.
|
|
"""
|
|
if len(text) <= limit:
|
|
return text
|
|
total = len(text)
|
|
head_len = limit - len(_elision_marker(omitted=total, total=total))
|
|
if head_len <= 0:
|
|
return _elision_marker(omitted=total, total=total)
|
|
kept = text[:head_len].rstrip()
|
|
return kept + _elision_marker(omitted=total - len(kept), total=total)
|
|
|
|
|
|
def elide_middle(text: str, head: int, tail: int) -> str:
|
|
"""Keep ``head`` chars from the start and ``tail`` from the end, eliding the middle."""
|
|
if len(text) <= head + tail:
|
|
return text
|
|
marker = _elision_marker(omitted=len(text) - head - tail, total=len(text))
|
|
return text[:head] + marker + text[len(text) - tail:]
|