44982309b8
build_tool_preview -> _PREVIEW_BUILDERS per-tool table; git @refs -> _GIT_REFERENCE_ARGS; context_breakdown _skills_block/_append_overflow dedupe; prune_pre_checkpoint_items summary retention folded into one closure; build_skill_invocation_message reuses _render_skill_block; ruff SIM collapses; restored two compacted cache-policy invariant comments.
515 lines
19 KiB
Python
515 lines
19 KiB
Python
"""Anthropic prompt caching strategy — pure functions, no AIAgent dependency.
|
|
|
|
Default layout: 4 cache_control breakpoints — the static system prefix, the end
|
|
of the system prompt, and the last 2 non-system messages. Without a static
|
|
prefix: one system breakpoint plus the last 3 messages. All markers share one
|
|
TTL (5m or 1h). This keeps intra-session caching while letting new sessions
|
|
reuse the stable system-prompt prefix.
|
|
"""
|
|
|
|
import copy
|
|
from dataclasses import dataclass
|
|
from typing import Any, Dict, List
|
|
|
|
from agent.prompt_cache_boundary import find_stable_prefix
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PromptCachePlan:
|
|
"""Request-local message and tool sections with their cache markers."""
|
|
|
|
messages: List[Dict[str, Any]]
|
|
tools: List[Dict[str, Any]]
|
|
|
|
|
|
def envelope_tool_part_cache_markers_supported(
|
|
provider: str | None, base_url: str | None
|
|
) -> bool:
|
|
"""Whether the envelope-layout route honors part-level markers on role:tool.
|
|
|
|
OpenRouter (and Nous Portal, which proxies to it) relocate a part-level
|
|
``cache_control`` onto the ``tool_result`` block during OpenAI→Anthropic
|
|
translation. LiteLLM-style proxies copy parts verbatim, so the marker lands
|
|
at ``tool_result.content[0]`` — forbidden by the Anthropic schema, a
|
|
non-retryable 400. On those routes tool messages carry no part markers and
|
|
the breakpoint budget reallocates to the nearest eligible message.
|
|
"""
|
|
from agent.agent_runtime_helpers import _is_litellm_route
|
|
|
|
return not _is_litellm_route((provider or "").strip().lower(), base_url or "")
|
|
|
|
|
|
def _apply_cache_marker(
|
|
msg: dict,
|
|
cache_marker: dict,
|
|
native_anthropic: bool = False,
|
|
tool_part_markers: bool = True,
|
|
) -> None:
|
|
"""Add cache_control to a single message, handling all format variations."""
|
|
role = msg.get("role", "")
|
|
content = msg.get("content")
|
|
|
|
if role == "tool" and native_anthropic:
|
|
# Top-level marker; the native adapter moves it inside tool_result.
|
|
msg["cache_control"] = cache_marker
|
|
return
|
|
if role == "tool" and not tool_part_markers:
|
|
# LiteLLM-style envelope: a part marker becomes
|
|
# tool_result.content[0].cache_control → non-retryable 400.
|
|
return
|
|
|
|
if content is None or content == "":
|
|
# Envelope layout: OpenRouter rejects top-level cache_control on
|
|
# role:tool (silent hang), and ignores it on empty assistant turns
|
|
# (pure tool_calls) — neither has a content part to carry it.
|
|
if role in ("tool", "assistant") and not native_anthropic:
|
|
return
|
|
msg["cache_control"] = cache_marker
|
|
return
|
|
|
|
if isinstance(content, str):
|
|
if role == "user":
|
|
stable_prefix = find_stable_prefix(content)
|
|
if stable_prefix is not None:
|
|
suffix = content[len(stable_prefix):]
|
|
if suffix.strip():
|
|
# Builder-declared boundary: the scaffold carries the
|
|
# breakpoint and the volatile tail rides unmarked, so a
|
|
# changed ticket ID/timestamp no longer invalidates the
|
|
# skill body. Request-local only — the stored message
|
|
# stays a plain string.
|
|
msg["content"] = [
|
|
{"type": "text", "text": stable_prefix, "cache_control": cache_marker},
|
|
{"type": "text", "text": suffix},
|
|
]
|
|
return
|
|
msg["content"] = [
|
|
{"type": "text", "text": content, "cache_control": cache_marker}
|
|
]
|
|
return
|
|
|
|
if isinstance(content, list) and content:
|
|
last = content[-1]
|
|
if isinstance(last, dict):
|
|
last["cache_control"] = cache_marker
|
|
|
|
|
|
def _can_carry_marker(
|
|
msg: dict, native_anthropic: bool, tool_part_markers: bool = True
|
|
) -> bool:
|
|
"""True if a marker on this message is actually honored by the provider.
|
|
|
|
Native Anthropic honors every message (the adapter relocates top-level
|
|
markers). The envelope layout only honors markers inside content parts, so
|
|
empty-content messages would waste one of the four breakpoints; with
|
|
``tool_part_markers=False`` (LiteLLM-style routes) every role:tool message
|
|
is excluded too, since its part marker would be rejected with a 400.
|
|
Must agree with :func:`_apply_cache_marker`, which marks only the LAST part.
|
|
"""
|
|
if native_anthropic:
|
|
return True
|
|
if msg.get("role") == "tool" and not tool_part_markers:
|
|
return False
|
|
content = msg.get("content")
|
|
if content is None or content == "":
|
|
return False
|
|
if isinstance(content, list):
|
|
# Mirrors _apply_cache_marker (marks only the LAST part): a list whose
|
|
# last element isn't a dict cannot receive a marker.
|
|
return bool(content) and isinstance(content[-1], dict)
|
|
return isinstance(content, str)
|
|
|
|
|
|
def _build_marker(ttl: str) -> Dict[str, str]:
|
|
"""Build a cache_control marker dict for the given TTL ('5m' or '1h')."""
|
|
marker: Dict[str, str] = {"type": "ephemeral"}
|
|
if ttl == "1h":
|
|
marker["ttl"] = "1h"
|
|
return marker
|
|
|
|
|
|
# Alibaba-family providers (Qwen routes): documented five-minute context cache,
|
|
# Anthropic 1h tier rejected. Shared with
|
|
# agent_runtime_helpers.anthropic_prompt_cache_policy so the cache-policy
|
|
# opt-in and the TTL clamp never desync. Do NOT narrow this set to extend a
|
|
# TTL — it also drives the marker-layout opt-in, so narrowing DISABLES caching.
|
|
ALIBABA_FAMILY_PROVIDERS = frozenset({
|
|
"opencode",
|
|
"opencode-go",
|
|
"opencode-zen",
|
|
"alibaba",
|
|
})
|
|
|
|
# 1h-tier ALLOW-list: only routes wire-measured to retain a 1h marker (delayed
|
|
# read past 5 minutes with no intervening call — an intervening read renews the
|
|
# window and masks expiry). Other opencode routes stay clamped because they are
|
|
# UNMEASURED, not known-bad. Note opencode-go labels every write
|
|
# `ephemeral_5m_input_tokens` regardless of requested ttl; that label is not
|
|
# evidence of the retention window.
|
|
MEASURED_1H_PROVIDERS = frozenset({
|
|
"opencode-go",
|
|
})
|
|
|
|
# Models measured to ignore the 1h tier on a MEASURED_1H_PROVIDERS route.
|
|
# Consulted only there: the same model on its own Anthropic-compatible endpoint
|
|
# is a separate cache-eligible route and must not inherit this clamp.
|
|
NO_1H_TIER_MODELS = frozenset({
|
|
"minimax-m2.5",
|
|
})
|
|
|
|
|
|
def _flat_model(model: str) -> str:
|
|
"""Bare model id, tolerating aggregator prefixes (``vendor/model``)."""
|
|
return (model or "").strip().rsplit("/", 1)[-1].lower()
|
|
|
|
|
|
def is_qwen_model(model: str) -> bool:
|
|
"""True when ``model`` names a Qwen-family model (case-insensitive).
|
|
|
|
Shared with ``agent_runtime_helpers.anthropic_prompt_cache_policy`` so the
|
|
cache-policy opt-in and the TTL clamp never desync.
|
|
"""
|
|
return "qwen" in (model or "").lower()
|
|
|
|
|
|
def effective_cache_ttl(
|
|
ttl: str | None,
|
|
*,
|
|
model: str = "",
|
|
provider: str = "",
|
|
) -> str:
|
|
"""Clamp a requested cache TTL to what the destination route supports.
|
|
|
|
Qwen/Alibaba routes document a five-minute window and drop the ``1h``
|
|
tier, so a configured ``1h`` regresses to ``5m`` there instead of creating
|
|
a false 1h-cache expectation — except on ``MEASURED_1H_PROVIDERS``, which
|
|
keep ``1h`` minus any ``NO_1H_TIER_MODELS`` model. The measured-route check
|
|
runs BEFORE the generic Qwen clamp, which would otherwise swallow every
|
|
Qwen model on it. ``None`` resolves to ``5m``.
|
|
"""
|
|
if ttl != "1h":
|
|
return ttl or "5m"
|
|
if (provider or "").lower() in MEASURED_1H_PROVIDERS:
|
|
# Checked BEFORE the generic Qwen clamp (which would swallow every Qwen
|
|
# model on this route); the per-model denial stays nested so an
|
|
# opencode-go observation cannot reclamp the same model on another route.
|
|
return "5m" if _flat_model(model) in NO_1H_TIER_MODELS else "1h"
|
|
if is_qwen_model(model) or (provider or "").lower() in ALIBABA_FAMILY_PROVIDERS:
|
|
return "5m"
|
|
return "1h"
|
|
|
|
|
|
def _apply_system_cache_markers(
|
|
message: dict,
|
|
cache_marker: dict,
|
|
static_system_prefix: str | None,
|
|
*,
|
|
native_anthropic: bool,
|
|
mark_suffix: bool = True,
|
|
fallback_to_whole: bool = True,
|
|
) -> int:
|
|
"""Mark the static system prefix (and optionally the full prompt).
|
|
|
|
The system prompt stays one stored string; it is split only in the
|
|
outgoing request so persistence and non-Anthropic transports are
|
|
unchanged. ``mark_suffix=False`` is the tool-cache-plan layout (suffix
|
|
unmarked, its budget spent on the tools array). ``fallback_to_whole=False``
|
|
marks nothing when the prefix split is impossible. When the prompt IS the
|
|
prefix (empty/whitespace suffix) the whole message is marked as one block —
|
|
never a split with an empty text block, which Anthropic rejects.
|
|
|
|
Returns the number of markers applied (0, 1, or 2).
|
|
"""
|
|
content = message.get("content")
|
|
if (
|
|
isinstance(static_system_prefix, str)
|
|
and static_system_prefix
|
|
and isinstance(content, str)
|
|
and content.startswith(static_system_prefix)
|
|
):
|
|
suffix = content[len(static_system_prefix):]
|
|
if suffix.strip():
|
|
suffix_part: dict = {"type": "text", "text": suffix}
|
|
if mark_suffix:
|
|
suffix_part["cache_control"] = cache_marker
|
|
message["content"] = [
|
|
{"type": "text", "text": static_system_prefix, "cache_control": cache_marker},
|
|
suffix_part,
|
|
]
|
|
return 2 if mark_suffix else 1
|
|
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
|
|
return 1
|
|
|
|
if not fallback_to_whole:
|
|
return 0
|
|
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)
|
|
return 1
|
|
|
|
|
|
def strip_anthropic_cache_control(
|
|
api_messages: List[Dict[str, Any]],
|
|
) -> List[Dict[str, Any]]:
|
|
"""Remove ``cache_control`` markers and undo decoration-produced list shapes.
|
|
|
|
Used before re-decorating after a mid-turn provider failover, so the
|
|
mutated undecorated shape is preserved while markers match the new
|
|
provider's policy. Flattening back to a plain string is restricted to the
|
|
exact shapes :func:`apply_anthropic_cache_control` produces from string
|
|
content — a single text part, the two-part ``[static, volatile]`` system
|
|
split, or the two-part skill split — so the ``""``-join is provably
|
|
byte-exact; organic multi-part text and parts with extra keys keep their
|
|
structure. Marker removal is copy-on-write on part dicts: parts can alias
|
|
caller-held lists and stripping must never rewrite the stored transcript.
|
|
|
|
Mutates the top-level message dicts in place and returns the same list.
|
|
"""
|
|
for msg in api_messages:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
msg.pop("cache_control", None)
|
|
content = msg.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
# The builder-declared skill split is the only decoration that marks
|
|
# the FIRST part of a user message (list content is otherwise marked
|
|
# on the last part; the [static, volatile] split is system-only), so
|
|
# the shape alone identifies it even after the prefix registry has
|
|
# evicted the entry.
|
|
skill_split_shape = (
|
|
msg.get("role") == "user"
|
|
and len(content) == 2
|
|
and isinstance(content[0], dict)
|
|
and isinstance(content[1], dict)
|
|
and "cache_control" in content[0]
|
|
and "cache_control" not in content[1]
|
|
)
|
|
if any(isinstance(part, dict) and "cache_control" in part for part in content):
|
|
content = [
|
|
{k: v for k, v in part.items() if k != "cache_control"}
|
|
if isinstance(part, dict) and "cache_control" in part
|
|
else part
|
|
for part in content
|
|
]
|
|
msg["content"] = content
|
|
decoration_shape = content and all(
|
|
isinstance(part, dict)
|
|
and part.get("type", "text") == "text"
|
|
and isinstance(part.get("text"), str)
|
|
and set(part.keys()) <= {"type", "text"}
|
|
for part in content
|
|
) and (
|
|
len(content) == 1
|
|
or (msg.get("role") == "system" and len(content) == 2)
|
|
or skill_split_shape
|
|
)
|
|
if decoration_shape:
|
|
msg["content"] = "".join(part["text"] for part in content)
|
|
return api_messages
|
|
|
|
|
|
def strip_anthropic_tool_cache_control(tools: List[Dict[str, Any]] | None) -> List[Dict[str, Any]]:
|
|
"""Return copied tools without request-local Anthropic cache markers."""
|
|
cleaned = copy.deepcopy(tools or [])
|
|
for tool in cleaned:
|
|
if isinstance(tool, dict):
|
|
tool.pop("cache_control", None)
|
|
return cleaned
|
|
|
|
|
|
def _count_cache_markers(messages: List[Dict[str, Any]], tools: List[Dict[str, Any]]) -> int:
|
|
"""Count the wire-visible cache markers in a request-local plan."""
|
|
count = sum(
|
|
1
|
|
for message in messages
|
|
if isinstance(message, dict) and "cache_control" in message
|
|
)
|
|
count += sum(
|
|
1
|
|
for message in messages
|
|
if isinstance(message, dict) and isinstance(message.get("content"), list)
|
|
for part in message["content"]
|
|
if isinstance(part, dict) and "cache_control" in part
|
|
)
|
|
return count + sum(
|
|
1 for tool in tools if isinstance(tool, dict) and "cache_control" in tool
|
|
)
|
|
|
|
|
|
def _completed_transaction_endpoint_indexes(
|
|
messages: List[Dict[str, Any]], *, native_anthropic: bool,
|
|
) -> List[int]:
|
|
"""Select legal ends of completed tool runs and ordinary turns."""
|
|
endpoints: List[int] = []
|
|
index = 0
|
|
while index < len(messages):
|
|
message = messages[index]
|
|
if not isinstance(message, dict) or message.get("role") == "system":
|
|
index += 1
|
|
continue
|
|
|
|
if message.get("role") == "assistant" and message.get("tool_calls"):
|
|
result_start = index + 1
|
|
result_end = result_start
|
|
while result_end < len(messages):
|
|
result = messages[result_end]
|
|
if not isinstance(result, dict) or result.get("role") != "tool":
|
|
break
|
|
result_end += 1
|
|
if result_end > result_start:
|
|
endpoint = result_end - 1
|
|
if _can_carry_marker(messages[endpoint], native_anthropic):
|
|
endpoints.append(endpoint)
|
|
index = result_end
|
|
continue
|
|
|
|
if message.get("role") == "tool":
|
|
while index < len(messages):
|
|
result = messages[index]
|
|
if not isinstance(result, dict) or result.get("role") != "tool":
|
|
break
|
|
index += 1
|
|
continue
|
|
|
|
if message.get("role") == "user" and index + 1 < len(messages):
|
|
index += 1
|
|
continue
|
|
|
|
if (
|
|
message.get("role") == "assistant"
|
|
and message.get("content") in (None, "")
|
|
):
|
|
index += 1
|
|
continue
|
|
|
|
if _can_carry_marker(message, native_anthropic):
|
|
endpoints.append(index)
|
|
index += 1
|
|
return endpoints
|
|
|
|
|
|
def build_prompt_cache_plan(
|
|
api_messages: List[Dict[str, Any]],
|
|
tools: List[Dict[str, Any]] | None,
|
|
*,
|
|
cache_ttl: str = "5m",
|
|
native_anthropic: bool = False,
|
|
static_system_prefix: str | None = None,
|
|
direct_native_tool_cache: bool = False,
|
|
tool_part_markers: bool = True,
|
|
) -> PromptCachePlan:
|
|
"""Build isolated cache sections for one resolved request destination.
|
|
|
|
``tool_part_markers=False`` (LiteLLM-style envelope routes) keeps
|
|
``cache_control`` off role:tool content parts; breakpoints reallocate to
|
|
the nearest eligible non-tool message.
|
|
"""
|
|
messages = copy.deepcopy(api_messages or [])
|
|
strip_anthropic_cache_control(messages)
|
|
planned_tools = strip_anthropic_tool_cache_control(tools)
|
|
|
|
if not direct_native_tool_cache or not planned_tools:
|
|
planned_messages = apply_anthropic_cache_control(
|
|
messages,
|
|
cache_ttl=cache_ttl,
|
|
native_anthropic=native_anthropic,
|
|
static_system_prefix=static_system_prefix,
|
|
tool_part_markers=tool_part_markers,
|
|
)
|
|
return PromptCachePlan(messages=planned_messages, tools=planned_tools)
|
|
|
|
marker = _build_marker(cache_ttl)
|
|
if (
|
|
messages
|
|
and isinstance(messages[0], dict)
|
|
and messages[0].get("role") == "system"
|
|
):
|
|
# Tool-cache layout: only the static prefix carries a system-side
|
|
# marker; the volatile suffix's budget is spent on the tools array.
|
|
_apply_system_cache_markers(
|
|
messages[0],
|
|
marker,
|
|
static_system_prefix,
|
|
native_anthropic=True,
|
|
mark_suffix=False,
|
|
fallback_to_whole=False,
|
|
)
|
|
planned_tools[-1]["cache_control"] = dict(marker)
|
|
for endpoint in _completed_transaction_endpoint_indexes(
|
|
messages,
|
|
native_anthropic=True,
|
|
)[-2:]:
|
|
_apply_cache_marker(messages[endpoint], marker, native_anthropic=True)
|
|
|
|
return PromptCachePlan(messages=messages, tools=planned_tools)
|
|
|
|
|
|
def apply_anthropic_cache_control(
|
|
api_messages: List[Dict[str, Any]],
|
|
cache_ttl: str = "5m",
|
|
native_anthropic: bool = False,
|
|
static_system_prefix: str | None = None,
|
|
tool_part_markers: bool = True,
|
|
) -> List[Dict[str, Any]]:
|
|
"""Apply Anthropic cache-control markers to API messages.
|
|
|
|
With a matching ``static_system_prefix`` the prefix gets an early marker
|
|
and the full system prompt a trailing one; the remaining two markers go to
|
|
the latest cacheable non-system messages. Without it, the legacy
|
|
system-and-3 layout applies. Idempotent: pre-existing markers are stripped
|
|
from a per-message copy first, so repeated calls never accumulate past 4
|
|
markers; a shallow top-level copy suffices because
|
|
:func:`strip_anthropic_cache_control` is copy-on-write on content parts.
|
|
|
|
Returns:
|
|
Shallow copy of message list with selective deep copies of modified messages.
|
|
"""
|
|
if not api_messages:
|
|
return api_messages
|
|
|
|
messages = list(api_messages)
|
|
marker = _build_marker(cache_ttl)
|
|
|
|
for i, msg in enumerate(messages):
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
content = msg.get("content")
|
|
has_marker = "cache_control" in msg or (
|
|
isinstance(content, list)
|
|
and any(isinstance(part, dict) and "cache_control" in part for part in content)
|
|
)
|
|
if has_marker:
|
|
messages[i] = strip_anthropic_cache_control([dict(msg)])[0]
|
|
|
|
breakpoints_used = 0
|
|
|
|
if messages[0].get("role") == "system":
|
|
messages[0] = copy.deepcopy(messages[0])
|
|
breakpoints_used = _apply_system_cache_markers(
|
|
messages[0],
|
|
marker,
|
|
static_system_prefix,
|
|
native_anthropic=native_anthropic,
|
|
)
|
|
|
|
remaining = 4 - breakpoints_used
|
|
non_sys = [
|
|
i
|
|
for i in range(len(messages))
|
|
if messages[i].get("role") != "system"
|
|
and _can_carry_marker(
|
|
messages[i],
|
|
native_anthropic=native_anthropic,
|
|
tool_part_markers=tool_part_markers,
|
|
)
|
|
]
|
|
for idx in non_sys[-remaining:]:
|
|
messages[idx] = copy.deepcopy(messages[idx])
|
|
_apply_cache_marker(
|
|
messages[idx],
|
|
marker,
|
|
native_anthropic=native_anthropic,
|
|
tool_part_markers=tool_part_markers,
|
|
)
|
|
|
|
return messages
|