fix(caching): prevent whitespace-only text blocks in prompt cache prefix splits

This commit is contained in:
ericmaddox
2026-08-28 22:43:08 -04:00
committed by kshitij
parent 9c137e6163
commit f0d5f1298b
3 changed files with 26 additions and 21 deletions

View File

@@ -67,10 +67,11 @@ def register_stable_prefix(prefix: str) -> None:
def find_stable_prefix(content: str) -> Optional[str]:
"""Longest registered prefix that is a *proper* prefix of ``content``.
"""Longest registered prefix that is a *proper* prefix of ``content`` with non-whitespace tail.
Proper (``len(content) > len(prefix)``) so the split never produces an
empty volatile text block, which Anthropic rejects on the wire.
Proper with non-whitespace tail (``bool(content[len(prefix):].strip())``) so the
split never produces an empty or whitespace-only volatile text block, which
Anthropic rejects on the wire (HTTP 400).
A hit refreshes the entry's LRU position: a scaffold fired every minute
by cron must not be evicted by a burst of one-off skill invocations,
@@ -79,7 +80,7 @@ def find_stable_prefix(content: str) -> Optional[str]:
with _lock:
best: Optional[str] = None
for prefix in _prefixes:
if len(content) > len(prefix) and content.startswith(prefix):
if content.startswith(prefix) and bool(content[len(prefix):].strip()):
if best is None or len(prefix) > len(best):
best = prefix
if best is not None:

View File

@@ -63,20 +63,22 @@ def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool =
if role == "user":
stable_prefix = find_stable_prefix(content)
if stable_prefix is not None:
# Builder-declared boundary (#81867): the scaffold carries the
# breakpoint, the volatile invocation tail rides unmarked so a
# changed ticket ID or timestamp no longer invalidates the
# whole skill body. Request-local only — the canonical session
# message stays a plain string.
msg["content"] = [
{
"type": "text",
"text": stable_prefix,
"cache_control": cache_marker,
},
{"type": "text", "text": content[len(stable_prefix):]},
]
return
suffix = content[len(stable_prefix):]
if suffix.strip():
# Builder-declared boundary (#81867): the scaffold carries the
# breakpoint, the volatile invocation tail rides unmarked so a
# changed ticket ID or timestamp no longer invalidates the
# whole skill body. Request-local only — the canonical session
# message stays a plain string.
msg["content"] = [
{
"type": "text",
"text": stable_prefix,
"cache_control": cache_marker,
},
{"type": "text", "text": suffix},
]
return
msg["content"] = [
{"type": "text", "text": content, "cache_control": cache_marker}
]
@@ -204,7 +206,7 @@ def _apply_system_cache_markers(
and content.startswith(static_system_prefix)
):
suffix = content[len(static_system_prefix):]
if suffix:
if suffix.strip():
suffix_part: dict = {"type": "text", "text": suffix}
if mark_suffix:
suffix_part["cache_control"] = cache_marker
@@ -217,7 +219,7 @@ def _apply_system_cache_markers(
suffix_part,
]
return 2 if mark_suffix else 1
# Empty suffix: the stored prompt IS the static prefix. Mark it as
# Empty/whitespace-only suffix: the stored prompt IS the static prefix. Mark it as
# one whole block — a [marked-prefix, ""] split would put an empty
# text block on the wire (HTTP 400 on native Anthropic).
_apply_cache_marker(message, cache_marker, native_anthropic=native_anthropic)

View File

@@ -80,8 +80,10 @@ class TestRegistry:
def test_requires_proper_prefix(self):
register_stable_prefix("scaffold")
assert find_stable_prefix("scaffold volatile") == "scaffold"
# Exact match would leave an empty volatile block — never split.
# Exact match or whitespace-only tail would leave an empty/whitespace volatile block — never split.
assert find_stable_prefix("scaffold") is None
assert find_stable_prefix("scaffold ") is None
assert find_stable_prefix("scaffold\n\n\t") is None
assert find_stable_prefix("other") is None
def test_longest_registered_prefix_wins(self):