44982309b8
build_tool_preview -> _PREVIEW_BUILDERS per-tool table; git @refs -> _GIT_REFERENCE_ARGS; context_breakdown _skills_block/_append_overflow dedupe; prune_pre_checkpoint_items summary retention folded into one closure; build_skill_invocation_message reuses _render_skill_block; ruff SIM collapses; restored two compacted cache-policy invariant comments.
67 lines
2.8 KiB
Python
67 lines
2.8 KiB
Python
"""Builder-declared stable prefixes for Anthropic prompt caching.
|
|
|
|
Skill/webhook/cron builders concatenate a large static scaffold with a small
|
|
volatile invocation tail into one user-message string. Only the builder knows
|
|
where the tail begins, so it registers the stable prefix here and the cache
|
|
planner places a breakpoint at that boundary instead of caching the whole
|
|
message. Re-parsing marker strings out of the message at request time is
|
|
deliberately avoided: markers can legitimately appear inside skill bodies or
|
|
event payloads, and any delimiter heuristic then shrinks the cached prefix or
|
|
silently absorbs volatile bytes into it.
|
|
|
|
Process-local by design: a webhook/cron fire is built and sent by the same
|
|
process, and any miss (restart, eviction, historic message) falls back to the
|
|
whole-message policy. The split only applies while the message is one of the
|
|
plan's marked endpoints; once it rotates out it ships as one block again
|
|
(one-time re-ingest in long interactive sessions, never for webhook/cron).
|
|
"""
|
|
|
|
import threading
|
|
from collections import OrderedDict
|
|
from typing import Optional
|
|
|
|
# A couple dozen active scaffolds is generous for one gateway process.
|
|
_MAX_ENTRIES = 32
|
|
# Entries hold whole expanded skill bodies, so also bound total retained chars
|
|
# (1-4x bytes). The newest entry is always kept so one oversized scaffold still
|
|
# gets a boundary instead of silently disabling the split.
|
|
_MAX_CHARS = 4 * 1024 * 1024
|
|
|
|
_lock = threading.Lock()
|
|
_prefixes: "OrderedDict[str, None]" = OrderedDict()
|
|
|
|
|
|
def register_stable_prefix(prefix: str) -> None:
|
|
"""Record ``prefix`` as the stable scaffold of a just-built message."""
|
|
if not prefix:
|
|
return
|
|
with _lock:
|
|
_prefixes[prefix] = None
|
|
_prefixes.move_to_end(prefix)
|
|
while len(_prefixes) > _MAX_ENTRIES:
|
|
_prefixes.popitem(last=False)
|
|
while len(_prefixes) > 1 and sum(map(len, _prefixes)) > _MAX_CHARS:
|
|
_prefixes.popitem(last=False)
|
|
|
|
|
|
def find_stable_prefix(content: str) -> Optional[str]:
|
|
"""Longest registered *proper* prefix of ``content`` with a non-whitespace tail.
|
|
|
|
The tail must be non-whitespace so the split never yields an empty text
|
|
block (Anthropic rejects it with HTTP 400). A hit refreshes the entry's LRU
|
|
position so a scaffold fired every minute by cron is not evicted by a
|
|
burst of one-off skill invocations.
|
|
"""
|
|
with _lock:
|
|
best: Optional[str] = None
|
|
for prefix in _prefixes:
|
|
if (
|
|
content.startswith(prefix)
|
|
and content[len(prefix):].strip()
|
|
and (best is None or len(prefix) > len(best))
|
|
):
|
|
best = prefix
|
|
if best is not None:
|
|
_prefixes.move_to_end(best) # after the scan: never mutate mid-iteration
|
|
return best
|