fix(redaction): gate ambiguous assignment values

This commit is contained in:
fangliquanflq
2026-08-28 04:03:26 +08:00
committed by Teknium
parent e4c35e397c
commit a8ddb231aa
2 changed files with 83 additions and 0 deletions

View File

@@ -270,6 +270,16 @@ _KEY_KEYWORD_RE = re.compile(
re.IGNORECASE,
)
# Key names that are credential-specific even when their values are short or
# human-readable. Bare ``token`` / ``key`` are intentionally absent: those
# words also describe model limits, tensor names, cache keys, and other public
# technical values. Their assignments are gated on value shape below.
_STRONG_KEY_KEYWORD_RE = re.compile(
r"(?:api|auth|access|refresh|session|id|bearer)[ _.\\-]?(?:key|token)"
r"|key[ _.\\-]?material|secret|passwd|password|pass|pw|credential|auth|bearer",
re.IGNORECASE,
)
def _is_word_start(s: str, i: int) -> bool:
"""True if position ``i`` in ``s`` begins a word (not mid-word)."""
@@ -326,6 +336,42 @@ def _key_has_secret_keyword(key: str) -> bool:
return True
return False
def _key_has_strong_secret_keyword(key: str) -> bool:
"""Return whether ``key`` names an unambiguously credential-bearing field."""
for match in _STRONG_KEY_KEYWORD_RE.finditer(key):
if _is_word_start(key, match.start()) and _is_word_end(key, match.end()):
return True
return False
def _looks_like_opaque_credential(value: str) -> bool:
"""Return whether an ambiguous token/key value has credential-like shape.
Known vendor prefixes and JWTs have dedicated redactors. This catches the
remaining opaque family without treating short technical scalars such as
``CPU``, ``local``, or training captions as secrets merely because their
key contains ``token`` or ``key``.
"""
if value == "***" or value.startswith("«redacted:"):
return True
if len(value) >= 16 and re.fullmatch(r"[A-Fa-f0-9]+", value):
return True
if len(value) >= 20 and re.fullmatch(r"[A-Za-z0-9_./+=-]+", value):
return True
if len(value) < 12:
return False
classes = sum(
bool(re.search(pattern, value))
for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]")
)
return classes >= 2
def _assignment_value_requires_redaction(key: str, value: str) -> bool:
"""Apply value-aware gating to key-name-only assignment matches."""
return _key_has_strong_secret_keyword(key) or _looks_like_opaque_credential(value)
# JSON field patterns: "apiKey": "value", "token": "value", etc.
_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
_JSON_FIELD_RE = re.compile(
@@ -870,6 +916,8 @@ def redact_sensitive_text(
# embedded matching inside the helper.
if not _key_has_secret_keyword(name):
return m.group(0)
if not _assignment_value_requires_redaction(name, value):
return m.group(0)
return f"{name}={quote}{_mask_token(value)}{quote}"
text = _ENV_ASSIGN_RE.sub(_redact_env, text)
# Lowercase env names (``openai_key=…``). Skip URLs — the query
@@ -905,6 +953,8 @@ def redact_sensitive_text(
# not a leaked secret value.
if _ENV_LOOKUP_VALUE_RE.match(value):
return m.group(0)
if not _assignment_value_requires_redaction(key, value):
return m.group(0)
return f'{key}: "{_mask_token(value)}"'
text = _JSON_FIELD_RE.sub(_redact_json, text)
@@ -924,6 +974,8 @@ def redact_sensitive_text(
# document text, not credentials (nearai/ironclaw#6129).
if not _key_has_secret_keyword(key):
return m.group(0)
if not _assignment_value_requires_redaction(key, value):
return m.group(0)
return f"{key}{sep}{_mask_token(value)}"
text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)

View File

@@ -100,6 +100,37 @@ class TestEnvAssignments:
result = redact_sensitive_text(text)
assert result == text
@pytest.mark.parametrize(
"text",
[
'IDENTITY_TOKEN="bailu"',
"--override-tensor per_layer_token_embd.weight=CPU",
'runtime.token="local"',
'{"token": "CPU"}',
"token: CPU",
],
)
def test_ambiguous_key_preserves_obviously_noncredential_value(self, text):
assert redact_sensitive_text(text, force=True) == text
@pytest.mark.parametrize(
"text, cleartext",
[
("PASSWORD=hunter2", "hunter2"),
("SECRET_TOKEN=bailu", "bailu"),
("id_token=local", "local"),
("CUSTOM_TOKEN=opaqueValue123456789", "opaqueValue123456789"),
('{"token": "opaqueValue123456789"}', "opaqueValue123456789"),
('{"key_material": "CPU"}', "CPU"),
('{"bearer": "local"}', "local"),
("TOKEN=" + "sk-" + "a" * 30, "a" * 20),
],
)
def test_strong_key_or_credential_shaped_value_still_redacts(
self, text, cleartext
):
assert cleartext not in redact_sensitive_text(text, force=True)