fix(redaction): gate ambiguous assignment values
This commit is contained in:
@@ -270,6 +270,16 @@ _KEY_KEYWORD_RE = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Key names that are credential-specific even when their values are short or
|
||||
# human-readable. Bare ``token`` / ``key`` are intentionally absent: those
|
||||
# words also describe model limits, tensor names, cache keys, and other public
|
||||
# technical values. Their assignments are gated on value shape below.
|
||||
_STRONG_KEY_KEYWORD_RE = re.compile(
|
||||
r"(?:api|auth|access|refresh|session|id|bearer)[ _.\\-]?(?:key|token)"
|
||||
r"|key[ _.\\-]?material|secret|passwd|password|pass|pw|credential|auth|bearer",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_word_start(s: str, i: int) -> bool:
|
||||
"""True if position ``i`` in ``s`` begins a word (not mid-word)."""
|
||||
@@ -326,6 +336,42 @@ def _key_has_secret_keyword(key: str) -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _key_has_strong_secret_keyword(key: str) -> bool:
|
||||
"""Return whether ``key`` names an unambiguously credential-bearing field."""
|
||||
for match in _STRONG_KEY_KEYWORD_RE.finditer(key):
|
||||
if _is_word_start(key, match.start()) and _is_word_end(key, match.end()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _looks_like_opaque_credential(value: str) -> bool:
|
||||
"""Return whether an ambiguous token/key value has credential-like shape.
|
||||
|
||||
Known vendor prefixes and JWTs have dedicated redactors. This catches the
|
||||
remaining opaque family without treating short technical scalars such as
|
||||
``CPU``, ``local``, or training captions as secrets merely because their
|
||||
key contains ``token`` or ``key``.
|
||||
"""
|
||||
if value == "***" or value.startswith("«redacted:"):
|
||||
return True
|
||||
if len(value) >= 16 and re.fullmatch(r"[A-Fa-f0-9]+", value):
|
||||
return True
|
||||
if len(value) >= 20 and re.fullmatch(r"[A-Za-z0-9_./+=-]+", value):
|
||||
return True
|
||||
if len(value) < 12:
|
||||
return False
|
||||
classes = sum(
|
||||
bool(re.search(pattern, value))
|
||||
for pattern in (r"[a-z]", r"[A-Z]", r"[0-9]")
|
||||
)
|
||||
return classes >= 2
|
||||
|
||||
|
||||
def _assignment_value_requires_redaction(key: str, value: str) -> bool:
|
||||
"""Apply value-aware gating to key-name-only assignment matches."""
|
||||
return _key_has_strong_secret_keyword(key) or _looks_like_opaque_credential(value)
|
||||
|
||||
# JSON field patterns: "apiKey": "value", "token": "value", etc.
|
||||
_JSON_KEY_NAMES = r"(?:api_?[Kk]ey|token|secret|password|access_token|refresh_token|auth_token|bearer|secret_value|raw_secret|secret_input|key_material)"
|
||||
_JSON_FIELD_RE = re.compile(
|
||||
@@ -870,6 +916,8 @@ def redact_sensitive_text(
|
||||
# embedded matching inside the helper.
|
||||
if not _key_has_secret_keyword(name):
|
||||
return m.group(0)
|
||||
if not _assignment_value_requires_redaction(name, value):
|
||||
return m.group(0)
|
||||
return f"{name}={quote}{_mask_token(value)}{quote}"
|
||||
text = _ENV_ASSIGN_RE.sub(_redact_env, text)
|
||||
# Lowercase env names (``openai_key=…``). Skip URLs — the query
|
||||
@@ -905,6 +953,8 @@ def redact_sensitive_text(
|
||||
# not a leaked secret value.
|
||||
if _ENV_LOOKUP_VALUE_RE.match(value):
|
||||
return m.group(0)
|
||||
if not _assignment_value_requires_redaction(key, value):
|
||||
return m.group(0)
|
||||
return f'{key}: "{_mask_token(value)}"'
|
||||
text = _JSON_FIELD_RE.sub(_redact_json, text)
|
||||
|
||||
@@ -924,6 +974,8 @@ def redact_sensitive_text(
|
||||
# document text, not credentials (nearai/ironclaw#6129).
|
||||
if not _key_has_secret_keyword(key):
|
||||
return m.group(0)
|
||||
if not _assignment_value_requires_redaction(key, value):
|
||||
return m.group(0)
|
||||
return f"{key}{sep}{_mask_token(value)}"
|
||||
text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
|
||||
|
||||
|
||||
@@ -100,6 +100,37 @@ class TestEnvAssignments:
|
||||
result = redact_sensitive_text(text)
|
||||
assert result == text
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'IDENTITY_TOKEN="bailu"',
|
||||
"--override-tensor per_layer_token_embd.weight=CPU",
|
||||
'runtime.token="local"',
|
||||
'{"token": "CPU"}',
|
||||
"token: CPU",
|
||||
],
|
||||
)
|
||||
def test_ambiguous_key_preserves_obviously_noncredential_value(self, text):
|
||||
assert redact_sensitive_text(text, force=True) == text
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, cleartext",
|
||||
[
|
||||
("PASSWORD=hunter2", "hunter2"),
|
||||
("SECRET_TOKEN=bailu", "bailu"),
|
||||
("id_token=local", "local"),
|
||||
("CUSTOM_TOKEN=opaqueValue123456789", "opaqueValue123456789"),
|
||||
('{"token": "opaqueValue123456789"}', "opaqueValue123456789"),
|
||||
('{"key_material": "CPU"}', "CPU"),
|
||||
('{"bearer": "local"}', "local"),
|
||||
("TOKEN=" + "sk-" + "a" * 30, "a" * 20),
|
||||
],
|
||||
)
|
||||
def test_strong_key_or_credential_shaped_value_still_redacts(
|
||||
self, text, cleartext
|
||||
):
|
||||
assert cleartext not in redact_sensitive_text(text, force=True)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user