perf(debug): redact support egress once
redact_debug_support_text ran redact_for_egress (itself a forced redact_sensitive_text pass) and then a second forced pass with URL credentials; collect_share_bundle then re-scrubbed every bundle member and build_nous_bundle walked it a third time. A ~1MB log took ~1.0s vs 0.32s for one pass, per log. Run redact_sensitive_text once (the support bearer sweep already covers redact_for_egress's residue sweep), drop the bundle re-scrub and the Nous re-walk (every member is already scrubbed at capture), and with them the now-unused structured value walker. Also drop _redact_log_text's ignored redact_url_credentials kwarg and fail closed with the shared REDACTION_UNAVAILABLE sentinel.
This commit is contained in:
kshitijk4poor
@@ -18,7 +18,6 @@ from hermes_constants import get_hermes_home
|
||||
from hermes_cli.debug_redaction import (
|
||||
redact_debug_support_error,
|
||||
redact_debug_support_text,
|
||||
redact_debug_support_value,
|
||||
)
|
||||
from utils import atomic_json_write
|
||||
|
||||
@@ -257,8 +256,8 @@ def _resolve_log_path(log_name: str) -> Optional[Path]:
|
||||
return None
|
||||
|
||||
|
||||
def _redact_log_text(text: str, *, redact_url_credentials: bool = True) -> str:
|
||||
"""Strict support-egress scrub; the compatibility flag is intentionally ignored.
|
||||
def _redact_log_text(text: str) -> str:
|
||||
"""Strict support-egress scrub (``hermes_cli.debug_redaction``).
|
||||
|
||||
Debug/support payloads always use strict URL handling. Ordinary tool output
|
||||
retains its separate, less destructive URL policy in ``agent.redact``.
|
||||
@@ -371,7 +370,7 @@ def _capture_dump(redact: bool = True) -> str:
|
||||
with contextlib.redirect_stdout(capture), contextlib.suppress(SystemExit):
|
||||
run_dump(SimpleNamespace(show_keys=False))
|
||||
text = capture.getvalue()
|
||||
return _redact_log_text(text, redact_url_credentials=True) if redact else text
|
||||
return _redact_log_text(text) if redact else text
|
||||
|
||||
|
||||
def collect_debug_report(
|
||||
@@ -429,20 +428,15 @@ def collect_share_bundle(log_lines: int = 200, redact: bool = True) -> dict[str,
|
||||
if full := log_snapshots[name].full_text:
|
||||
filename = LOG_FILES.get(name, f"{name}.log")
|
||||
bundle[filename] = banner + dump_text + f"\n\n--- full {filename} ---\n" + full
|
||||
if redact:
|
||||
# Final textual boundary: future bundle members cannot bypass the
|
||||
# shared support scrub merely because their collector forgot it.
|
||||
bundle = {label: _redact_log_text(text) for label, text in bundle.items()}
|
||||
return bundle
|
||||
|
||||
|
||||
def build_nous_bundle(bundle: dict[str, str], redact: bool = True) -> bytes:
|
||||
"""Gzip a :func:`collect_share_bundle` mapping into the Nous envelope (shape parsed by the
|
||||
discord-support viewer — keep it stable)."""
|
||||
files = redact_debug_support_value(bundle) if redact else bundle
|
||||
envelope = {"format": _NOUS_BUNDLE_FORMAT, "redacted": bool(redact),
|
||||
"created": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"files": files}
|
||||
"files": bundle}
|
||||
return gzip.compress(json.dumps(envelope).encode("utf-8"))
|
||||
|
||||
|
||||
|
||||
@@ -3,14 +3,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
from urllib.parse import unquote_plus
|
||||
|
||||
from agent.redact import (
|
||||
_SENSITIVE_QUERY_PARAMS,
|
||||
REDACTION_UNAVAILABLE,
|
||||
SECRET_HEADER_NAME_LIST,
|
||||
is_secret_field_name,
|
||||
redact_sensitive_text,
|
||||
)
|
||||
|
||||
_REDACTED = "[REDACTED]"
|
||||
@@ -100,59 +99,6 @@ _SUPPORT_BEARER_RE = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_SECRET_FIELD_COMPACT_NAMES = frozenset(
|
||||
{
|
||||
"apikey",
|
||||
"token",
|
||||
"accesstoken",
|
||||
"refreshtoken",
|
||||
"idtoken",
|
||||
"authtoken",
|
||||
"clientsecret",
|
||||
"privatekey",
|
||||
"secretkey",
|
||||
"password",
|
||||
"passwd",
|
||||
"credential",
|
||||
"credentials",
|
||||
"authorization",
|
||||
"bearer",
|
||||
"keymaterial",
|
||||
"rawsecret",
|
||||
"secretvalue",
|
||||
"secretinput",
|
||||
}
|
||||
)
|
||||
_SECRET_ENV_PARTS = (
|
||||
"apikey",
|
||||
"token",
|
||||
"secret",
|
||||
"password",
|
||||
"passwd",
|
||||
"credential",
|
||||
"authorization",
|
||||
"privatekey",
|
||||
"accesskey",
|
||||
)
|
||||
_SECRET_ARG_FLAGS = frozenset(
|
||||
{
|
||||
"apikey",
|
||||
"accesstoken",
|
||||
"refreshtoken",
|
||||
"idtoken",
|
||||
"authtoken",
|
||||
"clientsecret",
|
||||
"privatekey",
|
||||
"secretkey",
|
||||
"password",
|
||||
"passwd",
|
||||
"credential",
|
||||
"token",
|
||||
"secret",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _decoded_compact_name(value: object) -> str:
|
||||
text = str(value or "")
|
||||
for _ in range(3):
|
||||
@@ -238,75 +184,15 @@ def redact_debug_support_text(value: object, *, max_chars: int | None = None) ->
|
||||
if not text:
|
||||
return text
|
||||
try:
|
||||
from agent.redact import REDACTION_UNAVAILABLE, redact_for_egress, redact_sensitive_text
|
||||
|
||||
text = redact_for_egress(text)
|
||||
if text == REDACTION_UNAVAILABLE:
|
||||
return text
|
||||
text = redact_sensitive_text(
|
||||
text, force=True, redact_url_credentials=True
|
||||
)
|
||||
text = redact_sensitive_text(text, force=True, redact_url_credentials=True)
|
||||
text = _redact_url_params(text)
|
||||
text = _redact_headers_and_argv(text)
|
||||
text = _EMAIL_ADDRESS_RE.sub("[REDACTED_EMAIL]", text)
|
||||
except Exception:
|
||||
return "[redaction-unavailable]"
|
||||
return REDACTION_UNAVAILABLE
|
||||
return text[:max_chars] if max_chars is not None else text
|
||||
|
||||
|
||||
def redact_debug_support_error(error: object) -> str:
|
||||
"""Render an outward-facing debug/support failure without raw exception data."""
|
||||
return redact_debug_support_text(error)
|
||||
|
||||
|
||||
def _is_secret_field(key: object, *, parent: str) -> bool:
|
||||
compact = _decoded_compact_name(key)
|
||||
if compact in _SECRET_FIELD_COMPACT_NAMES:
|
||||
return True
|
||||
if parent in {"headers", "extraheaders"}:
|
||||
return compact in _SENSITIVE_HEADER_COMPACT_NAMES
|
||||
if parent == "env":
|
||||
return any(part in compact for part in _SECRET_ENV_PARTS)
|
||||
return is_secret_field_name(key)
|
||||
|
||||
|
||||
def _redact_sequence(value: Sequence[Any]) -> list[Any]:
|
||||
items = list(value)
|
||||
output = [redact_debug_support_value(item) for item in items]
|
||||
for index, item in enumerate(items):
|
||||
compact = _decoded_compact_name(item) if isinstance(item, str) else ""
|
||||
if compact in _SECRET_ARG_FLAGS and index + 1 < len(output):
|
||||
output[index + 1] = _REDACTED
|
||||
if compact != "header" or not str(item).lstrip().startswith("--") or index + 1 >= len(items):
|
||||
continue
|
||||
header = str(items[index + 1])
|
||||
if ":" in header:
|
||||
output[index + 1] = redact_debug_support_text(header)
|
||||
continue
|
||||
if _decoded_compact_name(header) not in _SENSITIVE_HEADER_COMPACT_NAMES:
|
||||
continue
|
||||
operand = index + 2
|
||||
if operand < len(items) and str(items[operand]).casefold() in {"bearer", "basic", "digest"}:
|
||||
operand += 1
|
||||
if operand < len(output):
|
||||
output[operand] = _REDACTED
|
||||
return output
|
||||
|
||||
|
||||
def redact_debug_support_value(value: Any, *, _parent: str = "") -> Any:
|
||||
"""Recursively scrub mappings and argv-like sequences for support serialization."""
|
||||
if isinstance(value, Mapping):
|
||||
output: dict[Any, Any] = {}
|
||||
for key, child in value.items():
|
||||
if _is_secret_field(key, parent=_parent):
|
||||
output[key] = _REDACTED
|
||||
else:
|
||||
output[key] = redact_debug_support_value(
|
||||
child, _parent=_decoded_compact_name(key)
|
||||
)
|
||||
return output
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return _redact_sequence(value)
|
||||
if isinstance(value, str):
|
||||
return redact_debug_support_text(value)
|
||||
return value
|
||||
|
||||
Reference in New Issue
Block a user