hermes-agent/tools/skillevaluator_scan.py

#!/usr/bin/env python3
"""Advisory NVIDIA SkillEvaluator Tier 1 scan for skill installs.

Runs alongside (never instead of) ``tools/skills_guard.py``, which stays the
enforcement layer (trust levels, install policy, block verdicts). This adds a
second, advisory opinion: NVIDIA SkillEvaluator's keyless Tier 1 static checks.

Contract:
- Warn, don't block: PII-class findings are shown with file/line and the
  install continues (the upstream PII scanner has known false-positive classes
  such as ``git@github.com`` and ``op://`` references).
- Prompt only for secrets-class criticals (private keys, cloud keys, tokens,
  credentialed connection strings); ``--force`` / non-interactive installs
  proceed with a loud warning instead of wedging on a prompt.
- Never break installs: scanner missing, crashing, timing out or emitting
  unparseable output all degrade to a no-op.

Optional binary::

    uv tool install --python 3.13 \
        "skillevaluator @ git+https://github.com/NVIDIA/SkillEvaluator.git@v0.1.0"

Toggle via ``skills.tier1_advisory`` in config.yaml (default on; a no-op
unless the binary is installed).
"""

from __future__ import annotations

import json
import logging
import shutil
import subprocess
import tempfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import List, Optional

logger = logging.getLogger(__name__)

SCANNER_BIN = "skillevaluator"

# Keyless, deterministic Tier 1 checks. Schema/quality are excluded: they are
# hygiene signal for the index pipeline, not install-time signal. `security`
# invokes NVIDIA SkillSpector (second optional binary, static-rules mode, no
# LLM); when absent or inconsistent it reports status="incomplete" = no opinion.
TIER1_CHECKS = "pii,unicode,lint,license,security"
SCAN_TIMEOUT_SECONDS = 120

# check_name values (from SkillEvaluator's pii_patterns.yaml categories)
# that indicate a possible REAL credential rather than personal-info
# hygiene. These are the only findings that earn a confirmation prompt.
SECRETS_CLASS_CHECKS = frozenset({
    "database_credentials",
    "hardcoded_secrets",
    "jwt_tokens",
    "webhook_urls",
    "aws_identifiers",
    "github_tokens",
    "private_keys",
})


@dataclass
class Tier1Finding:
    check: str          # e.g. "emails", "database_credentials"
    validator: str      # e.g. "PII Scan"
    severity: str       # "critical" | "high" | "medium" | "low" | "info"
    message: str
    file: str = ""
    line: int = 0
    suggestion: str = ""

    @property
    def is_secrets_class(self) -> bool:
        return self.check in SECRETS_CLASS_CHECKS

    def location(self) -> str:
        if self.file and self.line:
            return f"{self.file}:{self.line}"
        return self.file or "?"


@dataclass
class Tier1Report:
    available: bool                 # scanner ran and produced a report
    passed: bool = True
    findings: List[Tier1Finding] = field(default_factory=list)
    incomplete_checks: List[str] = field(default_factory=list)
    error: str = ""                 # why the scan is unavailable (debug only)

    @property
    def advisory_findings(self) -> List[Tier1Finding]:
        return [f for f in self.findings if not f.is_secrets_class]

    @property
    def secrets_findings(self) -> List[Tier1Finding]:
        return [f for f in self.findings if f.is_secrets_class]


def scanner_available() -> bool:
    return shutil.which(SCANNER_BIN) is not None


def tier1_advisory_enabled() -> bool:
    """Read skills.tier1_advisory from config (default True; safe because the
    scan is a no-op without the optional binary on PATH)."""
    try:
        from hermes_cli.config import load_config
        cfg = load_config()
        skills_cfg = cfg.get("skills") or {}
        if not isinstance(skills_cfg, dict):
            return True
        value = skills_cfg.get("tier1_advisory", True)
        if isinstance(value, str):
            return value.strip().lower() not in ("false", "0", "no", "off")
        return bool(value)
    except Exception:
        return True


def _parse_report(report: dict) -> Tier1Report:
    """Reduce a SkillEvaluator JSON report to install-relevant findings.

    Findings from ``status == "incomplete"`` validators are kept (partial
    evidence is still evidence) but those validators are excluded from the
    pass/fail signal so an evidence-free fail can't render as a failure.
    """
    findings: List[Tier1Finding] = []
    incomplete: List[str] = []
    any_complete_failed = False
    for res in report.get("results", []) or []:
        validator = str(res.get("validator", "unknown"))
        is_incomplete = str(res.get("status", "")).lower() == "incomplete"
        if is_incomplete:
            incomplete.append(validator)
        elif not res.get("passed", True):
            any_complete_failed = True
        for f in res.get("findings", []) or []:
            if not isinstance(f, dict):
                continue
            findings.append(Tier1Finding(
                check=str(f.get("check_name", "")),
                validator=validator,
                severity=str(f.get("severity", "info")).lower(),
                message=str(f.get("message", ""))[:200],
                file=str(f.get("file_path", "")),
                line=int(f.get("line_number") or 0),
                suggestion=str(f.get("suggestion", ""))[:200],
            ))
    return Tier1Report(
        available=True,
        passed=not any_complete_failed and not findings,
        findings=findings,
        incomplete_checks=incomplete,
    )


def run_tier1_scan(skill_dir: Path, timeout: int = SCAN_TIMEOUT_SECONDS) -> Tier1Report:
    """Run SkillEvaluator Tier 1 over one skill directory; any failure returns
    ``available=False`` (no advisory opinion), never raises."""
    if not scanner_available():
        return Tier1Report(available=False, error="scanner not on PATH")
    with tempfile.TemporaryDirectory(prefix="se-tier1-") as outdir:
        try:
            subprocess.run(
                [SCANNER_BIN, "validate", str(skill_dir),
                 "--checks", TIER1_CHECKS, "--no-dedup",
                 "-r", "json", "-o", outdir],
                capture_output=True, text=True, timeout=timeout,
            )
        except subprocess.TimeoutExpired:
            return Tier1Report(available=False, error=f"scan timed out after {timeout}s")
        except OSError as exc:
            return Tier1Report(available=False, error=f"scanner failed to launch: {exc}")
        reports = sorted(Path(outdir).glob("skillevaluator-output-*.json"))
        if not reports:
            return Tier1Report(available=False, error="scanner produced no JSON report")
        try:
            parsed = json.loads(reports[-1].read_text(encoding="utf-8"))
        except (json.JSONDecodeError, OSError) as exc:
            return Tier1Report(available=False, error=f"unparseable report: {exc}")
        if not isinstance(parsed, dict):
            return Tier1Report(available=False, error="unexpected report shape")
        return _parse_report(parsed)


def format_tier1_report(report: Tier1Report, limit: int = 10) -> str:
    """Plain-text advisory summary for console display."""
    if not report.available:
        return ""
    lines: List[str] = []
    if not report.findings:
        if report.incomplete_checks:
            lines.append("SkillEvaluator Tier 1: no findings from completed checks.")
        else:
            lines.append("SkillEvaluator Tier 1: no findings.")
    else:
        lines.append(
            f"SkillEvaluator Tier 1 (advisory): "
            f"{len(report.findings)} finding(s) — informational, verify before relying on this skill."
        )
        shown = report.secrets_findings + report.advisory_findings
        for f in shown[:limit]:
            tag = "SECRETS" if f.is_secrets_class else f.severity.upper()
            lines.append(f"  [{tag}] {f.location()} — {f.message}")
        if len(shown) > limit:
            lines.append(f"  … and {len(shown) - limit} more")
    if report.incomplete_checks:
        names = ", ".join(report.incomplete_checks)
        lines.append(f"  (not run: {names} — no opinion from these checks)")
    return "\n".join(lines)