From 563f0a6fdec1c84cd0304b08c8f1e78fe891c2c9 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 7 Aug 2026 07:42:31 -0700 Subject: [PATCH] =?UTF-8?q?feat(cli):=20add=20`hermes=20approvals=20test`?= =?UTF-8?q?=20=E2=80=94=20dry-run=20approval=20verdict=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers "what would the approval system do with this command?" without executing it, prompting anyone, or persisting anything. Composes the REAL runtime evaluators from tools/approval.py in the same order as check_all_command_guards: container-skip gate, hardline blocklist, sudo-stdin guard, user approvals.deny rules, yolo/mode-off bypass, permanent command_allowlist, dangerous-pattern detection. Because the same functions run — including _command_detection_variants's normalization/de-obfuscation path — an obfuscated command gets exactly the verdict its plain form would get at runtime, and the output shows the normalized-variant trace the detectors actually evaluated. - hermes_cli/approvals_test.py: evaluate_command() + text/JSON output. Script-friendly exit codes: 0 allow, 1 usage, 2 ask-approval, 3 deny (hardline / sudo-stdin / user deny rule). - hermes_cli/subcommands/approvals.py: `test` subparser with --env-type (default local), --json, and a REMAINDER command (dest command_words — NOT "command", which main.py's startup path reads as the top-level subcommand name). - hermes_cli/approvals_suggest.py: dispatch `test` and mention it in the bare-`hermes approvals` usage text. - tests/hermes_cli/test_approvals_test.py: verdict matrix (benign / hardline / dangerous / user-deny from config / container skip / mode=off vs hardline), obfuscated==plain verdict parity with normalized trace, spy proof that the real runtime detectors are the ones invoked, read-only invariants (nothing executed; prompt and persistence paths rigged to explode), JSON shape, dispatcher and parser wiring. Read-only by construction: only detection/matching functions are called; the approval gate, prompts, gateway notify, and allowlist writers are never reached. Inspired by: Amp `permissions test` (idea-level, proprietary — zero code) --- hermes_cli/approvals_suggest.py | 7 +- hermes_cli/approvals_test.py | 178 +++++++++++++++++++ hermes_cli/subcommands/approvals.py | 38 ++++ tests/hermes_cli/test_approvals_test.py | 221 ++++++++++++++++++++++++ 4 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 hermes_cli/approvals_test.py create mode 100644 tests/hermes_cli/test_approvals_test.py diff --git a/hermes_cli/approvals_suggest.py b/hermes_cli/approvals_suggest.py index 362171f447..1288d79e2b 100644 --- a/hermes_cli/approvals_suggest.py +++ b/hermes_cli/approvals_suggest.py @@ -470,13 +470,18 @@ def approvals_command(args) -> int: sub = getattr(args, "approvals_command", None) if sub == "suggest": return suggest_command(args) + if sub == "test": + from hermes_cli.approvals_test import approvals_test_command + return approvals_test_command(args) print( "usage: hermes approvals \n" "\n" "subcommands:\n" " suggest Mine past approval decisions into a proposed\n" " command_allowlist (dry by default; --apply N,M to merge)\n" + " test Dry-run the approval verdict for a command without\n" + " executing it (exit 0 allow / 2 ask / 3 deny)\n" "\n" - "Run `hermes approvals suggest -h` for details." + "Run `hermes approvals -h` for details." ) return 1 diff --git a/hermes_cli/approvals_test.py b/hermes_cli/approvals_test.py new file mode 100644 index 0000000000..2fb9aec94b --- /dev/null +++ b/hermes_cli/approvals_test.py @@ -0,0 +1,178 @@ +"""``hermes approvals test`` — dry-run approval verdict for a command. + +Answers "what would the approval system do with this command?" WITHOUT +running it, prompting anyone, or persisting anything. It composes the REAL +runtime evaluators from ``tools.approval`` in the same order the runtime +guard (``check_all_command_guards``) applies them: + + 1. container-skip gate (isolated backends bypass all guards), + 2. hardline blocklist (never bypassable, fires before yolo/off), + 3. sudo-stdin guard (unconditional), + 4. user ``approvals.deny`` rules (fire before yolo/off), + 5. yolo / ``approvals.mode: off`` bypass, + 6. permanent ``command_allowlist``, + 7. dangerous-pattern detection → would ask for approval. + +Because the same functions run — including ``_command_detection_variants``'s +normalization/de-obfuscation path — an obfuscated command (``r\\m -rf /``) +gets exactly the verdict its plain form would get at runtime, and the trace +shows the normalized variants that were actually evaluated. + +Read-only invariants: the command is never executed, no approval prompt is +raised, nothing is written to config or approval history, no gateway +notification fires. + +Exit codes (script-friendly): + 0 allow (would run without a prompt) + 1 usage error + 2 ask-approval (would raise an interactive approval prompt) + 3 deny (hardline blocklist, sudo-stdin guard, or user deny rule) +""" + +from __future__ import annotations + +import json + +EXIT_ALLOW = 0 +EXIT_USAGE = 1 +EXIT_ASK = 2 +EXIT_DENY = 3 + +_VERDICT_EXIT = { + "allow": EXIT_ALLOW, + "ask-approval": EXIT_ASK, + "hardline-deny": EXIT_DENY, + "user-deny": EXIT_DENY, +} + + +def evaluate_command(command: str, env_type: str = "local") -> dict: + """Return the dry-run verdict for *command* on *env_type*. + + Pure composition of the runtime evaluators — no execution, no prompt, + no persistence. Returns a dict with ``verdict``, ``exit_code``, + ``rule`` (matching guard/pattern name or None), ``detail`` (human + explanation), and ``normalized_variants`` (the trace of normalized / + de-obfuscated forms the detectors actually evaluated). + """ + import tools.approval as approval + + # Sync config-persisted "always" patterns so the allowlist check below + # sees what the runtime would see (load is read-only). + try: + approval.load_permanent_allowlist() + except Exception: + pass + + variants = list(approval._command_detection_variants(command)) + + def result(verdict: str, rule=None, detail: str = "") -> dict: + return { + "command": command, + "env_type": env_type, + "verdict": verdict, + "exit_code": _VERDICT_EXIT[verdict], + "rule": rule, + "detail": detail, + "normalized_variants": variants, + } + + # 1. Isolated container backends skip every guard (runtime parity: + # this fires BEFORE the hardline floor in check_all_command_guards). + if approval._should_skip_container_guards(env_type): + return result( + "allow", + detail=(f"env_type '{env_type}' is an isolated container backend; " + "the runtime skips all command guards for it"), + ) + + # 2. Hardline blocklist — never bypassable, even under yolo. + is_hardline, hardline_desc = approval.detect_hardline_command(command) + if is_hardline: + return result( + "hardline-deny", rule=hardline_desc, + detail="matches the hardline blocklist (never bypassable, " + "blocked even under --yolo / approvals.mode=off)", + ) + + # 3. Sudo stdin guard — unconditional, like the hardline floor. + is_sudo_guess, sudo_desc = approval._check_sudo_stdin_guard(command) + if is_sudo_guess: + return result( + "hardline-deny", rule=sudo_desc, + detail="sudo stdin guard (unconditional block)", + ) + + # 4. User-defined approvals.deny rules — fire before yolo/off. + deny_pattern = approval._match_user_deny_rule(command) + if deny_pattern is not None: + return result( + "user-deny", rule=deny_pattern, + detail="matches a user-defined approvals.deny rule in " + "config.yaml (blocked even under --yolo / mode=off)", + ) + + # 5. Yolo / approvals.mode=off bypass. + if (approval._YOLO_MODE_FROZEN + or approval.is_current_session_yolo_enabled() + or approval._get_approval_mode() == "off"): + return result( + "allow", + detail="approval bypass active (--yolo or approvals.mode: off); " + "only hardline/deny rules would block", + ) + + # 6. Permanent command_allowlist. + if approval._command_matches_permanent_allowlist(command): + return result( + "allow", + detail="matches command_allowlist in config.yaml " + "(permanently approved)", + ) + + # 7. Dangerous-pattern detection → would prompt. + is_dangerous, pattern_key, description = approval.detect_dangerous_command(command) + if is_dangerous: + return result( + "ask-approval", rule=description, + detail="matches a dangerous-command pattern; the runtime would " + f"raise an interactive approval prompt (pattern key: " + f"{pattern_key!r})", + ) + + return result("allow", detail="no guard matched; would run without a prompt") + + +def _render_text(verdict: dict) -> None: + print(f"command : {verdict['command']}") + print(f"env-type: {verdict['env_type']}") + print(f"verdict : {verdict['verdict']} (exit {verdict['exit_code']})") + if verdict["rule"]: + print(f"rule : {verdict['rule']}") + if verdict["detail"]: + print(f"detail : {verdict['detail']}") + print("normalized trace (variants the detectors evaluated):") + for v in verdict["normalized_variants"]: + print(f" - {v}") + + +def approvals_test_command(args) -> int: + """Handle ``hermes approvals test ``. Returns the exit code.""" + words = list(getattr(args, "command_words", None) or []) + # argparse REMAINDER keeps a leading "--" separator; it is not part of + # the command being evaluated. + if words and words[0] == "--": + words = words[1:] + if not words: + print("usage: hermes approvals test [--env-type TYPE] [--json] -- ") + return EXIT_USAGE + command = " ".join(words) + env_type = getattr(args, "env_type", None) or "local" + + verdict = evaluate_command(command, env_type=env_type) + + if getattr(args, "json", False): + print(json.dumps(verdict, indent=2)) + else: + _render_text(verdict) + return verdict["exit_code"] diff --git a/hermes_cli/subcommands/approvals.py b/hermes_cli/subcommands/approvals.py index ed5760cced..2aecf6178e 100644 --- a/hermes_cli/subcommands/approvals.py +++ b/hermes_cli/subcommands/approvals.py @@ -7,6 +7,7 @@ handler is injected by ``main.py`` so this module never imports ``main`` from __future__ import annotations +import argparse from typing import Callable @@ -74,4 +75,41 @@ def build_approvals_parser(subparsers, *, cmd_approvals: Callable) -> None: help="Path to an alternate session database (default: ~/.hermes/state.db)", ) suggest_parser.set_defaults(func=cmd_approvals) + + test_parser = approvals_subparsers.add_parser( + "test", + help="Dry-run the approval verdict for a command (never executes it)", + description=( + "Evaluate a command against the REAL runtime approval guards — " + "hardline blocklist, user approvals.deny rules, dangerous-pattern " + "detection, allowlist, yolo/off bypass — and print the verdict, " + "the matching rule, and the normalized-command trace, without " + "executing the command, prompting anyone, or persisting anything. " + "Exit codes: 0 allow, 2 ask-approval, 3 deny (hardline or user " + "deny rule). Tip: use `--` before the command so its own flags " + "aren't parsed: hermes approvals test -- rm -rf /tmp/x" + ), + ) + test_parser.add_argument( + "--env-type", + dest="env_type", + default="local", + help="Terminal backend type to evaluate against (default: local; " + "isolated container backends like docker skip the guards)", + ) + test_parser.add_argument( + "--json", + action="store_true", + help="Emit machine-readable JSON instead of human-readable text", + ) + test_parser.add_argument( + "command_words", + nargs=argparse.REMAINDER, + metavar="command", + # NOTE: dest must NOT be "command" — main.py's startup path reads + # args.command as the top-level subcommand name ("approvals"). + help="The command to evaluate (prefix with -- to protect its flags)", + ) + test_parser.set_defaults(func=cmd_approvals) + approvals_parser.set_defaults(func=cmd_approvals) diff --git a/tests/hermes_cli/test_approvals_test.py b/tests/hermes_cli/test_approvals_test.py new file mode 100644 index 0000000000..cda10a05f6 --- /dev/null +++ b/tests/hermes_cli/test_approvals_test.py @@ -0,0 +1,221 @@ +"""Tests for ``hermes approvals test`` — dry-run approval verdict CLI. + +The tester must compose the REAL runtime evaluators from ``tools.approval`` +(detect_hardline_command, _match_user_deny_rule, detect_dangerous_command, +the container-skip gate, and the same ``_command_detection_variants`` +normalization/de-obfuscation path) — never reimplement them. It is strictly +read-only: nothing is executed, no prompt fires, nothing is persisted. + +Exit-code contract (script-friendly, documented in the CLI help): + 0 = allow, 2 = ask-approval, 3 = deny (hardline / user deny rule). +""" + +import argparse +import json + +import pytest + +import tools.approval as A +from hermes_cli import approvals_test as at + + +def _args(command, env_type="local", as_json=False): + return argparse.Namespace( + command_words=list(command) if isinstance(command, (list, tuple)) else [command], + env_type=env_type, + json=as_json, + ) + + +@pytest.fixture +def isolated_approvals(monkeypatch): + """Isolate the evaluators from the dev machine's real config/state.""" + monkeypatch.setattr(A, "_get_approval_config", lambda: {"mode": "manual"}) + monkeypatch.setattr(A, "_YOLO_MODE_FROZEN", False) + monkeypatch.setattr(A, "is_current_session_yolo_enabled", lambda: False) + monkeypatch.setattr(A, "load_permanent_allowlist", lambda: set()) + saved = set(A._permanent_approved) + A._permanent_approved.clear() + # The tester must NEVER prompt or persist — make any attempt explode. + def _boom(*_a, **_kw): # pragma: no cover - failure path + raise AssertionError("read-only tester touched a prompt/persistence path") + monkeypatch.setattr(A, "prompt_dangerous_approval", _boom) + monkeypatch.setattr(A, "save_permanent_allowlist", _boom) + monkeypatch.setattr(A, "submit_pending", _boom, raising=False) + yield A + A._permanent_approved.clear() + A._permanent_approved.update(saved) + + +class TestVerdicts: + def test_benign_command_allows_with_exit_0(self, isolated_approvals, capsys): + rc = at.approvals_test_command(_args(["ls", "-la"])) + out = capsys.readouterr().out + assert rc == 0 + assert "allow" in out + + def test_hardline_command_denies_with_rule_name(self, isolated_approvals, capsys): + rc = at.approvals_test_command(_args(["sudo", "re" + "boot"])) + out = capsys.readouterr().out + assert rc == 3 + assert "hardline-deny" in out + assert "system shutdown/reboot" in out + + def test_dangerous_command_asks_with_exit_2(self, isolated_approvals, capsys): + rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"])) + out = capsys.readouterr().out + assert rc == 2 + assert "ask-approval" in out + assert "recursive delete" in out + + def test_user_deny_rule_from_config_honored(self, isolated_approvals, capsys, + monkeypatch): + monkeypatch.setattr( + A, "_get_approval_config", + lambda: {"mode": "manual", "deny": ["git push *"]}) + rc = at.approvals_test_command(_args(["git", "push", "origin", "main"])) + out = capsys.readouterr().out + assert rc == 3 + assert "user-deny" in out + assert "git push *" in out + + def test_container_env_type_skips_guards_like_runtime(self, isolated_approvals, + capsys): + # Mirrors check_all_command_guards: isolated docker skips BEFORE the + # hardline floor, so even a catastrophic command reports allow. + rc = at.approvals_test_command(_args(["rm", "-rf", "/"], env_type="docker")) + out = capsys.readouterr().out + assert rc == 0 + assert "allow" in out + assert "container" in out or "isolated" in out + + def test_mode_off_bypasses_dangerous_but_not_hardline(self, isolated_approvals, + capsys, monkeypatch): + monkeypatch.setattr(A, "_get_approval_config", lambda: {"mode": "off"}) + rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"])) + out = capsys.readouterr().out + assert rc == 0 + assert "off" in out + rc = at.approvals_test_command(_args(["sudo", "re" + "boot"])) + assert rc == 3 + + +class TestNormalizationParity: + """The tester must run the same de-obfuscation path as the runtime.""" + + def test_obfuscated_command_matches_plain_verdict(self, isolated_approvals, + capsys): + rc_plain = at.approvals_test_command(_args(["rm", "-rf", "/"])) + out_plain = capsys.readouterr().out + rc_obf = at.approvals_test_command(_args(["r\\m", "-rf", "/"])) + out_obf = capsys.readouterr().out + assert rc_plain == rc_obf == 3 + assert "recursive delete of root filesystem" in out_plain + assert "recursive delete of root filesystem" in out_obf + # The trace must show the de-obfuscated form the runtime evaluated. + assert "rm -rf /" in out_obf + + def test_normalized_trace_shown_when_command_normalizes(self, + isolated_approvals, + capsys): + rc = at.approvals_test_command(_args(['git', 'st""atus'])) + out = capsys.readouterr().out + assert rc == 0 + assert "git status" in out + + def test_composes_real_runtime_detectors(self, isolated_approvals, capsys, + monkeypatch): + """Prove the tester calls the real evaluators, not a reimplementation.""" + calls = {} + + def _spy(name, real): + def wrapper(c): + calls[name] = c + return real(c) + return wrapper + + monkeypatch.setattr(A, "detect_hardline_command", + _spy("hardline", A.detect_hardline_command)) + monkeypatch.setattr(A, "detect_dangerous_command", + _spy("dangerous", A.detect_dangerous_command)) + monkeypatch.setattr(A, "_match_user_deny_rule", + _spy("deny", A._match_user_deny_rule)) + monkeypatch.setattr(A, "_command_detection_variants", + _spy("variants", A._command_detection_variants)) + cmd = "rm -rf ~/project/build" + at.approvals_test_command(_args(cmd.split())) + capsys.readouterr() + assert calls.get("hardline") == cmd + assert calls.get("dangerous") == cmd + assert calls.get("deny") == cmd + assert calls.get("variants") == cmd + + +class TestReadOnly: + def test_nothing_executed(self, isolated_approvals, capsys, tmp_path): + sentinel = tmp_path / "must_not_exist" + rc = at.approvals_test_command(_args(["touch", str(sentinel)])) + capsys.readouterr() + assert rc == 0 + assert not sentinel.exists() + + def test_dangerous_command_never_prompts_or_persists(self, isolated_approvals, + capsys): + # isolated_approvals wires prompt/persistence to AssertionError; a + # dangerous command must complete without touching either. + rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"])) + capsys.readouterr() + assert rc == 2 + + +class TestOutputAndWiring: + def test_json_output_is_machine_readable(self, isolated_approvals, capsys): + rc = at.approvals_test_command(_args(["sudo", "re" + "boot"], as_json=True)) + payload = json.loads(capsys.readouterr().out) + assert rc == 3 + assert payload["verdict"] == "hardline-deny" + assert payload["exit_code"] == 3 + assert payload["rule"] == "system shutdown/reboot" + assert payload["command"] == "sudo re" + "boot" + assert isinstance(payload["normalized_variants"], list) + + def test_empty_command_is_usage_error(self, isolated_approvals, capsys): + rc = at.approvals_test_command(_args([])) + assert rc == 1 + + def test_dispatcher_routes_test_subcommand(self, isolated_approvals, capsys): + from hermes_cli.approvals_suggest import approvals_command + args = _args(["ls"]) + args.approvals_command = "test" + rc = approvals_command(args) + out = capsys.readouterr().out + assert rc == 0 + assert "allow" in out + + def test_parser_wires_test_subcommand(self, isolated_approvals, capsys): + from hermes_cli.subcommands.approvals import build_approvals_parser + parser = argparse.ArgumentParser() + sub = parser.add_subparsers() + sentinel = [] + build_approvals_parser(sub, cmd_approvals=lambda a: sentinel.append(a) or 0) + args = parser.parse_args( + ["approvals", "test", "--env-type", "ssh", "--", "ls", "-la"]) + assert args.approvals_command == "test" + assert args.env_type == "ssh" + # argparse REMAINDER keeps the leading "--"; the handler strips it. + # dest is command_words (NOT command) so main.py's startup path can + # keep reading args.command as the top-level subcommand name. + assert args.command_words == ["--", "ls", "-la"] + # The subparser must NOT claim the "command" dest — main.py's startup + # path reads args.command as the top-level subcommand name. + assert getattr(args, "command", None) != ["--", "ls", "-la"] + args.func(args) + assert sentinel + + def test_leading_separator_stripped_from_command(self, isolated_approvals, + capsys): + rc = at.approvals_test_command(_args(["--", "ls", "-la"])) + out = capsys.readouterr().out + assert rc == 0 + assert "ls -la" in out + assert "-- ls" not in out