feat(cli): add hermes approvals test — dry-run approval verdict CLI
Answers "what would the approval system do with this command?" without executing it, prompting anyone, or persisting anything. Composes the REAL runtime evaluators from tools/approval.py in the same order as check_all_command_guards: container-skip gate, hardline blocklist, sudo-stdin guard, user approvals.deny rules, yolo/mode-off bypass, permanent command_allowlist, dangerous-pattern detection. Because the same functions run — including _command_detection_variants's normalization/de-obfuscation path — an obfuscated command gets exactly the verdict its plain form would get at runtime, and the output shows the normalized-variant trace the detectors actually evaluated. - hermes_cli/approvals_test.py: evaluate_command() + text/JSON output. Script-friendly exit codes: 0 allow, 1 usage, 2 ask-approval, 3 deny (hardline / sudo-stdin / user deny rule). - hermes_cli/subcommands/approvals.py: `test` subparser with --env-type (default local), --json, and a REMAINDER command (dest command_words — NOT "command", which main.py's startup path reads as the top-level subcommand name). - hermes_cli/approvals_suggest.py: dispatch `test` and mention it in the bare-`hermes approvals` usage text. - tests/hermes_cli/test_approvals_test.py: verdict matrix (benign / hardline / dangerous / user-deny from config / container skip / mode=off vs hardline), obfuscated==plain verdict parity with normalized trace, spy proof that the real runtime detectors are the ones invoked, read-only invariants (nothing executed; prompt and persistence paths rigged to explode), JSON shape, dispatcher and parser wiring. Read-only by construction: only detection/matching functions are called; the approval gate, prompts, gateway notify, and allowlist writers are never reached. Inspired by: Amp `permissions test` (idea-level, proprietary — zero code)
This commit is contained in:
Teknium
@@ -470,13 +470,18 @@ def approvals_command(args) -> int:
|
||||
sub = getattr(args, "approvals_command", None)
|
||||
if sub == "suggest":
|
||||
return suggest_command(args)
|
||||
if sub == "test":
|
||||
from hermes_cli.approvals_test import approvals_test_command
|
||||
return approvals_test_command(args)
|
||||
print(
|
||||
"usage: hermes approvals <subcommand>\n"
|
||||
"\n"
|
||||
"subcommands:\n"
|
||||
" suggest Mine past approval decisions into a proposed\n"
|
||||
" command_allowlist (dry by default; --apply N,M to merge)\n"
|
||||
" test Dry-run the approval verdict for a command without\n"
|
||||
" executing it (exit 0 allow / 2 ask / 3 deny)\n"
|
||||
"\n"
|
||||
"Run `hermes approvals suggest -h` for details."
|
||||
"Run `hermes approvals <subcommand> -h` for details."
|
||||
)
|
||||
return 1
|
||||
|
||||
178
hermes_cli/approvals_test.py
Normal file
178
hermes_cli/approvals_test.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""``hermes approvals test`` — dry-run approval verdict for a command.
|
||||
|
||||
Answers "what would the approval system do with this command?" WITHOUT
|
||||
running it, prompting anyone, or persisting anything. It composes the REAL
|
||||
runtime evaluators from ``tools.approval`` in the same order the runtime
|
||||
guard (``check_all_command_guards``) applies them:
|
||||
|
||||
1. container-skip gate (isolated backends bypass all guards),
|
||||
2. hardline blocklist (never bypassable, fires before yolo/off),
|
||||
3. sudo-stdin guard (unconditional),
|
||||
4. user ``approvals.deny`` rules (fire before yolo/off),
|
||||
5. yolo / ``approvals.mode: off`` bypass,
|
||||
6. permanent ``command_allowlist``,
|
||||
7. dangerous-pattern detection → would ask for approval.
|
||||
|
||||
Because the same functions run — including ``_command_detection_variants``'s
|
||||
normalization/de-obfuscation path — an obfuscated command (``r\\m -rf /``)
|
||||
gets exactly the verdict its plain form would get at runtime, and the trace
|
||||
shows the normalized variants that were actually evaluated.
|
||||
|
||||
Read-only invariants: the command is never executed, no approval prompt is
|
||||
raised, nothing is written to config or approval history, no gateway
|
||||
notification fires.
|
||||
|
||||
Exit codes (script-friendly):
|
||||
0 allow (would run without a prompt)
|
||||
1 usage error
|
||||
2 ask-approval (would raise an interactive approval prompt)
|
||||
3 deny (hardline blocklist, sudo-stdin guard, or user deny rule)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
EXIT_ALLOW = 0
|
||||
EXIT_USAGE = 1
|
||||
EXIT_ASK = 2
|
||||
EXIT_DENY = 3
|
||||
|
||||
_VERDICT_EXIT = {
|
||||
"allow": EXIT_ALLOW,
|
||||
"ask-approval": EXIT_ASK,
|
||||
"hardline-deny": EXIT_DENY,
|
||||
"user-deny": EXIT_DENY,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_command(command: str, env_type: str = "local") -> dict:
|
||||
"""Return the dry-run verdict for *command* on *env_type*.
|
||||
|
||||
Pure composition of the runtime evaluators — no execution, no prompt,
|
||||
no persistence. Returns a dict with ``verdict``, ``exit_code``,
|
||||
``rule`` (matching guard/pattern name or None), ``detail`` (human
|
||||
explanation), and ``normalized_variants`` (the trace of normalized /
|
||||
de-obfuscated forms the detectors actually evaluated).
|
||||
"""
|
||||
import tools.approval as approval
|
||||
|
||||
# Sync config-persisted "always" patterns so the allowlist check below
|
||||
# sees what the runtime would see (load is read-only).
|
||||
try:
|
||||
approval.load_permanent_allowlist()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
variants = list(approval._command_detection_variants(command))
|
||||
|
||||
def result(verdict: str, rule=None, detail: str = "") -> dict:
|
||||
return {
|
||||
"command": command,
|
||||
"env_type": env_type,
|
||||
"verdict": verdict,
|
||||
"exit_code": _VERDICT_EXIT[verdict],
|
||||
"rule": rule,
|
||||
"detail": detail,
|
||||
"normalized_variants": variants,
|
||||
}
|
||||
|
||||
# 1. Isolated container backends skip every guard (runtime parity:
|
||||
# this fires BEFORE the hardline floor in check_all_command_guards).
|
||||
if approval._should_skip_container_guards(env_type):
|
||||
return result(
|
||||
"allow",
|
||||
detail=(f"env_type '{env_type}' is an isolated container backend; "
|
||||
"the runtime skips all command guards for it"),
|
||||
)
|
||||
|
||||
# 2. Hardline blocklist — never bypassable, even under yolo.
|
||||
is_hardline, hardline_desc = approval.detect_hardline_command(command)
|
||||
if is_hardline:
|
||||
return result(
|
||||
"hardline-deny", rule=hardline_desc,
|
||||
detail="matches the hardline blocklist (never bypassable, "
|
||||
"blocked even under --yolo / approvals.mode=off)",
|
||||
)
|
||||
|
||||
# 3. Sudo stdin guard — unconditional, like the hardline floor.
|
||||
is_sudo_guess, sudo_desc = approval._check_sudo_stdin_guard(command)
|
||||
if is_sudo_guess:
|
||||
return result(
|
||||
"hardline-deny", rule=sudo_desc,
|
||||
detail="sudo stdin guard (unconditional block)",
|
||||
)
|
||||
|
||||
# 4. User-defined approvals.deny rules — fire before yolo/off.
|
||||
deny_pattern = approval._match_user_deny_rule(command)
|
||||
if deny_pattern is not None:
|
||||
return result(
|
||||
"user-deny", rule=deny_pattern,
|
||||
detail="matches a user-defined approvals.deny rule in "
|
||||
"config.yaml (blocked even under --yolo / mode=off)",
|
||||
)
|
||||
|
||||
# 5. Yolo / approvals.mode=off bypass.
|
||||
if (approval._YOLO_MODE_FROZEN
|
||||
or approval.is_current_session_yolo_enabled()
|
||||
or approval._get_approval_mode() == "off"):
|
||||
return result(
|
||||
"allow",
|
||||
detail="approval bypass active (--yolo or approvals.mode: off); "
|
||||
"only hardline/deny rules would block",
|
||||
)
|
||||
|
||||
# 6. Permanent command_allowlist.
|
||||
if approval._command_matches_permanent_allowlist(command):
|
||||
return result(
|
||||
"allow",
|
||||
detail="matches command_allowlist in config.yaml "
|
||||
"(permanently approved)",
|
||||
)
|
||||
|
||||
# 7. Dangerous-pattern detection → would prompt.
|
||||
is_dangerous, pattern_key, description = approval.detect_dangerous_command(command)
|
||||
if is_dangerous:
|
||||
return result(
|
||||
"ask-approval", rule=description,
|
||||
detail="matches a dangerous-command pattern; the runtime would "
|
||||
f"raise an interactive approval prompt (pattern key: "
|
||||
f"{pattern_key!r})",
|
||||
)
|
||||
|
||||
return result("allow", detail="no guard matched; would run without a prompt")
|
||||
|
||||
|
||||
def _render_text(verdict: dict) -> None:
|
||||
print(f"command : {verdict['command']}")
|
||||
print(f"env-type: {verdict['env_type']}")
|
||||
print(f"verdict : {verdict['verdict']} (exit {verdict['exit_code']})")
|
||||
if verdict["rule"]:
|
||||
print(f"rule : {verdict['rule']}")
|
||||
if verdict["detail"]:
|
||||
print(f"detail : {verdict['detail']}")
|
||||
print("normalized trace (variants the detectors evaluated):")
|
||||
for v in verdict["normalized_variants"]:
|
||||
print(f" - {v}")
|
||||
|
||||
|
||||
def approvals_test_command(args) -> int:
|
||||
"""Handle ``hermes approvals test <command...>``. Returns the exit code."""
|
||||
words = list(getattr(args, "command_words", None) or [])
|
||||
# argparse REMAINDER keeps a leading "--" separator; it is not part of
|
||||
# the command being evaluated.
|
||||
if words and words[0] == "--":
|
||||
words = words[1:]
|
||||
if not words:
|
||||
print("usage: hermes approvals test [--env-type TYPE] [--json] -- <command...>")
|
||||
return EXIT_USAGE
|
||||
command = " ".join(words)
|
||||
env_type = getattr(args, "env_type", None) or "local"
|
||||
|
||||
verdict = evaluate_command(command, env_type=env_type)
|
||||
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(verdict, indent=2))
|
||||
else:
|
||||
_render_text(verdict)
|
||||
return verdict["exit_code"]
|
||||
@@ -7,6 +7,7 @@ handler is injected by ``main.py`` so this module never imports ``main``
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import Callable
|
||||
|
||||
|
||||
@@ -74,4 +75,41 @@ def build_approvals_parser(subparsers, *, cmd_approvals: Callable) -> None:
|
||||
help="Path to an alternate session database (default: ~/.hermes/state.db)",
|
||||
)
|
||||
suggest_parser.set_defaults(func=cmd_approvals)
|
||||
|
||||
test_parser = approvals_subparsers.add_parser(
|
||||
"test",
|
||||
help="Dry-run the approval verdict for a command (never executes it)",
|
||||
description=(
|
||||
"Evaluate a command against the REAL runtime approval guards — "
|
||||
"hardline blocklist, user approvals.deny rules, dangerous-pattern "
|
||||
"detection, allowlist, yolo/off bypass — and print the verdict, "
|
||||
"the matching rule, and the normalized-command trace, without "
|
||||
"executing the command, prompting anyone, or persisting anything. "
|
||||
"Exit codes: 0 allow, 2 ask-approval, 3 deny (hardline or user "
|
||||
"deny rule). Tip: use `--` before the command so its own flags "
|
||||
"aren't parsed: hermes approvals test -- rm -rf /tmp/x"
|
||||
),
|
||||
)
|
||||
test_parser.add_argument(
|
||||
"--env-type",
|
||||
dest="env_type",
|
||||
default="local",
|
||||
help="Terminal backend type to evaluate against (default: local; "
|
||||
"isolated container backends like docker skip the guards)",
|
||||
)
|
||||
test_parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Emit machine-readable JSON instead of human-readable text",
|
||||
)
|
||||
test_parser.add_argument(
|
||||
"command_words",
|
||||
nargs=argparse.REMAINDER,
|
||||
metavar="command",
|
||||
# NOTE: dest must NOT be "command" — main.py's startup path reads
|
||||
# args.command as the top-level subcommand name ("approvals").
|
||||
help="The command to evaluate (prefix with -- to protect its flags)",
|
||||
)
|
||||
test_parser.set_defaults(func=cmd_approvals)
|
||||
|
||||
approvals_parser.set_defaults(func=cmd_approvals)
|
||||
|
||||
221
tests/hermes_cli/test_approvals_test.py
Normal file
221
tests/hermes_cli/test_approvals_test.py
Normal file
@@ -0,0 +1,221 @@
|
||||
"""Tests for ``hermes approvals test`` — dry-run approval verdict CLI.
|
||||
|
||||
The tester must compose the REAL runtime evaluators from ``tools.approval``
|
||||
(detect_hardline_command, _match_user_deny_rule, detect_dangerous_command,
|
||||
the container-skip gate, and the same ``_command_detection_variants``
|
||||
normalization/de-obfuscation path) — never reimplement them. It is strictly
|
||||
read-only: nothing is executed, no prompt fires, nothing is persisted.
|
||||
|
||||
Exit-code contract (script-friendly, documented in the CLI help):
|
||||
0 = allow, 2 = ask-approval, 3 = deny (hardline / user deny rule).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
import tools.approval as A
|
||||
from hermes_cli import approvals_test as at
|
||||
|
||||
|
||||
def _args(command, env_type="local", as_json=False):
|
||||
return argparse.Namespace(
|
||||
command_words=list(command) if isinstance(command, (list, tuple)) else [command],
|
||||
env_type=env_type,
|
||||
json=as_json,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_approvals(monkeypatch):
|
||||
"""Isolate the evaluators from the dev machine's real config/state."""
|
||||
monkeypatch.setattr(A, "_get_approval_config", lambda: {"mode": "manual"})
|
||||
monkeypatch.setattr(A, "_YOLO_MODE_FROZEN", False)
|
||||
monkeypatch.setattr(A, "is_current_session_yolo_enabled", lambda: False)
|
||||
monkeypatch.setattr(A, "load_permanent_allowlist", lambda: set())
|
||||
saved = set(A._permanent_approved)
|
||||
A._permanent_approved.clear()
|
||||
# The tester must NEVER prompt or persist — make any attempt explode.
|
||||
def _boom(*_a, **_kw): # pragma: no cover - failure path
|
||||
raise AssertionError("read-only tester touched a prompt/persistence path")
|
||||
monkeypatch.setattr(A, "prompt_dangerous_approval", _boom)
|
||||
monkeypatch.setattr(A, "save_permanent_allowlist", _boom)
|
||||
monkeypatch.setattr(A, "submit_pending", _boom, raising=False)
|
||||
yield A
|
||||
A._permanent_approved.clear()
|
||||
A._permanent_approved.update(saved)
|
||||
|
||||
|
||||
class TestVerdicts:
|
||||
def test_benign_command_allows_with_exit_0(self, isolated_approvals, capsys):
|
||||
rc = at.approvals_test_command(_args(["ls", "-la"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "allow" in out
|
||||
|
||||
def test_hardline_command_denies_with_rule_name(self, isolated_approvals, capsys):
|
||||
rc = at.approvals_test_command(_args(["sudo", "re" + "boot"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 3
|
||||
assert "hardline-deny" in out
|
||||
assert "system shutdown/reboot" in out
|
||||
|
||||
def test_dangerous_command_asks_with_exit_2(self, isolated_approvals, capsys):
|
||||
rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 2
|
||||
assert "ask-approval" in out
|
||||
assert "recursive delete" in out
|
||||
|
||||
def test_user_deny_rule_from_config_honored(self, isolated_approvals, capsys,
|
||||
monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
A, "_get_approval_config",
|
||||
lambda: {"mode": "manual", "deny": ["git push *"]})
|
||||
rc = at.approvals_test_command(_args(["git", "push", "origin", "main"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 3
|
||||
assert "user-deny" in out
|
||||
assert "git push *" in out
|
||||
|
||||
def test_container_env_type_skips_guards_like_runtime(self, isolated_approvals,
|
||||
capsys):
|
||||
# Mirrors check_all_command_guards: isolated docker skips BEFORE the
|
||||
# hardline floor, so even a catastrophic command reports allow.
|
||||
rc = at.approvals_test_command(_args(["rm", "-rf", "/"], env_type="docker"))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "allow" in out
|
||||
assert "container" in out or "isolated" in out
|
||||
|
||||
def test_mode_off_bypasses_dangerous_but_not_hardline(self, isolated_approvals,
|
||||
capsys, monkeypatch):
|
||||
monkeypatch.setattr(A, "_get_approval_config", lambda: {"mode": "off"})
|
||||
rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "off" in out
|
||||
rc = at.approvals_test_command(_args(["sudo", "re" + "boot"]))
|
||||
assert rc == 3
|
||||
|
||||
|
||||
class TestNormalizationParity:
|
||||
"""The tester must run the same de-obfuscation path as the runtime."""
|
||||
|
||||
def test_obfuscated_command_matches_plain_verdict(self, isolated_approvals,
|
||||
capsys):
|
||||
rc_plain = at.approvals_test_command(_args(["rm", "-rf", "/"]))
|
||||
out_plain = capsys.readouterr().out
|
||||
rc_obf = at.approvals_test_command(_args(["r\\m", "-rf", "/"]))
|
||||
out_obf = capsys.readouterr().out
|
||||
assert rc_plain == rc_obf == 3
|
||||
assert "recursive delete of root filesystem" in out_plain
|
||||
assert "recursive delete of root filesystem" in out_obf
|
||||
# The trace must show the de-obfuscated form the runtime evaluated.
|
||||
assert "rm -rf /" in out_obf
|
||||
|
||||
def test_normalized_trace_shown_when_command_normalizes(self,
|
||||
isolated_approvals,
|
||||
capsys):
|
||||
rc = at.approvals_test_command(_args(['git', 'st""atus']))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "git status" in out
|
||||
|
||||
def test_composes_real_runtime_detectors(self, isolated_approvals, capsys,
|
||||
monkeypatch):
|
||||
"""Prove the tester calls the real evaluators, not a reimplementation."""
|
||||
calls = {}
|
||||
|
||||
def _spy(name, real):
|
||||
def wrapper(c):
|
||||
calls[name] = c
|
||||
return real(c)
|
||||
return wrapper
|
||||
|
||||
monkeypatch.setattr(A, "detect_hardline_command",
|
||||
_spy("hardline", A.detect_hardline_command))
|
||||
monkeypatch.setattr(A, "detect_dangerous_command",
|
||||
_spy("dangerous", A.detect_dangerous_command))
|
||||
monkeypatch.setattr(A, "_match_user_deny_rule",
|
||||
_spy("deny", A._match_user_deny_rule))
|
||||
monkeypatch.setattr(A, "_command_detection_variants",
|
||||
_spy("variants", A._command_detection_variants))
|
||||
cmd = "rm -rf ~/project/build"
|
||||
at.approvals_test_command(_args(cmd.split()))
|
||||
capsys.readouterr()
|
||||
assert calls.get("hardline") == cmd
|
||||
assert calls.get("dangerous") == cmd
|
||||
assert calls.get("deny") == cmd
|
||||
assert calls.get("variants") == cmd
|
||||
|
||||
|
||||
class TestReadOnly:
|
||||
def test_nothing_executed(self, isolated_approvals, capsys, tmp_path):
|
||||
sentinel = tmp_path / "must_not_exist"
|
||||
rc = at.approvals_test_command(_args(["touch", str(sentinel)]))
|
||||
capsys.readouterr()
|
||||
assert rc == 0
|
||||
assert not sentinel.exists()
|
||||
|
||||
def test_dangerous_command_never_prompts_or_persists(self, isolated_approvals,
|
||||
capsys):
|
||||
# isolated_approvals wires prompt/persistence to AssertionError; a
|
||||
# dangerous command must complete without touching either.
|
||||
rc = at.approvals_test_command(_args(["rm", "-rf", "~/project/build"]))
|
||||
capsys.readouterr()
|
||||
assert rc == 2
|
||||
|
||||
|
||||
class TestOutputAndWiring:
|
||||
def test_json_output_is_machine_readable(self, isolated_approvals, capsys):
|
||||
rc = at.approvals_test_command(_args(["sudo", "re" + "boot"], as_json=True))
|
||||
payload = json.loads(capsys.readouterr().out)
|
||||
assert rc == 3
|
||||
assert payload["verdict"] == "hardline-deny"
|
||||
assert payload["exit_code"] == 3
|
||||
assert payload["rule"] == "system shutdown/reboot"
|
||||
assert payload["command"] == "sudo re" + "boot"
|
||||
assert isinstance(payload["normalized_variants"], list)
|
||||
|
||||
def test_empty_command_is_usage_error(self, isolated_approvals, capsys):
|
||||
rc = at.approvals_test_command(_args([]))
|
||||
assert rc == 1
|
||||
|
||||
def test_dispatcher_routes_test_subcommand(self, isolated_approvals, capsys):
|
||||
from hermes_cli.approvals_suggest import approvals_command
|
||||
args = _args(["ls"])
|
||||
args.approvals_command = "test"
|
||||
rc = approvals_command(args)
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "allow" in out
|
||||
|
||||
def test_parser_wires_test_subcommand(self, isolated_approvals, capsys):
|
||||
from hermes_cli.subcommands.approvals import build_approvals_parser
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers()
|
||||
sentinel = []
|
||||
build_approvals_parser(sub, cmd_approvals=lambda a: sentinel.append(a) or 0)
|
||||
args = parser.parse_args(
|
||||
["approvals", "test", "--env-type", "ssh", "--", "ls", "-la"])
|
||||
assert args.approvals_command == "test"
|
||||
assert args.env_type == "ssh"
|
||||
# argparse REMAINDER keeps the leading "--"; the handler strips it.
|
||||
# dest is command_words (NOT command) so main.py's startup path can
|
||||
# keep reading args.command as the top-level subcommand name.
|
||||
assert args.command_words == ["--", "ls", "-la"]
|
||||
# The subparser must NOT claim the "command" dest — main.py's startup
|
||||
# path reads args.command as the top-level subcommand name.
|
||||
assert getattr(args, "command", None) != ["--", "ls", "-la"]
|
||||
args.func(args)
|
||||
assert sentinel
|
||||
|
||||
def test_leading_separator_stripped_from_command(self, isolated_approvals,
|
||||
capsys):
|
||||
rc = at.approvals_test_command(_args(["--", "ls", "-la"]))
|
||||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "ls -la" in out
|
||||
assert "-- ls" not in out
|
||||
Reference in New Issue
Block a user