hermes-agent/evals/postmortem/forensics/tools.py

"""Lane 3: tool-layer friction — hardline false blocks, foreground refusals, whole-file rewrites, output
volume by tool. All OBSERVED from ``messages`` (role='tool') in the run tree.

    python -m evals.postmortem.forensics.tools --db state_copy.db
"""
from __future__ import annotations

import collections
import json
import re
from typing import Any, Dict

from evals.postmortem.forensics.common import Run

_HARDLINE = "BLOCKED (hardline)"
_MALFORMED = "command parser limit or malformed executable payload"
_FG_TIMEOUT = re.compile(r"Foreground timeout (\d+)s exceeds the maximum")
_FG_AMP = "Foreground command uses '&' backgrounding"
_FG_WRAP = "shell-level background wrappers"
_DEADLINE = re.compile(r"timed out after ([\d.]+)s")


def main(argv=None) -> int:
    run = Run.from_args(argv, (__doc__ or "").split("\n\n")[0])
    by_tool_bytes: Dict[str, int] = collections.Counter()
    by_tool_calls: Dict[str, int] = collections.Counter()
    hardline = malformed = fg_timeout = fg_amp = fg_wrap = tool_deadline = 0
    fg_timeout_requested: Dict[int, int] = collections.Counter()
    write_calls = big_writes = rewrite_of_existing = 0
    write_chars = 0
    read_paths_by_sess: Dict[str, set] = collections.defaultdict(set)
    for sid in run.in_run:
        for m in run.messages(sid, "role, tool_name, content, tool_calls"):
            if m["role"] == "assistant" and m.get("tool_calls"):
                try:
                    tcs = json.loads(m["tool_calls"])
                except ValueError:
                    continue
                for tc in tcs:
                    fn = (tc.get("function") or {})
                    name = fn.get("name") or ""
                    by_tool_calls[name] += 1
                    if name in ("read_file", "write_file", "patch"):
                        try:
                            args = json.loads(fn.get("arguments") or "{}")
                        except ValueError:
                            args = {}
                        if name == "read_file" and args.get("path"):
                            read_paths_by_sess[sid].add(args["path"])
                        if name == "write_file":
                            write_calls += 1
                            content = args.get("content") or ""
                            write_chars += len(content)
                            if len(content) > 20_000:
                                big_writes += 1
                                if args.get("path") in read_paths_by_sess[sid]:
                                    rewrite_of_existing += 1
                continue
            if m["role"] != "tool":
                continue
            c = m.get("content") or ""
            by_tool_bytes[m.get("tool_name") or "?"] += len(c)
            if _HARDLINE in c:
                hardline += 1
                if _MALFORMED in c:
                    malformed += 1
            mt = _FG_TIMEOUT.search(c)
            if mt:
                fg_timeout += 1; fg_timeout_requested[int(mt.group(1))] += 1
            if _FG_AMP in c:
                fg_amp += 1
            if _FG_WRAP in c:
                fg_wrap += 1
            if (m.get("tool_name") or "") == "terminal" and _DEADLINE.search(c):
                tool_deadline += 1
    top_bytes = sorted(by_tool_bytes.items(), key=lambda kv: -kv[1])[:8]
    report: Dict[str, Any] = {"observed": {
        "tool_results": sum(by_tool_calls.values()),
        "output_bytes_by_tool_top8": {k: v for k, v in top_bytes},
        "hardline_blocks": hardline, "hardline_blocks_malformed_class": malformed,
        "foreground_timeout_refusals": fg_timeout, "foreground_timeout_requested_top": dict(fg_timeout_requested.most_common(5)),
        "foreground_ampersand_refusals": fg_amp, "foreground_wrapper_refusals": fg_wrap,
        "terminal_deadline_kills": tool_deadline,
        "write_file": {"calls": write_calls, "chars": write_chars, "writes_over_20k_chars": big_writes,
                       "rewrites_of_a_file_read_this_session_over_20k": rewrite_of_existing,
                       "output_usd_at_fitted_price": round(write_chars / 3.5 * run.price_per_token["output_tokens"], 2)},
        "patch_calls": by_tool_calls.get("patch", 0),
    }}
    path = run.write("tools.json", report)
    o = report["observed"]
    print(f"[tools] {o['tool_results']:,} tool calls; hardline blocks {o['hardline_blocks']} ({o['hardline_blocks_malformed_class']} 'malformed' class); "
          f"foreground timeout refusals {o['foreground_timeout_refusals']} (asked: {o['foreground_timeout_requested_top']}); '&' {o['foreground_ampersand_refusals']}, nohup {o['foreground_wrapper_refusals']}")
    w = o["write_file"]
    print(f"[tools] write_file {w['calls']:,} calls, {w['chars']/1e6:.1f}M chars (~${w['output_usd_at_fitted_price']:,}); >20k: {w['writes_over_20k_chars']}, of which rewrites of a file read this session: {w['rewrites_of_a_file_read_this_session_over_20k']}; patch calls {o['patch_calls']:,}")
    print(f"[tools] wrote {path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())