101 lines
5.0 KiB
Python
101 lines
5.0 KiB
Python
"""Lane 3: tool-layer friction — hardline false blocks, foreground refusals, whole-file rewrites, output
|
|
volume by tool. All OBSERVED from ``messages`` (role='tool') in the run tree.
|
|
|
|
python -m evals.postmortem.forensics.tools --db state_copy.db
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import collections
|
|
import json
|
|
import re
|
|
from typing import Any, Dict
|
|
|
|
from evals.postmortem.forensics.common import Run
|
|
|
|
_HARDLINE = "BLOCKED (hardline)"
|
|
_MALFORMED = "command parser limit or malformed executable payload"
|
|
_FG_TIMEOUT = re.compile(r"Foreground timeout (\d+)s exceeds the maximum")
|
|
_FG_AMP = "Foreground command uses '&' backgrounding"
|
|
_FG_WRAP = "shell-level background wrappers"
|
|
_DEADLINE = re.compile(r"timed out after ([\d.]+)s")
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
run = Run.from_args(argv, (__doc__ or "").split("\n\n")[0])
|
|
by_tool_bytes: dict[str, int] = collections.Counter()
|
|
by_tool_calls: dict[str, int] = collections.Counter()
|
|
hardline = malformed = fg_timeout = fg_amp = fg_wrap = tool_deadline = 0
|
|
fg_timeout_requested: dict[int, int] = collections.Counter()
|
|
write_calls = big_writes = rewrite_of_existing = 0
|
|
write_chars = 0
|
|
read_paths_by_sess: dict[str, set] = collections.defaultdict(set)
|
|
for sid in run.in_run:
|
|
for m in run.messages(sid, "role, tool_name, content, tool_calls"):
|
|
if m["role"] == "assistant" and m.get("tool_calls"):
|
|
try:
|
|
tcs = json.loads(m["tool_calls"])
|
|
except ValueError:
|
|
continue
|
|
for tc in tcs:
|
|
fn = (tc.get("function") or {})
|
|
name = fn.get("name") or ""
|
|
by_tool_calls[name] += 1
|
|
if name in ("read_file", "write_file", "patch"):
|
|
try:
|
|
args = json.loads(fn.get("arguments") or "{}")
|
|
except ValueError:
|
|
args = {}
|
|
if name == "read_file" and args.get("path"):
|
|
read_paths_by_sess[sid].add(args["path"])
|
|
if name == "write_file":
|
|
write_calls += 1
|
|
content = args.get("content") or ""
|
|
write_chars += len(content)
|
|
if len(content) > 20_000:
|
|
big_writes += 1
|
|
if args.get("path") in read_paths_by_sess[sid]:
|
|
rewrite_of_existing += 1
|
|
continue
|
|
if m["role"] != "tool":
|
|
continue
|
|
c = m.get("content") or ""
|
|
by_tool_bytes[m.get("tool_name") or "?"] += len(c)
|
|
if _HARDLINE in c:
|
|
hardline += 1
|
|
if _MALFORMED in c:
|
|
malformed += 1
|
|
mt = _FG_TIMEOUT.search(c)
|
|
if mt:
|
|
fg_timeout += 1; fg_timeout_requested[int(mt.group(1))] += 1
|
|
if _FG_AMP in c:
|
|
fg_amp += 1
|
|
if _FG_WRAP in c:
|
|
fg_wrap += 1
|
|
if (m.get("tool_name") or "") == "terminal" and _DEADLINE.search(c):
|
|
tool_deadline += 1
|
|
top_bytes = sorted(by_tool_bytes.items(), key=lambda kv: -kv[1])[:8]
|
|
report: dict[str, Any] = {"observed": {
|
|
"tool_results": sum(by_tool_calls.values()),
|
|
"output_bytes_by_tool_top8": {k: v for k, v in top_bytes},
|
|
"hardline_blocks": hardline, "hardline_blocks_malformed_class": malformed,
|
|
"foreground_timeout_refusals": fg_timeout, "foreground_timeout_requested_top": dict(fg_timeout_requested.most_common(5)),
|
|
"foreground_ampersand_refusals": fg_amp, "foreground_wrapper_refusals": fg_wrap,
|
|
"terminal_deadline_kills": tool_deadline,
|
|
"write_file": {"calls": write_calls, "chars": write_chars, "writes_over_20k_chars": big_writes,
|
|
"rewrites_of_a_file_read_this_session_over_20k": rewrite_of_existing,
|
|
"output_usd_at_fitted_price": round(write_chars / 3.5 * run.price_per_token["output_tokens"], 2)},
|
|
"patch_calls": by_tool_calls.get("patch", 0),
|
|
}}
|
|
path = run.write("tools.json", report)
|
|
o = report["observed"]
|
|
print(f"[tools] {o['tool_results']:,} tool calls; hardline blocks {o['hardline_blocks']} ({o['hardline_blocks_malformed_class']} 'malformed' class); "
|
|
f"foreground timeout refusals {o['foreground_timeout_refusals']} (asked: {o['foreground_timeout_requested_top']}); '&' {o['foreground_ampersand_refusals']}, nohup {o['foreground_wrapper_refusals']}")
|
|
w = o["write_file"]
|
|
print(f"[tools] write_file {w['calls']:,} calls, {w['chars']/1e6:.1f}M chars (~${w['output_usd_at_fitted_price']:,}); >20k: {w['writes_over_20k_chars']}, of which rewrites of a file read this session: {w['rewrites_of_a_file_read_this_session_over_20k']}; patch calls {o['patch_calls']:,}")
|
|
print(f"[tools] wrote {path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|