From 00919d1e8a9866fe7958993127044d8e8314fefb Mon Sep 17 00:00:00 2001
From: teknium1 <127238744+teknium1@users.noreply.github.com>
Date: Sat, 19 Sep 2026 11:49:42 -0700
Subject: [PATCH] evals(compaction): document the Jev repeated-compaction
 recipe and add the table renderer

jev_cycles_report.py renders the cycles/freed/floor/end-state table from
jev_cycles.py outputs; README carries the exact commands, thresholds and
cost so the diminishing-returns and stuck/fallback findings can be
reproduced. Fallback records now count the real tool calls.
---
 evals/compaction/README.md                    | 26 ++++++++++
 .../cycles-afff57-500k.json                   |  2 +-
 evals/compaction/scripts/jev_cycles.py        |  5 +-
 evals/compaction/scripts/jev_cycles_report.py | 47 +++++++++++++++++++
 4 files changed, 77 insertions(+), 3 deletions(-)
 create mode 100644 evals/compaction/scripts/jev_cycles_report.py

diff --git a/evals/compaction/README.md b/evals/compaction/README.md
index b56a59f2c4..3693c08499 100644
--- a/evals/compaction/README.md
+++ b/evals/compaction/README.md
@@ -88,6 +88,32 @@ priced at the OpenRouter list price of the model that answered). The run
 also writes `eval_usage.json` — the harness's own question/answer/judge
 token bill.
 
+## Repeated-compaction simulation (`scripts/jev_cycles.py`)
+
+A one-shot recall score misses the failure mode of "decide, don't summarise"
+compaction: it never removes user/assistant text, so each cycle frees only
+`threshold − text_floor` and the floor grows monotonically. `jev_cycles.py`
+feeds a lineage chronologically and compacts with the Jev arm every time the
+estimate crosses the threshold, recording per cycle: tokens before/after,
+percent freed, text floor, candidate/dropped calls, fitting stage, state
+tokens, requests and Jev cost. It stops at end of transcript, when a cycle
+frees nothing (`stuck`), or when the state cannot fit Jev's 25K ceiling
+(`fallback` — the plugin throws there).
+
+```bash
+# lineage from a state.db COPY (see above), then, with OPENROUTER_API_KEY set:
+python evals/compaction/scripts/jev_cycles.py /path/lineage.json 500000 40 > cycles-500k.json
+python evals/compaction/scripts/jev_cycles.py /path/lineage.json 160000 60 > cycles-160k.json
+python evals/compaction/scripts/jev_cycles_report.py cycles-*.json      # markdown table
+```
+
+Threshold 500000 ≈ Hermes' 1M-window posture; 160000 ≈ a 200K-window host.
+Each cycle costs 1–8 Jev requests (< 1¢); a 40-cycle run is ~$0.20. The
+2026-09-19 runs are committed under `results/jev-cycles-2026-09-19/` (counts
+only, no transcript content) and summarised in `SCORECARD-2026-09-19-jev.md`:
+freed-per-cycle decayed 63% → 8% / 76% → 20% / 89% → 55% over 32–40 cycles,
+one 200K run was stuck after 0.42M tokens of work, one transcript never fit.
+
 ## Notes
 
 - Question generation and judging use `agent.auxiliary_client.call_llm`
diff --git a/evals/compaction/results/jev-cycles-2026-09-19/cycles-afff57-500k.json b/evals/compaction/results/jev-cycles-2026-09-19/cycles-afff57-500k.json
index 78fced42ee..04af79f44e 100644
--- a/evals/compaction/results/jev-cycles-2026-09-19/cycles-afff57-500k.json
+++ b/evals/compaction/results/jev-cycles-2026-09-19/cycles-afff57-500k.json
@@ -11,7 +11,7 @@
    "before": 500301,
    "fallback": "history too large for Jev (~25972 tokens after truncation, limit 25000)",
    "floor": 22650,
-   "calls": 0
+   "calls": 542
   }
  ],
  "jev_usd_total": 0.0
diff --git a/evals/compaction/scripts/jev_cycles.py b/evals/compaction/scripts/jev_cycles.py
index 983854f50c..7af0b626ea 100644
--- a/evals/compaction/scripts/jev_cycles.py
+++ b/evals/compaction/scripts/jev_cycles.py
@@ -14,7 +14,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
 from evals.compaction.fixtures import estimate_tokens, load_transcript  # noqa: E402
-from evals.compaction.jev_arm import JevCompactor, JevOptions  # noqa: E402
+from evals.compaction.jev_arm import JevCompactor, JevOptions, collect_tool_calls  # noqa: E402
 
 path, threshold, max_cycles = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
 name = Path(path).stem
@@ -40,7 +40,8 @@ while i < len(msgs) and len(cycles) < max_cycles:
         out = jc.compress(ctx)
     except ValueError as e:
         cycles.append({"cycle": len(cycles) + 1, "at_msg": i, "before": before, "fallback": str(e)[:90],
-                       "floor": text_floor(ctx), "calls": len(jc.decisions)})
+                       "floor": text_floor(ctx),
+                       "calls": len(collect_tool_calls(ctx, jc.opt.preserve_recent_messages))})
         break
     total_jev_usd += jc.usage.cost_usd
     after = tokens(out)
diff --git a/evals/compaction/scripts/jev_cycles_report.py b/evals/compaction/scripts/jev_cycles_report.py
new file mode 100644
index 0000000000..d989ddbd29
--- /dev/null
+++ b/evals/compaction/scripts/jev_cycles_report.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Render the repeated-compaction table from jev_cycles.py outputs.
+
+Usage: jev_cycles_report.py <cycles_json> [<cycles_json> ...]
+
+One row per run: cycles reached, raw session consumed, freed-per-cycle at the
+first and last cycle, text floor at the first and last cycle, and how the run
+ended (still working / stuck / plugin fallback). Prints a markdown table.
+"""
+import json
+import sys
+
+
+def describe(d: dict) -> dict:
+    cycles = d["cycles"]
+    scored = [c for c in cycles if "freed_pct" in c]
+    row = {
+        "run": f"{d['transcript']} @{d['threshold'] // 1000}K",
+        "cycles": len(scored),
+        "raw": f"{d['raw_tokens_consumed'] / 1e6:.2f}M of {d['raw_total'] / 1e6:.1f}M",
+        "freed": "—",
+        "floor": "—",
+        "end": "still working",
+    }
+    if scored:
+        row["freed"] = f"{scored[0]['freed_pct']:.0f}% → {scored[-1]['freed_pct']:.0f}%"
+        row["floor"] = f"{scored[0]['floor'] / 1000:.0f}K → {scored[-1]['floor'] / 1000:.0f}K"
+        if scored[-1].get("stuck"):
+            row["end"] = "STUCK (floor ≥ threshold, 0% freed)"
+        elif scored[-1]["freed_pct"] < 10:
+            row["end"] = f"degraded: {scored[-1]['before'] - scored[-1]['after']:,} tokens freed/cycle"
+    if cycles and "fallback" in cycles[-1]:
+        c = cycles[-1]
+        row["end"] = f"fallback on cycle {c['cycle']} ({c['calls']} calls, state does not fit)"
+    return row
+
+
+def main() -> None:
+    rows = [describe(json.load(open(p, encoding="utf-8"))) for p in sys.argv[1:]]
+    print("| run | cycles | raw session consumed | freed per cycle | text floor | end state |")
+    print("|---|---|---|---|---|---|")
+    for r in rows:
+        print(f"| {r['run']} | {r['cycles']} | {r['raw']} | {r['freed']} | {r['floor']} | {r['end']} |")
+
+
+if __name__ == "__main__":
+    main()