diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index f86c3c6da1..3776bf8f21 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1458,6 +1458,10 @@ DEFAULT_CONFIG = { # curator ledger` / `rollback `. Never a gate — failures can't block. # See #79686. "ledger": True, + # Size cap for that ledger: once the file grows past this, the next append rewrites it + # through the unchanged-file dedup and, if still over, drops the oldest entries (0 = keep + # the ledger append-only forever, the previous behaviour). + "ledger_max_bytes": 5 * 1024 * 1024, }, # Curator — background maintenance of AGENT-CREATED skills (never hub-installed): marks diff --git a/tests/tools/test_skill_ledger.py b/tests/tools/test_skill_ledger.py index 57d444bcf9..89371ab6dd 100644 --- a/tests/tools/test_skill_ledger.py +++ b/tests/tools/test_skill_ledger.py @@ -600,3 +600,107 @@ def test_backup_fill_ignores_tar_path_traversal(ledger_env): ) # Malicious members are not. assert not any(p.endswith("evil.md") or p.endswith("outside.md") for p in paths) +import json +from pathlib import Path + +import pytest + + +def _append_padded(skill_ledger, action: str, pad: str, n: int = 1) -> None: + """Append *n* entries padded with evidence text so the ledger file grows fast.""" + for _ in range(n): + skill_ledger.append_entry(action, "my-skill", before=[], after=[], evidence={"pad": pad}) + + +def test_auto_compact_triggers_at_threshold(ledger_env, monkeypatch): + """Crossing skills.ledger_max_bytes rewrites the ledger through the delta + dedup: identical before/after manifests shrink to nothing while ids and + entry order survive.""" + from tools import skill_ledger + + import hermes_cli.config as _cfg + + monkeypatch.setattr(_cfg, "load_config", lambda *a, **k: { + "skills": {"ledger_max_bytes": 8192}}) + + for i in range(3): # pre-delta-style entries: identical fat manifests on both sides + fat = [{"path": f"my-skill/f{i}j{j}.md", "sha256": "a" * 64} for j in range(40)] + skill_ledger.append_entry("patch", "my-skill", before=fat, after=list(fat)) + + # the maintenance sweep fired mid-append: the file stays under the cap + assert skill_ledger.ledger_path().stat().st_size <= 8192 + rows = skill_ledger.list_entries() + assert len(rows) == 3, "dedup alone must reach the cap — nothing trimmed" + assert all(r["before"] == [] and r["after"] == [] for r in rows), ( + "identical manifests must be dropped by compaction" + ) + + +def test_trim_oldest_when_still_over_cap(ledger_env, monkeypatch): + """When compaction alone cannot reach the cap (every entry genuinely + differs), the oldest entries are dropped until it fits — newest entries + survive, malformed lines are kept verbatim.""" + from tools import skill_ledger + + import hermes_cli.config as _cfg + + monkeypatch.setattr(_cfg, "load_config", lambda *a, **k: { + "skills": {"ledger_max_bytes": 8192}}) + + newest_id = None + for i in range(5): + before = [{"path": f"my-skill/old{i}.md", "sha256": f"{i}" * 64}] + after = [{"path": f"my-skill/new{i}.md", "sha256": f"{i + 1}" * 64}] + newest_id = skill_ledger.append_entry( + "edit", "my-skill", before=before, after=after, + evidence={"pad": "y" * 2048}) + with open(skill_ledger.ledger_path(), "a", encoding="utf-8") as fh: + fh.write("{not json at all\n") + + skill_ledger._maintain_size() + + rows = skill_ledger.list_entries() + assert len(rows) < 5, "oldest entries must be trimmed when compaction is not enough" + assert rows[0]["id"] == newest_id, "the newest entry always survives" + # malformed lines are never parsed away — they stay in the file verbatim + raw = skill_ledger.ledger_path().read_text(encoding="utf-8") + assert "{not json at all" in raw + assert skill_ledger.ledger_path().stat().st_size <= 8192 + + +def test_disabled_threshold_never_touches_the_file(ledger_env, monkeypatch): + """ledger_max_bytes: 0 keeps the append-only contract: no + rewrite ever happens, however large the file is.""" + from tools import skill_ledger + + import hermes_cli.config as _cfg + + monkeypatch.setattr(_cfg, "load_config", lambda *a, **k: { + "skills": {"ledger": True, "ledger_max_bytes": 0}}) + + _append_padded(skill_ledger, "patch", "z" * 4096, n=4) + raw = skill_ledger.ledger_path().read_text(encoding="utf-8") + + skill_ledger._maintain_size() + + assert skill_ledger.ledger_path().read_text(encoding="utf-8") == raw + + +def test_maintenance_failure_never_blocks_append(ledger_env, monkeypatch): + """Telemetry contract: a broken maintenance sweep logs and leaves the + appended entries on disk.""" + from tools import skill_ledger + + import hermes_cli.config as _cfg + + monkeypatch.setattr(_cfg, "load_config", lambda *a, **k: { + "skills": {"ledger_max_bytes": 8192}}) + + def _boom(*a, **k): + raise OSError("disk full") + + monkeypatch.setattr(skill_ledger, "compact_ledger", _boom) + # appends past the threshold hit the broken sweep and must survive it + _append_padded(skill_ledger, "patch", "w" * 4096, n=6) + + assert len(skill_ledger.list_entries()) == 6 diff --git a/tools/skill_ledger.py b/tools/skill_ledger.py index 5e3831980e..0a5f2aa98e 100644 --- a/tools/skill_ledger.py +++ b/tools/skill_ledger.py @@ -96,6 +96,21 @@ def ledger_enabled() -> bool: return True +def _max_ledger_bytes() -> int: + """Config ``skills.ledger_max_bytes`` (default 5 MB, 0 disables): above it the + next append triggers the maintenance sweep instead of growing the file forever.""" + try: + from hermes_cli.config import cfg_get, load_config + return int( + cfg_get( + load_config(), "skills", "ledger_max_bytes", default=5 * 1024 * 1024 + ) + ) + except Exception as e: # pragma: no cover — best-effort config read + logger.debug("skill_ledger: config read failed (%s); defaulting to 5 MB", e) + return 5 * 1024 * 1024 + + def _rel_posix(path: Path | str, root: Path) -> Optional[str]: """POSIX path of ``path`` relative to ``root`` (both normalized), or None when outside.""" try: @@ -290,6 +305,7 @@ def append_entry( path.parent.mkdir(parents=True, exist_ok=True) with open(path, "a", encoding="utf-8") as fh: fh.write(json.dumps(entry, ensure_ascii=False) + "\n") + _maintain_size() return entry["id"] except Exception as e: logger.warning("skill_ledger: failed to append entry (%s) — mutation unaffected", e) @@ -309,6 +325,57 @@ def _read_ledger(what: str, *, quiet_missing: bool = False) -> Optional[bytes]: return None +def _maintain_size() -> None: + """Keep the ledger bounded: once it crosses ``skills.ledger_max_bytes`` run the + delta-dedup rewrite, and if genuinely-divergent entries still exceed the cap, + drop the oldest ones until it fits. Best-effort telemetry, never a gate: any + failure is logged and the just-appended entry stays on disk.""" + max_bytes = _max_ledger_bytes() + if max_bytes <= 0: + return + try: + if ledger_path().stat().st_size <= max_bytes: + return + _, _, size_after = compact_ledger() + if size_after > max_bytes: + _trim_oldest(max_bytes) + gc_blobs() + except Exception as e: + logger.warning( + "skill_ledger: maintenance sweep failed (%s) — ledger left as-is", e + ) + + +def _trim_oldest(max_bytes: int) -> None: + """Rewrite the ledger without its oldest parsed entries until it is at most + *max_bytes*; the newest entry always survives, malformed lines are kept verbatim.""" + path = ledger_path() + raw = _read_ledger("trim skipped") + if raw is None: + return + lines = raw.decode("utf-8").splitlines() + newest_json = None + for line in reversed(lines): + if line.strip(): + newest_json = line + break + kept: List[str] = [] + size = 2 # trailing newline + rounding slack + for line in reversed(lines): + addition = len(line.encode("utf-8")) + 1 + if kept and size + addition > max_bytes: + break + kept.append(line) + size += addition + kept.reverse() + data = ("\n".join(kept) + "\n").encode("utf-8") if kept else b"" + if newest_json is not None and newest_json not in kept: + data = (newest_json + "\n").encode("utf-8") + tmp = path.with_name(path.name + ".trim.tmp") + tmp.write_bytes(data) + os.replace(tmp, path) + + def compact_ledger() -> Tuple[int, int, int]: """Rewrite the ledger with every entry's unchanged paths dropped (see ``_delta``); ids, order and rollback semantics are preserved. Returns ``(entries, bytes_before, bytes_after)``. Atomic: the diff --git a/website/docs/user-guide/features/curator.md b/website/docs/user-guide/features/curator.md index ad668cf2b8..303dc69184 100644 --- a/website/docs/user-guide/features/curator.md +++ b/website/docs/user-guide/features/curator.md @@ -175,6 +175,13 @@ skills: ledger: false ``` +The file is also size-bounded: once it grows past `skills.ledger_max_bytes` (default 5 MB), the next mutation first rewrites it through the unchanged-file dedup (exactly what `hermes curator ledger --compact` does) and, if genuinely-divergent entries still exceed the cap, drops the oldest ones — the newest entry and any malformed lines always survive, and the sweep frees blobs nothing references anymore. Set it to `0` to keep the ledger append-only forever. + +```yaml +skills: + ledger_max_bytes: 5242880 # 0 = never auto-maintain +``` + ## Archive TTL purge Archived skills are kept forever by default. If you want `~/.hermes/skills/.archive/` bounded, set a TTL and purge explicitly — purging never runs automatically, and every purged skill is captured into the ledger (with blobs) first, so even a purge leaves an auditable, recoverable trail: