From dddefaefaebbf00753bcaceaabb7808546dc156d Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:20:43 -0700 Subject: [PATCH] fix: paged, extracted and post-compaction reads count as a write_file baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stale-overwrite refusal made write_file permanently unusable for any existing file it could not show in one read_file page: every >2000-line (or >100K-char) page was recorded as partial, no full baseline ever existed, and the refusal told the model to "re-read the whole file", which the tool cannot do. Track the line ranges each task pages through per path at one mtime; contiguous pages from line 1 to total_lines are a full read (a new mtime between pages restarts the coverage). The same gap hit two siblings: the extracted-document branch (.ipynb, text-authorable) returned before any read bookkeeping, so an existing notebook could never be overwritten; and reset_file_dedup dropped every baseline on compaction while keeping read_timestamps, so every write after compaction was refused even for files unchanged on disk. Baselines now survive compaction exactly like the dedup mtime map does — only while the recorded mtime still matches. Refusal texts no longer embed the pre-PR "Warning: … Consider re-reading" copy inside "Refusing to overwrite", and every refusal names a recovery the model can perform: read the remaining pages, or use patch. --- tests/tools/test_file_staleness.py | 31 +++++++++++- tools/file_state.py | 4 +- tools/file_tools.py | 51 ++++++++++++++----- tools/file_tools_read_tracking.py | 78 +++++++++++++++++++++++------- tools/file_tools_write_guards.py | 30 +++++++----- 5 files changed, 147 insertions(+), 47 deletions(-) diff --git a/tests/tools/test_file_staleness.py b/tests/tools/test_file_staleness.py index 1ac568f0db..cb580aee62 100644 --- a/tests/tools/test_file_staleness.py +++ b/tests/tools/test_file_staleness.py @@ -19,7 +19,7 @@ from unittest.mock import patch, MagicMock from tools import file_state from tools.file_tools import read_file_tool, write_file_tool, patch_tool -from tools.file_tools_read_tracking import _check_file_staleness, _read_tracker +from tools.file_tools_read_tracking import _check_file_staleness, _read_tracker, reset_file_dedup # --------------------------------------------------------------------------- @@ -133,7 +133,7 @@ class TestStalenessCheck(unittest.TestCase): write is a baseline for its next write.""" refused = json.loads(write_file_tool(self._tmpfile, "x\n", task_id="t2")) self.assertTrue(refused.get("stale_write_blocked"), refused) - self.assertIn("has not read it in full", refused["error"]) + self.assertIn("has not seen its full current content", refused["error"]) patched = json.loads(patch_tool(mode="replace", path=self._tmpfile, old_string="original", new_string="patched", task_id="t2")) @@ -163,6 +163,33 @@ class TestStalenessCheck(unittest.TestCase): self.assertEqual(f.read(), "two\n") os.unlink(new_path) + def test_paged_read_of_large_file_is_a_full_baseline_that_survives_compaction(self): + """A file too big for one read_file page (>2000 lines) can only be seen by + paging; contiguous pages reaching the last line at one mtime count as a full + read, so write_file is not permanently refused. A compaction reset keeps that + baseline while the file is unchanged, and an edit between pages voids it.""" + with open(self._tmpfile, "w") as f: + f.write("".join(f"line {i}\n" for i in range(1, 2501))) + first = json.loads(read_file_tool(self._tmpfile, task_id="t3")) + self.assertTrue(first.get("truncated"), first) + self.assertTrue(json.loads(write_file_tool(self._tmpfile, "x\n", task_id="t3")).get("stale_write_blocked")) + + self.assertNotIn("error", json.loads(read_file_tool(self._tmpfile, offset=2001, task_id="t3"))) + reset_file_dedup("t3") + written = json.loads(write_file_tool(self._tmpfile, "merged\n", task_id="t3")) + self.assertNotIn("error", written, written) + with open(self._tmpfile) as f: + self.assertEqual(f.read(), "merged\n") + + with open(self._tmpfile, "w") as f: + f.write("".join(f"line {i}\n" for i in range(1, 2501))) + json.loads(read_file_tool(self._tmpfile, task_id="t3")) + _modify_externally(self._tmpfile, "".join(f"other {i}\n" for i in range(1, 2501))) + json.loads(read_file_tool(self._tmpfile, offset=2001, task_id="t3")) + refused = json.loads(write_file_tool(self._tmpfile, "x\n", task_id="t3")) + self.assertTrue(refused.get("stale_write_blocked"), refused) + self.assertNotIn("Warning:", refused["error"]) + @patch("tools.file_tools._get_file_ops") def test_relative_path_uses_recorded_session_cwd_for_staleness_tracking(self, mock_ops): diff --git a/tools/file_state.py b/tools/file_state.py index 2774605aa2..54518cb99c 100644 --- a/tools/file_state.py +++ b/tools/file_state.py @@ -166,8 +166,8 @@ class FileStateRegistry: if partial: return ( f"{resolved} was last read with offset/limit pagination " - "(partial view). Re-read the whole file before " - "overwriting it.") + "(partial view). Read the remaining pages, or use patch, " + "before overwriting it.") return None return ( diff --git a/tools/file_tools.py b/tools/file_tools.py index b519ca1fa6..3a55605849 100644 --- a/tools/file_tools.py +++ b/tools/file_tools.py @@ -33,7 +33,7 @@ from tools.file_tools_write_guards import ( _is_internal_file_tool_content, _stale_overwrite_blocker, _stale_write_refusal) from tools.file_tools_read_tracking import ( _bump_consecutive, _cap_read_tracker_data, _check_file_staleness, _check_not_found_cache, - _mark_full_write_baseline, _mark_verification_stale, _patch_failure_lock, + _mark_full_write_baseline, _mark_verification_stale, _note_read_coverage, _patch_failure_lock, _patch_failure_tracker, _read_tracker, _read_tracker_lock, _record_not_found, _record_patch_failure, _reset_patch_failures, _task_data, _update_read_timestamp) @@ -458,7 +458,18 @@ def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task if len(result_dict["content"]) > max_chars: _apply_char_budget(result_dict, result_dict["content"], offset, total_lines, max_chars) if result_dict["content"]: - result_dict["content"] = redact_sensitive_text(result_dict["content"], file_read=True) + rendered = result_dict["content"] + result_dict["content"] = redact_sensitive_text(rendered, file_read=True) + redacted = result_dict["content"] != rendered + else: + redacted = False + if offset == 1 and not result_dict["truncated"] and not redacted: + # The whole document was shown, so a text-authorable format (.ipynb) + # may later be overwritten by write_file; the binary-container guard + # keeps refusing .docx/.xlsx/.pdf regardless of this baseline. + _mark_full_write_baseline(str(_resolved), task_id) + _update_read_timestamp(str(_resolved), task_id) + file_state.record_read(task_id, str(_resolved)) return json.dumps(result_dict, ensure_ascii=False) @@ -493,17 +504,21 @@ def _dedup_stub_or_block(task_data: dict, dedup_key: tuple, path: str) -> str: def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_str: str, offset: int, limit: int, dedup_key: tuple, *, partial: bool, - redacted: bool = False) -> int: + redacted: bool = False, end_line: int | None = None, + total_lines=None) -> int: """Bookkeeping after a real (non-stub) read; returns the consecutive-read count. Per-task tracker under the lock (stub counter, history, consecutive count, - mtime for dedup + staleness, and — for a full UNREDACTED read — the write_file - baseline: a redacted read returned a non-round-trippable ``«redacted:…»`` - sentinel, so it must not bless an overwrite that would persist the sentinel - into a credential file). Then OUTSIDE our lock (no nested locking): the - cross-agent registry, and the background-review read-mark (a FULL read of a - skill file counts like skill_view so a follow-up skill_manage(patch) is accepted). + mtime for dedup + staleness, page coverage, and the write_file baseline once + the task has seen every line UNREDACTED — in one page or by paging + contiguously through a file too big for one; a redacted page returned a + non-round-trippable ``«redacted:…»`` sentinel, so it must not bless an + overwrite that would persist the sentinel into a credential file). Then + OUTSIDE our lock (no nested locking): the cross-agent registry, and the + background-review read-mark (a FULL read of a skill file counts like + skill_view so a follow-up skill_manage(patch) is accepted). """ + complete = not partial with _read_tracker_lock: task_data["dedup_hits"].pop(dedup_key, None) task_data["dedup_generation_reads"].add(dedup_key) @@ -513,18 +528,21 @@ def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_s _mtime_now = os.path.getmtime(resolved_str) task_data["dedup"][dedup_key] = _mtime_now task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now + if partial and end_line is not None: + complete, redacted = _note_read_coverage( + task_data, resolved_str, _mtime_now, offset, end_line, total_lines, redacted) except OSError: pass - if not partial and not redacted: + if complete and not redacted: task_data.setdefault("full_write_baselines", set()).add(resolved_str) _cap_read_tracker_data(task_data) try: - file_state.record_read(task_id, resolved_str, partial=partial) + file_state.record_read(task_id, resolved_str, partial=not complete) except Exception: logger.debug("file_state.record_read failed", exc_info=True) - if not partial: + if complete: try: # Background-review read-before-write guard integration (#61521): when the self-improvement # review fork reads a skill file with read_file (now whitelisted dispatch-side), register the @@ -640,9 +658,16 @@ def read_file_tool(path: str, offset: int = 1, limit: int = DEFAULT_READ_LIMIT, "Consider reading only the section you need with offset and limit " "to keep context usage efficient.")) + total_lines = result_dict.get("total_lines") + if result_dict.get("truncated_by") == "bytes": + end_line = int(result_dict.get("next_offset", offset)) - 1 + else: + end_line = offset + limit - 1 + if isinstance(total_lines, int) and total_lines > 0: + end_line = min(end_line, total_lines) count = _record_successful_read(task_data, task_id, path, resolved_str, offset, limit, dedup_key, partial=(offset > 1) or bool(result_dict.get("truncated")), - redacted=redacted) + redacted=redacted, end_line=end_line, total_lines=total_lines) if count >= 4: return tool_error( f"BLOCKED: You have read this exact file region {count} times in a row. " diff --git a/tools/file_tools_read_tracking.py b/tools/file_tools_read_tracking.py index d5f8aed847..494378c0d7 100644 --- a/tools/file_tools_read_tracking.py +++ b/tools/file_tools_read_tracking.py @@ -7,10 +7,12 @@ call), ``read_history`` (diagnostics), ``dedup`` (key -> mtime; survives context compression), ``dedup_generation_reads`` (keys whose full content was served since the last compaction boundary; cleared on compression so one recovery read returns full content), ``dedup_hits`` (stub-loop breaker), ``read_timestamps`` -(staleness warnings), ``full_write_baselines`` (resolved paths whose whole-file -content this task saw via a full unredacted read_file or wrote via write_file; -required before write_file may overwrite an existing file — patch never -qualifies) and ``not_found`` (short-TTL negative cache). Every +(staleness warnings), ``read_coverage`` (per resolved path: the line ranges the +task has paged through at one mtime — contiguous pages that reach the last line +count as a whole-file read), ``full_write_baselines`` (resolved paths whose +whole-file content this task saw via unredacted read_file page(s) or wrote via +write_file; required before write_file may overwrite an existing file — patch +never qualifies) and ``not_found`` (short-TTL negative cache). Every container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small. """ @@ -19,7 +21,7 @@ import os import threading import time -from tools.file_state import _evict_oldest +from tools.file_state import _evict_oldest, _mtime_or_none from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task logger = logging.getLogger("tools.file_tools") @@ -48,7 +50,7 @@ def _task_data(task_id: str) -> dict: (search_tool / tests create partial entries). Lock must be held.""" task_data = _read_tracker.setdefault(task_id, { "last_key": None, "consecutive": 0, "read_history": set()}) - for key in ("dedup", "dedup_hits", "read_timestamps"): + for key in ("dedup", "dedup_hits", "read_timestamps", "read_coverage"): task_data.setdefault(key, {}) for key in ("dedup_generation_reads", "full_write_baselines"): task_data.setdefault(key, set()) @@ -85,6 +87,7 @@ def _cap_read_tracker_data(task_data: dict) -> None: ("dedup_hits", _DEDUP_CAP), ("dedup_generation_reads", _DEDUP_CAP), ("read_timestamps", _READ_TIMESTAMPS_CAP), + ("read_coverage", _READ_TIMESTAMPS_CAP), ("full_write_baselines", _FULL_WRITE_BASELINES_CAP), ("not_found", _NOT_FOUND_CAP)): container = task_data.get(key) @@ -156,7 +159,10 @@ def reset_file_dedup(task_id: str = None): files keep returning stubs instead of re-bloating the reclaimed context; the generation-read set is cleared so the FIRST unchanged read of each key after compaction returns full content the summary may have dropped. Stub-hit counters - are cleared so the hard block restarts fresh.""" + are cleared so the hard block restarts fresh. write_file baselines survive + exactly like the dedup map does — for files whose mtime still matches the + stamp this task recorded; a baseline whose file changed underneath is dropped + (the stat runs outside the lock so a hung mount cannot stall other tasks).""" with _read_tracker_lock: if task_id: targets = [_read_tracker[task_id]] if _read_tracker.get(task_id) else [] @@ -166,9 +172,13 @@ def reset_file_dedup(task_id: str = None): if "dedup_hits" in task_data: task_data["dedup_hits"].clear() task_data.setdefault("dedup_generation_reads", set()).clear() - # The summary may have dropped the exact bytes the baseline vouched - # for: a full overwrite needs a fresh read_file after compaction. - task_data.setdefault("full_write_baselines", set()).clear() + candidates = [(task_data, list(task_data.get("full_write_baselines", ())), + dict(task_data.get("read_timestamps", {}))) for task_data in targets] + for task_data, baselines, stamps in candidates: + changed = {p for p in baselines if _mtime_or_none(p) != stamps.get(p)} + if changed: + with _read_tracker_lock: + task_data.get("full_write_baselines", set()).difference_update(changed) def notify_other_tool_call(task_id: str = "default"): @@ -244,22 +254,54 @@ def _has_full_write_baseline(resolved: str, task_id: str) -> bool: return str(resolved) in task_data.get("full_write_baselines", set()) -def _check_file_staleness(filepath: str, task_id: str) -> str | None: - """Warn (don't block) when the file's mtime changed since this task last read it. - ``None`` when never read, fresh, or unstattable (a deleted file is the write's problem).""" +_READ_COVERAGE_RANGES_CAP = 256 + + +def _note_read_coverage(task_data: dict, resolved: str, mtime: float, start: int, end: int, + total_lines, redacted: bool) -> tuple[bool, bool]: + """Merge the page ``start..end`` into this task's coverage of *resolved* and return + ``(complete, redacted_any)``: whether pages taken at this same *mtime* now reach from + line 1 to *total_lines*, and whether any of them came back redacted. A file too large + for one read_file page (>2000 lines / the char budget) can only ever be seen this + way, so paging through it must count as a whole-file read. A new mtime restarts the + coverage (the earlier pages describe a file that no longer exists). Lock must be held.""" + coverage = task_data.setdefault("read_coverage", {}) + entry = coverage.get(resolved) + if entry is None or entry["mtime"] != mtime or len(entry["ranges"]) > _READ_COVERAGE_RANGES_CAP: + entry = coverage[resolved] = {"mtime": mtime, "ranges": [], "redacted": False} + entry["redacted"] = entry["redacted"] or redacted + merged: list[tuple[int, int]] = [] + for s, e in sorted(entry["ranges"] + [(start, end)]): + if merged and s <= merged[-1][1] + 1: + merged[-1] = (merged[-1][0], max(merged[-1][1], e)) + else: + merged.append((s, e)) + entry["ranges"] = merged + complete = (isinstance(total_lines, int) and total_lines > 0 + and merged[0][0] <= 1 and merged[0][1] >= total_lines) + return complete, entry["redacted"] + + +def _read_mtime_drifted(filepath: str, task_id: str) -> bool: + """True when the file's mtime changed since this task last read it. False when + never read, fresh, or unstattable (a deleted file is the write's problem).""" resolved = _resolved_or_none(filepath, task_id) if resolved is None: - return None + return False with _read_tracker_lock: task_data = _read_tracker.get(task_id) read_mtime = task_data.get("read_timestamps", {}).get(resolved) if task_data else None if read_mtime is None: - return None + return False try: - current_mtime = os.path.getmtime(resolved) + return os.path.getmtime(resolved) != read_mtime except OSError: - return None - if current_mtime != read_mtime: + return False + + +def _check_file_staleness(filepath: str, task_id: str) -> str | None: + """Warn (don't block) when the file's mtime changed since this task last read it.""" + if _read_mtime_drifted(filepath, task_id): return ( f"Warning: {filepath} was modified since you last read it " "(external edit or concurrent agent). The content you read may be " diff --git a/tools/file_tools_write_guards.py b/tools/file_tools_write_guards.py index a0e7d94c2a..82899a0657 100644 --- a/tools/file_tools_write_guards.py +++ b/tools/file_tools_write_guards.py @@ -17,7 +17,7 @@ from pathlib import Path from tools import file_state from tools.binary_extensions import has_opaque_document_extension, is_pdf_path from tools.file_tools_paths import _expand_tilde, _resolve_path_for_task -from tools.file_tools_read_tracking import _check_file_staleness, _has_full_write_baseline +from tools.file_tools_read_tracking import _has_full_write_baseline, _read_mtime_drifted # Prefixes matched after realpath. macOS: /private/var mirrors /var — block the # sensitive subtrees only; a blanket "/private/var/" refuses every temp-file @@ -451,15 +451,19 @@ def _stale_overwrite_blocker(filepath: str, resolved: str | None, task_id: str) Refuses BEFORE any disk mutation (the pre-#65604 warning arrived after the clobber): a sibling/external/partial-read staleness finding, or an existing file with no full-content baseline for this task (never read in full, read - redacted, only patched, or evicted by compaction). Net-new files, files this - task fully read or wrote, unresolvable paths and the file-state kill switch - all let the write proceed. + redacted, only patched). Net-new files, files this task fully read (in one + page or by paging contiguously to the last line) or wrote, unresolvable + paths and the file-state kill switch all let the write proceed. """ if file_state.guard_disabled(): return None - stale = (file_state.check_stale(task_id, resolved) if resolved else None) or _check_file_staleness(filepath, task_id) + stale = file_state.check_stale(task_id, resolved) if resolved else None if stale: return stale + if _read_mtime_drifted(filepath, task_id): + return ( + f"{filepath} was modified since you last read it (external edit or " + "concurrent agent). Re-read the file before writing.") if not resolved or _has_full_write_baseline(resolved, task_id): return None try: @@ -469,9 +473,11 @@ def _stale_overwrite_blocker(filepath: str, resolved: str | None, task_id: str) if not exists: return None return ( - f"{resolved} exists but this task has not read it in full (or only saw a " - "redacted/partial view). Read the file before using write_file so a stale " - "conversation copy cannot overwrite the current disk content.") + f"{resolved} exists but this task has not seen its full current content " + "(never read, only patched, or only a redacted/partial view). Read the " + "file — every page of it, if it needs offset/limit — or use patch for a " + "targeted edit; a stale conversation copy must not overwrite the current " + "disk content.") def _stale_write_refusal(filepath: str, reason: str, resolved: str | None = None) -> dict: @@ -480,10 +486,10 @@ def _stale_write_refusal(filepath: str, reason: str, resolved: str | None = None result = { "error": ( f"Refusing to overwrite {filepath}: {reason} " - "The file was NOT modified. Use read_file to reload the current " - "contents, merge the requested change, then call write_file again. " - "For small edits, prefer patch so existing unrelated changes are " - "preserved."), + "The file was NOT modified. Reload the current contents with read_file " + "(every page, for a file that needs offset/limit), merge the requested " + "change, then call write_file again. For small edits, prefer patch so " + "existing unrelated changes are preserved."), "stale_write_blocked": True, "path": filepath, }