fix: preserve known file baselines across partial rereads

This commit is contained in:
teknium1
2026-09-21 15:34:23 -07:00
committed by Teknium
parent eeb220d40c
commit e636aedebf
6 changed files with 205 additions and 49 deletions

View File

@@ -0,0 +1,76 @@
"""A narrower read must not discard unchanged whole-file knowledge."""
import json
import os
from tools.file_tools import clear_file_ops_cache
from tools.registry import registry
def _call(name, path, task_id, **arguments):
result = registry.dispatch(name, {"path": str(path), **arguments}, task_id=task_id)
assert isinstance(result, str)
return json.loads(result)
def test_partial_reread_keeps_an_unchanged_full_read_or_write(tmp_path):
original = "first\nsecond\nthird\n"
for source in ("read", "write", "pages"):
task = f"known-{source}"
path = tmp_path / f"{source}.txt"
try:
if source == "write":
assert "error" not in _call("write_file", path, task, content=original)
else:
path.write_text(original, encoding="utf-8")
if source == "pages":
for offset in (1, 2, 3):
assert "error" not in _call("read_file", path, task, offset=offset, limit=1)
else:
assert "error" not in _call("read_file", path, task)
assert "error" not in _call("read_file", path, task, offset=2, limit=2)
written = _call("write_file", path, task, content="replacement\n")
assert "error" not in written, (source, written)
assert path.read_text(encoding="utf-8") == "replacement\n"
finally:
clear_file_ops_cache(task)
def test_partial_read_cannot_refresh_a_changed_full_baseline(tmp_path):
original = "first\nsecond\nthird\n"
for source in ("write", "pages"):
path = tmp_path / f"changed-{source}.txt"
task = f"snapshot-{source}"
try:
if source == "write":
assert "error" not in _call("write_file", path, task, content=original)
else:
path.write_text(original, encoding="utf-8")
assert "error" not in _call("read_file", path, task, offset=1, limit=1)
stamp = path.stat()
path.write_text("other\nsecond\nthird\n", encoding="utf-8")
# mtime alone cannot identify bytes: editors/copy tools can preserve it.
os.utime(path, ns=(stamp.st_atime_ns, stamp.st_mtime_ns))
assert "error" not in _call("read_file", path, task, offset=2, limit=2)
refused = _call("write_file", path, task, content="replacement\n")
assert refused.get("stale_write_blocked"), (source, refused)
assert path.read_text(encoding="utf-8") == "other\nsecond\nthird\n"
# Reading all of the new version is recovery, not a bypass.
assert "error" not in _call("read_file", path, task)
assert "error" not in _call("write_file", path, task, content="merged\n")
assert path.read_text(encoding="utf-8") == "merged\n"
finally:
clear_file_ops_cache(task)
# A page that hides part of a line never supplies whole-file knowledge.
from tools.tool_output_limits import get_max_line_length
path = tmp_path / "clamped.txt"
path.write_text("x" * (get_max_line_length() + 10) + "\nlast\n", encoding="utf-8")
try:
assert "error" not in _call("read_file", path, "clamped")
refused = _call("write_file", path, "clamped", content="replacement\n")
assert refused.get("stale_write_blocked"), refused
assert path.read_text(encoding="utf-8").startswith("x" * (get_max_line_length() + 10))
finally:
clear_file_ops_cache("clamped")

View File

@@ -581,9 +581,13 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
hint_parts.append( hint_parts.append(
f"Use offset={end_line + 1} to continue reading " f"Use offset={end_line + 1} to continue reading "
f"(showing {offset}-{end_line} of {total_lines} lines)") f"(showing {offset}-{end_line} of {total_lines} lines)")
from tools.tool_output_limits import get_max_line_length
max_line_length = get_max_line_length()
truncated_lines = any(len(line) > max_line_length for line in content.split('\n'))
return ReadResult( return ReadResult(
content=self._add_line_numbers(content, offset), total_lines=total_lines, content=self._add_line_numbers(content, offset), total_lines=total_lines,
file_size=file_size, truncated=truncated, hint=" ".join(hint_parts)) file_size=file_size, truncated=truncated, hint=" ".join(hint_parts),
truncated_lines=True if truncated_lines else None)
def read_file(self, path: str, offset: int = 1, limit: int = 2000) -> ReadResult: def read_file(self, path: str, offset: int = 1, limit: int = 2000) -> ReadResult:
"""Read a file with pagination, binary detection, and line numbers. """Read a file with pagination, binary detection, and line numbers.
@@ -724,6 +728,7 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
kept = bytearray() # first ``clamp`` bytes of that line kept = bytearray() # first ``clamp`` bytes of that line
have_partial = False # that line has bytes but no newline yet have_partial = False # that line has bytes but no newline yet
last_byte = b"" last_byte = b""
digest = hashlib.sha256()
try: try:
with open(full, "rb") as fh: with open(full, "rb") as fh:
sample = fh.read(1000) sample = fh.read(1000)
@@ -735,6 +740,7 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
chunk = fh.read(1 << 20) chunk = fh.read(1 << 20)
if not chunk: if not chunk:
break break
digest.update(chunk)
last_byte = chunk[-1:] last_byte = chunk[-1:]
if lineno > end_line: if lineno > end_line:
# Past the window: only the line count and trailing byte # Past the window: only the line count and trailing byte
@@ -767,10 +773,12 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
page.append(bytes(kept) + b"\n") page.append(bytes(kept) + b"\n")
read_output = _strip_terminal_fence_leaks(b"".join(page).decode("utf-8", errors="replace")) read_output = _strip_terminal_fence_leaks(b"".join(page).decode("utf-8", errors="replace"))
return self._assemble_read_result( result = self._assemble_read_result(
read_output, offset=offset, end_line=end_line, total_lines=total_lines, read_output, offset=offset, end_line=end_line, total_lines=total_lines,
file_size=file_size, file_size=file_size,
file_ends_with_newline=(last_byte == b"\n") if file_size else None) file_ends_with_newline=(last_byte == b"\n") if file_size else None)
result._snapshot = (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns, digest.digest())
return result
@staticmethod @staticmethod
def _image_redirect_result(file_size: int) -> ReadResult: def _image_redirect_result(file_size: int) -> ReadResult:
@@ -924,9 +932,13 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
f"Note: offset {offset} is beyond the end of the file " f"Note: offset {offset} is beyond the end of the file "
f"({total_lines} lines total). Retry with offset <= " f"({total_lines} lines total). Retry with offset <= "
f"{total_lines}.")) f"{total_lines}."))
from tools.tool_output_limits import get_max_line_length
max_line_length = get_max_line_length()
truncated_lines = any(len(line) > max_line_length for line in read_output.split('\n'))
return ReadResult( return ReadResult(
content=self._add_line_numbers(read_output, offset), total_lines=total_lines, content=self._add_line_numbers(read_output, offset), total_lines=total_lines,
file_size=file_size, truncated=truncated, hint=hint) file_size=file_size, truncated=truncated, hint=hint,
truncated_lines=True if truncated_lines else None)
# Confusable characters seen in real filenames, collapsed after NFC. # Confusable characters seen in real filenames, collapsed after NFC.
_CONFUSABLES = ( _CONFUSABLES = (
@@ -1295,6 +1307,7 @@ class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
lsp_diagnostics = self._maybe_lsp_diagnostics(path, pre_content=pre_content, post_content=content) or None lsp_diagnostics = self._maybe_lsp_diagnostics(path, pre_content=pre_content, post_content=content) or None
return WriteResult( return WriteResult(
bytes_written=len(content_bytes), dirs_created=dirs_created, verified=content_verified, bytes_written=len(content_bytes), dirs_created=dirs_created, verified=content_verified,
_content_sha256=hashlib.sha256(content_bytes).hexdigest(),
lint=lint_result.to_dict() if lint_result else None, lsp_diagnostics=lsp_diagnostics) lint=lint_result.to_dict() if lint_result else None, lsp_diagnostics=lsp_diagnostics)
# --- PATCH (replace mode) ----------------------------------------------- # --- PATCH (replace mode) -----------------------------------------------

View File

@@ -17,6 +17,7 @@ class ReadResult:
total_lines: int = 0 total_lines: int = 0
file_size: int = 0 file_size: int = 0
truncated: bool = False truncated: bool = False
truncated_lines: Optional[bool] = None
hint: Optional[str] = None hint: Optional[str] = None
is_binary: bool = False is_binary: bool = False
is_image: bool = False is_image: bool = False
@@ -25,9 +26,10 @@ class ReadResult:
dimensions: Optional[str] = None # For images: "WIDTHxHEIGHT" dimensions: Optional[str] = None # For images: "WIDTHxHEIGHT"
error: Optional[str] = None error: Optional[str] = None
similar_files: List[str] = field(default_factory=list) similar_files: List[str] = field(default_factory=list)
_snapshot: Optional[tuple] = None
def to_dict(self) -> dict: def to_dict(self) -> dict:
return {k: v for k, v in self.__dict__.items() if v is not None and v != []} return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and v is not None and v != []}
@dataclass @dataclass
@@ -38,6 +40,7 @@ class WriteResult:
# True when the on-disk sha256 matched the intended content; None when the # True when the on-disk sha256 matched the intended content; None when the
# backend couldn't verify (no sha256sum). A mismatch is a hard error, never a flag. # backend couldn't verify (no sha256sum). A mismatch is a hard error, never a flag.
verified: Optional[bool] = None verified: Optional[bool] = None
_content_sha256: Optional[str] = None
lint: Optional[Dict[str, Any]] = None lint: Optional[Dict[str, Any]] = None
# LSP semantic diagnostics, kept separate from ``lint`` (syntax) so the model # LSP semantic diagnostics, kept separate from ``lint`` (syntax) so the model
# reads the two as independent signals. None when LSP is off/inapplicable. # reads the two as independent signals. None when LSP is off/inapplicable.
@@ -46,7 +49,7 @@ class WriteResult:
warning: Optional[str] = None warning: Optional[str] = None
def to_dict(self) -> dict: def to_dict(self) -> dict:
return {k: v for k, v in self.__dict__.items() if v is not None} return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and v is not None}
@dataclass @dataclass

View File

@@ -35,6 +35,7 @@ from tools.file_tools_write_guards import (
_is_internal_file_tool_content, _stale_overwrite_blocker, _stale_write_refusal) _is_internal_file_tool_content, _stale_overwrite_blocker, _stale_write_refusal)
from tools.file_tools_read_tracking import ( from tools.file_tools_read_tracking import (
_bump_consecutive, _cap_read_tracker_data, _check_file_staleness, _check_not_found_cache, _bump_consecutive, _cap_read_tracker_data, _check_file_staleness, _check_not_found_cache,
_file_metadata, _file_version,
_mark_full_write_baseline, _mark_verification_stale, _note_read_coverage, _patch_failure_lock, _mark_full_write_baseline, _mark_verification_stale, _note_read_coverage, _patch_failure_lock,
_patch_failure_tracker, _read_tracker, _read_tracker_lock, _record_not_found, _patch_failure_tracker, _read_tracker, _read_tracker_lock, _record_not_found,
_record_patch_failure, _reset_patch_failures, _task_data, _update_read_timestamp) _record_patch_failure, _reset_patch_failures, _task_data, _update_read_timestamp)
@@ -104,6 +105,7 @@ def _apply_char_budget(result_dict: dict, content: str, offset: int, total_lines
f"{lines_kept} line(s) (showing lines {offset}-{next_offset - 1} of " f"{lines_kept} line(s) (showing lines {offset}-{next_offset - 1} of "
f"{total_lines}). Use offset={next_offset} to continue.") f"{total_lines}). Use offset={next_offset} to continue.")
if len(trimmed.split("\n", 1)[0]) >= max_chars: if len(trimmed.split("\n", 1)[0]) >= max_chars:
result_dict["truncated_lines"] = True
result_dict["hint"] += ( result_dict["hint"] += (
" Note: the first line alone exceeded the budget and was " " Note: the first line alone exceeded the budget and was "
"clamped mid-line; its remainder is not retrievable via offset.") "clamped mid-line; its remainder is not retrievable via offset.")
@@ -455,6 +457,9 @@ def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task
total_lines = len(lines) total_lines = len(lines)
end_line = offset + limit - 1 end_line = offset + limit - 1
page_text = "\n".join(lines[offset - 1:end_line]) page_text = "\n".join(lines[offset - 1:end_line])
from tools.tool_output_limits import get_max_line_length
max_line_length = get_max_line_length()
truncated_lines = any(len(line) > max_line_length for line in page_text.split('\n'))
result_dict = { result_dict = {
"content": file_ops._add_line_numbers(page_text, offset) if page_text else "", "content": file_ops._add_line_numbers(page_text, offset) if page_text else "",
"total_lines": total_lines, "total_lines": total_lines,
@@ -476,7 +481,8 @@ def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task
redacted = result_dict["content"] != rendered redacted = result_dict["content"] != rendered
else: else:
redacted = False redacted = False
if offset == 1 and not result_dict["truncated"] and not redacted: if (offset == 1 and not result_dict["truncated"] and not redacted
and not truncated_lines and not result_dict.get("truncated_lines")):
# The whole document was shown, so a text-authorable format (.ipynb) # The whole document was shown, so a text-authorable format (.ipynb)
# may later be overwritten by write_file; the binary-container guard # may later be overwritten by write_file; the binary-container guard
# keeps refusing .docx/.xlsx/.pdf regardless of this baseline. # keeps refusing .docx/.xlsx/.pdf regardless of this baseline.
@@ -522,7 +528,7 @@ def _dedup_stub_or_block(task_data: dict, dedup_key: tuple, path: str) -> str:
def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_str: str, def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_str: str,
offset: int, limit: int, dedup_key: tuple, *, partial: bool, offset: int, limit: int, dedup_key: tuple, *, partial: bool,
redacted: bool = False, end_line: int | None = None, redacted: bool = False, end_line: int | None = None,
total_lines=None) -> int: total_lines=None, version_before=None, snapshot=None) -> int:
"""Bookkeeping after a real (non-stub) read; returns the consecutive-read count. """Bookkeeping after a real (non-stub) read; returns the consecutive-read count.
Per-task tracker under the lock (stub counter, history, consecutive count, Per-task tracker under the lock (stub counter, history, consecutive count,
@@ -535,7 +541,9 @@ def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_s
background-review read-mark (a FULL read of a skill file counts like background-review read-mark (a FULL read of a skill file counts like
skill_view so a follow-up skill_manage(patch) is accepted). skill_view so a follow-up skill_manage(patch) is accepted).
""" """
complete = not partial version = (snapshot or _file_version(resolved_str)) if version_before is not None else None
stable = version is not None and version[:-1] == version_before == _file_metadata(resolved_str)
complete = False
with _read_tracker_lock: with _read_tracker_lock:
task_data["dedup_hits"].pop(dedup_key, None) task_data["dedup_hits"].pop(dedup_key, None)
task_data["dedup_generation_reads"].add(dedup_key) task_data["dedup_generation_reads"].add(dedup_key)
@@ -543,15 +551,28 @@ def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_s
count = _bump_consecutive(task_data, ("read", path, offset, limit)) count = _bump_consecutive(task_data, ("read", path, offset, limit))
try: try:
_mtime_now = os.path.getmtime(resolved_str) _mtime_now = os.path.getmtime(resolved_str)
task_data["dedup"][dedup_key] = _mtime_now
task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now
if partial and end_line is not None:
complete, redacted = _note_read_coverage(
task_data, resolved_str, _mtime_now, offset, end_line, total_lines, redacted)
except OSError: except OSError:
pass pass
if complete and not redacted: baselines = task_data["full_write_baselines"]
task_data.setdefault("full_write_baselines", set()).add(resolved_str) if stable and version is not None and count < 4:
task_data["dedup"][dedup_key] = version_before
# A narrower view does not undo knowledge of these same bytes. Do
# not revive a baseline after a partial read of a different version.
complete = baselines.get(resolved_str) == version
if not complete:
complete = not partial
if partial and end_line is not None:
complete, redacted = _note_read_coverage(
task_data, resolved_str, version, offset, end_line, total_lines, redacted)
complete = complete and not redacted
if complete:
baselines[resolved_str] = version
if not complete:
baselines.pop(resolved_str, None)
if not stable or count >= 4:
task_data["dedup"].pop(dedup_key, None)
task_data["dedup_generation_reads"].discard(dedup_key)
_cap_read_tracker_data(task_data) _cap_read_tracker_data(task_data)
try: try:
@@ -642,27 +663,27 @@ def read_file_tool(path: str, offset: int = 1, limit: int = DEFAULT_READ_LIMIT,
dedup_key = (resolved_str, offset, limit) dedup_key = (resolved_str, offset, limit)
with _read_tracker_lock: with _read_tracker_lock:
task_data = _task_data(task_id) task_data = _task_data(task_id)
cached_mtime = task_data["dedup"].get(dedup_key) cached_version = task_data["dedup"].get(dedup_key)
# First unchanged read after a compaction boundary serves full content # First unchanged read after a compaction boundary serves full content
# (the summary may have dropped exact bytes); later ones get the stub. # (the summary may have dropped exact bytes); later ones get the stub.
content_served_in_generation = dedup_key in task_data["dedup_generation_reads"] content_served_in_generation = dedup_key in task_data["dedup_generation_reads"]
# Same rule as skill_view: the review fork shares the parent's task_id and its # Same rule as skill_view: the review fork shares the parent's task_id and its
# read-before-write guard needs a real read, which the stub path never records (#95976). # read-before-write guard needs a real read, which the stub path never records (#95976).
if cached_mtime is not None and not is_background_review(): file_ops = _get_file_ops(task_id)
try: version_before = _file_metadata(resolved_str) if _file_ops_uses_host_paths(file_ops) else None
if os.path.getmtime(resolved_str) == cached_mtime and content_served_in_generation: if (cached_version is not None and not is_background_review()
return _dedup_stub_or_block(task_data, dedup_key, path) and version_before == cached_version and content_served_in_generation):
except OSError: return _dedup_stub_or_block(task_data, dedup_key, path)
pass # stat failed — fall through to full read
result = _get_file_ops(task_id).read_file(path, offset, limit) result = file_ops.read_file(path, offset, limit)
result_dict = result.to_dict() result_dict = result.to_dict()
# Cache a not-found result for retries. Deliberately NO early return: # Failed reads cannot establish whole-file knowledge.
# error results still flow through the tracking below unchanged.
_err = result_dict.get("error") or "" _err = result_dict.get("error") or ""
if isinstance(_err, str) and _err.startswith("File not found:"): if isinstance(_err, str) and _err.startswith("File not found:"):
_record_not_found("read", resolved_str, task_id, json.dumps(result_dict, ensure_ascii=False)) _record_not_found("read", resolved_str, task_id, json.dumps(result_dict, ensure_ascii=False))
if _err or result_dict.get("is_binary"):
return json.dumps(result_dict, ensure_ascii=False)
# Char budget on the FORMATTED content (what enters context), BEFORE # Char budget on the FORMATTED content (what enters context), BEFORE
# redaction (skip the regex pass on huge content); truncate gracefully # redaction (skip the regex pass on huge content); truncate gracefully
@@ -705,7 +726,10 @@ def read_file_tool(path: str, offset: int = 1, limit: int = DEFAULT_READ_LIMIT,
end_line = min(end_line, total_lines) end_line = min(end_line, total_lines)
count = _record_successful_read(task_data, task_id, path, resolved_str, offset, limit, count = _record_successful_read(task_data, task_id, path, resolved_str, offset, limit,
dedup_key, partial=(offset > 1) or bool(result_dict.get("truncated")), dedup_key, partial=(offset > 1) or bool(result_dict.get("truncated")),
redacted=redacted, end_line=end_line, total_lines=total_lines) redacted=redacted or bool(result_dict.get("truncated_lines")),
end_line=end_line, total_lines=total_lines,
version_before=version_before,
snapshot=getattr(result, "_snapshot", None))
if count >= 4: if count >= 4:
return tool_error( return tool_error(
f"BLOCKED: You have read this exact file region {count} times in a row. " f"BLOCKED: You have read this exact file region {count} times in a row. "
@@ -865,7 +889,8 @@ def write_file_tool(path: str, content: str, task_id: str = "default",
return json.dumps(_stale_write_refusal(path, blocker, _resolved), ensure_ascii=False) return json.dumps(_stale_write_refusal(path, blocker, _resolved), ensure_ascii=False)
warnings = _edit_warnings([path], path_to_resolved, task_id) warnings = _edit_warnings([path], path_to_resolved, task_id)
rewrite_hint = _whole_file_rewrite_hint(task_id, _resolved, content) rewrite_hint = _whole_file_rewrite_hint(task_id, _resolved, content)
result_dict = _get_file_ops(task_id).write_file(_resolved or path, content).to_dict() result = _get_file_ops(task_id).write_file(_resolved or path, content)
result_dict = result.to_dict()
if warnings: if warnings:
result_dict["_warning"] = warnings[0] result_dict["_warning"] = warnings[0]
if rewrite_hint and not result_dict.get("error"): if rewrite_hint and not result_dict.get("error"):
@@ -881,7 +906,7 @@ def write_file_tool(path: str, content: str, task_id: str = "default",
result_dict["files_modified"] = [_resolved] result_dict["files_modified"] = [_resolved]
# Own write = current whole-file content: consecutive # Own write = current whole-file content: consecutive
# same-task writes stay unblocked. patch never does this. # same-task writes stay unblocked. patch never does this.
_mark_full_write_baseline(_resolved, task_id) _mark_full_write_baseline(_resolved, task_id, getattr(result, "_content_sha256", None))
_note_edited(task_id, [path], path_to_resolved, session_id) _note_edited(task_id, [path], path_to_resolved, session_id)
return json.dumps(result_dict, ensure_ascii=False) return json.dumps(result_dict, ensure_ascii=False)
except Exception as e: except Exception as e:

View File

@@ -3,12 +3,12 @@
Process-lifetime state behind read_file/search_files/write_file/patch. Process-lifetime state behind read_file/search_files/write_file/patch.
Per task_id ``_read_tracker`` Per task_id ``_read_tracker``
stores: ``last_key``/``consecutive`` (loop detection; reset by any OTHER tool stores: ``last_key``/``consecutive`` (loop detection; reset by any OTHER tool
call), ``read_history`` (diagnostics), ``dedup`` (key -> mtime; survives context call), ``read_history`` (diagnostics), ``dedup`` (key -> file metadata; survives context
compression), ``dedup_generation_reads`` (keys whose full content was served since compression), ``dedup_generation_reads`` (keys whose full content was served since
the last compaction boundary; cleared on compression so one recovery read returns the last compaction boundary; cleared on compression so one recovery read returns
full content), ``dedup_hits`` (stub-loop breaker), ``read_timestamps`` full content), ``dedup_hits`` (stub-loop breaker), ``read_timestamps``
(staleness warnings), ``read_coverage`` (per resolved path: the line ranges the (staleness warnings), ``read_coverage`` (per resolved path: the line ranges the
task has paged through at one mtime — contiguous pages that reach the last line task has paged through at one file version — contiguous pages that reach the last line
count as a whole-file read), ``full_write_baselines`` (resolved paths whose count as a whole-file read), ``full_write_baselines`` (resolved paths whose
whole-file content this task saw via unredacted read_file page(s) or wrote via whole-file content this task saw via unredacted read_file page(s) or wrote via
write_file; required before write_file may overwrite an existing file — patch write_file; required before write_file may overwrite an existing file — patch
@@ -16,12 +16,14 @@ never qualifies) and ``not_found`` (short-TTL negative cache). Every
container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small. container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small.
""" """
import hashlib
import logging import logging
import os import os
import stat
import threading import threading
import time import time
from tools.file_state import _evict_oldest, _mtime_or_none from tools.file_state import _evict_oldest
from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task
logger = logging.getLogger("tools.file_tools") logger = logging.getLogger("tools.file_tools")
@@ -50,10 +52,9 @@ def _task_data(task_id: str) -> dict:
(search_tool / tests create partial entries). Lock must be held.""" (search_tool / tests create partial entries). Lock must be held."""
task_data = _read_tracker.setdefault(task_id, { task_data = _read_tracker.setdefault(task_id, {
"last_key": None, "consecutive": 0, "read_history": set()}) "last_key": None, "consecutive": 0, "read_history": set()})
for key in ("dedup", "dedup_hits", "read_timestamps", "read_coverage"): for key in ("dedup", "dedup_hits", "read_timestamps", "read_coverage", "full_write_baselines"):
task_data.setdefault(key, {}) task_data.setdefault(key, {})
for key in ("dedup_generation_reads", "full_write_baselines"): task_data.setdefault("dedup_generation_reads", set())
task_data.setdefault(key, set())
return task_data return task_data
@@ -155,13 +156,14 @@ def _bump_consecutive(task_data: dict, key: tuple) -> int:
def reset_file_dedup(task_id: str = None): def reset_file_dedup(task_id: str = None):
"""Advance the read-dedup generation after context compression (one task, or all """Advance the read-dedup generation after context compression (one task, or all
when ``task_id`` is None). The per-key ``dedup`` mtime map is PRESERVED so unchanged when ``task_id`` is None). The per-key ``dedup`` metadata map is PRESERVED so unchanged
files keep returning stubs instead of re-bloating the reclaimed context; the files keep returning stubs instead of re-bloating the reclaimed context; the
generation-read set is cleared so the FIRST unchanged read of each key after generation-read set is cleared so the FIRST unchanged read of each key after
compaction returns full content the summary may have dropped. Stub-hit counters compaction returns full content the summary may have dropped. Stub-hit counters
are cleared so the hard block restarts fresh. write_file baselines survive are cleared so the hard block restarts fresh. write_file baselines survive
exactly like the dedup map does — for files whose mtime still matches the exactly like the dedup map does — while the file metadata still matches the
stamp this task recorded; a baseline whose file changed underneath is dropped stamp this task recorded; byte identity is checked before overwriting. A baseline
whose file changed underneath is dropped
(the stat runs outside the lock so a hung mount cannot stall other tasks).""" (the stat runs outside the lock so a hung mount cannot stall other tasks)."""
with _read_tracker_lock: with _read_tracker_lock:
if task_id: if task_id:
@@ -172,13 +174,15 @@ def reset_file_dedup(task_id: str = None):
if "dedup_hits" in task_data: if "dedup_hits" in task_data:
task_data["dedup_hits"].clear() task_data["dedup_hits"].clear()
task_data.setdefault("dedup_generation_reads", set()).clear() task_data.setdefault("dedup_generation_reads", set()).clear()
candidates = [(task_data, list(task_data.get("full_write_baselines", ())), candidates = [(task_data, dict(task_data.get("full_write_baselines", {})))
dict(task_data.get("read_timestamps", {}))) for task_data in targets] for task_data in targets]
for task_data, baselines, stamps in candidates: for task_data, baselines in candidates:
changed = {p for p in baselines if _mtime_or_none(p) != stamps.get(p)} changed = {p for p, version in baselines.items() if _file_metadata(p) != version[:-1]}
if changed: if changed:
with _read_tracker_lock: with _read_tracker_lock:
task_data.get("full_write_baselines", set()).difference_update(changed) for p in changed:
if task_data["full_write_baselines"].get(p) == baselines[p]:
task_data["full_write_baselines"].pop(p, None)
def notify_other_tool_call(task_id: str = "default"): def notify_other_tool_call(task_id: str = "default"):
@@ -238,37 +242,70 @@ def _update_read_timestamp(filepath: str, task_id: str) -> None:
_cap_read_tracker_data(task_data) _cap_read_tracker_data(task_data)
def _mark_full_write_baseline(resolved: str, task_id: str) -> None: def _file_metadata(resolved: str) -> tuple | None:
try:
st = os.stat(resolved)
return st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns
except OSError:
return None
def _file_version(resolved: str) -> tuple | None:
"""A byte snapshot, not just mtime (editors/copy tools can preserve that)."""
try:
if not stat.S_ISREG(os.stat(resolved).st_mode):
return None
fd = os.open(resolved, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(fd, "rb") as stream:
before = os.fstat(stream.fileno())
if not stat.S_ISREG(before.st_mode):
return None
digest = hashlib.file_digest(stream, "sha256").digest()
after = os.stat(resolved)
fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns")
version = tuple(getattr(before, name) for name in fields)
if version == tuple(getattr(after, name) for name in fields):
return (*version, digest)
return None
except OSError:
return None
def _mark_full_write_baseline(resolved: str, task_id: str, expected_sha256: str | None = None) -> None:
"""Record that *task_id* saw the whole current content of *resolved* (full """Record that *task_id* saw the whole current content of *resolved* (full
unredacted read_file, or its own successful write_file), so a later unredacted read_file, or its own successful write_file), so a later
write_file may replace the file. Acquires the lock itself.""" write_file may replace the file. Acquires the lock itself."""
version = _file_version(resolved)
if version is None or (expected_sha256 is not None and version[-1].hex() != expected_sha256):
return
with _read_tracker_lock: with _read_tracker_lock:
task_data = _task_data(task_id) task_data = _task_data(task_id)
task_data["full_write_baselines"].add(str(resolved)) task_data["full_write_baselines"][str(resolved)] = version
_cap_read_tracker_data(task_data) _cap_read_tracker_data(task_data)
def _has_full_write_baseline(resolved: str, task_id: str) -> bool: def _has_full_write_baseline(resolved: str, task_id: str) -> bool:
with _read_tracker_lock: with _read_tracker_lock:
task_data = _read_tracker.get(task_id) or {} task_data = _read_tracker.get(task_id) or {}
return str(resolved) in task_data.get("full_write_baselines", set()) baseline = task_data.get("full_write_baselines", {}).get(str(resolved))
return baseline is not None and _file_version(resolved) == baseline
_READ_COVERAGE_RANGES_CAP = 256 _READ_COVERAGE_RANGES_CAP = 256
def _note_read_coverage(task_data: dict, resolved: str, mtime: float, start: int, end: int, def _note_read_coverage(task_data: dict, resolved: str, version: tuple, start: int, end: int,
total_lines, redacted: bool) -> tuple[bool, bool]: total_lines, redacted: bool) -> tuple[bool, bool]:
"""Merge the page ``start..end`` into this task's coverage of *resolved* and return """Merge the page ``start..end`` into this task's coverage of *resolved* and return
``(complete, redacted_any)``: whether pages taken at this same *mtime* now reach from ``(complete, redacted_any)``: whether pages taken at this same *version* now reach from
line 1 to *total_lines*, and whether any of them came back redacted. A file too large line 1 to *total_lines*, and whether any of them came back redacted. A file too large
for one read_file page (>2000 lines / the char budget) can only ever be seen this for one read_file page (>2000 lines / the char budget) can only ever be seen this
way, so paging through it must count as a whole-file read. A new mtime restarts the way, so paging through it must count as a whole-file read. A new version restarts the
coverage (the earlier pages describe a file that no longer exists). Lock must be held.""" coverage (the earlier pages describe a file that no longer exists). Lock must be held."""
coverage = task_data.setdefault("read_coverage", {}) coverage = task_data.setdefault("read_coverage", {})
entry = coverage.get(resolved) entry = coverage.get(resolved)
if entry is None or entry["mtime"] != mtime or len(entry["ranges"]) > _READ_COVERAGE_RANGES_CAP: if entry is None or entry["version"] != version or len(entry["ranges"]) > _READ_COVERAGE_RANGES_CAP:
entry = coverage[resolved] = {"mtime": mtime, "ranges": [], "redacted": False} entry = coverage[resolved] = {"version": version, "ranges": [], "redacted": False}
entry["redacted"] = entry["redacted"] or redacted entry["redacted"] = entry["redacted"] or redacted
merged: list[tuple[int, int]] = [] merged: list[tuple[int, int]] = []
for s, e in sorted(entry["ranges"] + [(start, end)]): for s, e in sorted(entry["ranges"] + [(start, end)]):

View File

@@ -117,6 +117,8 @@ Scoped to the Feishu document-comment handler. Drives comment read/write operati
| `search_files` | Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents. Content search (target='content'): Regex search inside files. Output modes: full matches with line… | — | | `search_files` | Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents. Content search (target='content'): Regex search inside files. Output modes: full matches with line… | — |
| `write_file` | Write content to a file, completely replacing existing content. Use this instead of echo/cat heredoc in terminal. Creates parent directories automatically. OVERWRITES the entire file — use 'patch' for targeted edits. For an existing file, call read_file first: write_file refuses (file untouched) when the task has no current full read/write of the file or the file changed on disk since — on refusal, read_file, merge, retry. Auto-runs syntax checks on .py/.json/.yaml/.toml and other linted languages; only NEW errors introduced by the write are surfaced. | — | | `write_file` | Write content to a file, completely replacing existing content. Use this instead of echo/cat heredoc in terminal. Creates parent directories automatically. OVERWRITES the entire file — use 'patch' for targeted edits. For an existing file, call read_file first: write_file refuses (file untouched) when the task has no current full read/write of the file or the file changed on disk since — on refusal, read_file, merge, retry. Auto-runs syntax checks on .py/.json/.yaml/.toml and other linted languages; only NEW errors introduced by the write are surfaced. | — |
For local files, a full unredacted read (including all pages of the same file version) or a successful `write_file` supplies a whole-file baseline. Reading a smaller region afterward does not discard that baseline while the bytes remain unchanged. A changed file, an unread file, or a view with hidden/redacted or clamped content still needs a full current read before replacement; `patch` remains available for targeted edits. Writes made through terminal commands or `execute_code` do not establish a `write_file` baseline.
## `homeassistant` toolset ## `homeassistant` toolset
| Tool | Description | Requires environment | | Tool | Description | Requires environment |