feat(file-ops): name the binary type in read_file refusals (magic-byte sniff)

'Binary file - use appropriate tools' names a recovery the model may
not have — in a file-only toolset it thrashed for 41 turns / 178 tool
calls / 1.5M tokens on a PNG-behind-.txt (readtool eval, qwen3.8-max)
hunting for tools that did not exist. Name the type instead: 25 magic
signatures (images, archives, executables, media, SQLite), ftyp check
for ISO media, size in human units. 'Binary file (PNG image data,
4.1 KB) - cannot display as text.' answers what-is-this in one read.

Both ShellFileOperations refusal sites (read_file + read_file_raw) use
the shared describe_binary_file(); the extension-based guard keeps its
extension message (an extension is a claim; only sniffed content earns
a type name).
This commit is contained in:
teknium1
2026-08-10 10:39:45 -07:00
committed by Teknium
parent 03da1606bc
commit 1362ffc7d2
3 changed files with 160 additions and 3 deletions

View File

@@ -0,0 +1,90 @@
"""Tests for magic-byte type disclosure in the binary-file refusal.
"Binary file — use appropriate tools" names a recovery the model may not
have (file-only toolsets), so it thrashes. Naming the TYPE answers
what-is-this in one read: "Binary file (PNG image data, 4.0 KB)".
"""
import json
import random
import pytest
from tools.file_operations import describe_binary_file, identify_binary_bytes
from tools.file_tools import read_file_tool
_RNG = random.Random(20260810)
def _noise(n: int) -> bytes:
return bytes(_RNG.getrandbits(8) for _ in range(n))
class TestIdentifyBinaryBytes:
@pytest.mark.parametrize(
"prefix,expected",
[
(b"\x89PNG\r\n\x1a\n", "PNG image data"),
(b"\xff\xd8\xff", "JPEG image data"),
(b"%PDF-", "PDF document"),
(b"PK\x03\x04", "ZIP archive"),
(b"\x1f\x8b", "gzip compressed data"),
(b"\x7fELF", "ELF executable"),
(b"MZ", "Windows PE executable"),
(b"SQLite format 3\x00", "SQLite database"),
],
)
def test_known_signatures(self, prefix, expected):
assert expected in identify_binary_bytes(prefix + _noise(64))
def test_unknown_binary(self):
assert identify_binary_bytes(b"\x00\x01\x02" * 100) == "unknown binary"
def test_empty_sample(self):
assert identify_binary_bytes(b"") == "unknown binary"
def test_iso_media_requires_ftyp(self):
# Three NULs alone are too weak; require ftyp at offset 4.
assert identify_binary_bytes(b"\x00\x00\x00\x18ftypmp42" + _noise(32)).startswith("ISO media")
assert identify_binary_bytes(b"\x00\x00\x00\x18AAAA" + _noise(32)) == "unknown binary"
class TestDescribeBinaryFile:
def test_size_units(self):
assert "512 bytes" in describe_binary_file(b"\x7fELF", 512)
assert "4.0 KB" in describe_binary_file(b"\x7fELF", 4096)
assert "2.0 MB" in describe_binary_file(b"\x7fELF", 2 * 1024 * 1024)
def test_none_sample(self):
assert "unknown binary" in describe_binary_file(None, 100)
class TestReadFileBinaryDisclosure:
def test_lying_extension_names_real_type(self, tmp_path, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
p = tmp_path / "data.txt" # extension says text; bytes say PNG
p.write_bytes(b"\x89PNG\r\n\x1a\n" + _noise(4096))
result = json.loads(read_file_tool(str(p)))
assert "PNG image data" in result.get("error", "")
assert "4.1 KB" in result["error"] or "4.0 KB" in result["error"]
def test_extensionless_binary_named(self, tmp_path, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
p = tmp_path / "mystery"
p.write_bytes(b"\x7fELF" + _noise(1024))
result = json.loads(read_file_tool(str(p)))
assert "ELF executable" in result.get("error", "")
def test_unknown_binary_still_refused(self, tmp_path, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
p = tmp_path / "junk.out"
p.write_bytes(b"\x00\x01\x02" * 500)
result = json.loads(read_file_tool(str(p)))
assert "unknown binary" in result.get("error", "")
def test_text_file_unaffected(self, tmp_path, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
p = tmp_path / "ok.txt"
p.write_text("hello world\n")
result = json.loads(read_file_tool(str(p)))
assert "hello world" in result.get("content", "")

View File

@@ -449,6 +449,70 @@ def _parse_search_context_line(line: str) -> tuple[str, int, str] | None:
# Abstract Interface
# =============================================================================
_MAGIC_SIGNATURES: tuple = (
# (prefix bytes, human name) — ordered, first match wins. Longest
# prefixes for a shared first byte come first.
(b"\x89PNG\r\n\x1a\n", "PNG image data"),
(b"\xff\xd8\xff", "JPEG image data"),
(b"GIF87a", "GIF image data"),
(b"GIF89a", "GIF image data"),
(b"RIFF", "RIFF container (WAV/AVI/WebP family)"),
(b"%PDF-", "PDF document"),
(b"PK\x03\x04", "ZIP archive (also docx/xlsx/jar/apk)"),
(b"PK\x05\x06", "ZIP archive (empty)"),
(b"\x1f\x8b", "gzip compressed data"),
(b"BZh", "bzip2 compressed data"),
(b"\xfd7zXZ\x00", "xz compressed data"),
(b"7z\xbc\xaf\x27\x1c", "7-Zip archive"),
(b"\x7fELF", "ELF executable"),
(b"MZ", "Windows PE executable"),
(b"\xcf\xfa\xed\xfe", "Mach-O executable (64-bit)"),
(b"\xca\xfe\xba\xbe", "Mach-O universal binary / Java class"),
(b"SQLite format 3\x00", "SQLite database"),
(b"OggS", "Ogg container"),
(b"fLaC", "FLAC audio"),
(b"ID3", "MP3 audio (ID3 tag)"),
(b"\x00\x00\x00", "ISO media container (MP4/MOV family)"), # ftyp at +4
(b"BM", "BMP image data"),
(b"II*\x00", "TIFF image data (little-endian)"),
(b"MM\x00*", "TIFF image data (big-endian)"),
)
def identify_binary_bytes(sample: bytes) -> str:
"""Best-effort human name for binary content from its magic bytes.
Returns e.g. ``"PNG image data"`` or ``"unknown binary"``. Never raises.
The ISO-media entry additionally checks for ``ftyp`` at offset 4, since
the leading size field alone (three NULs) is too weak a signature.
"""
if not sample:
return "unknown binary"
for prefix, name in _MAGIC_SIGNATURES:
if sample.startswith(prefix):
if name.startswith("ISO media") and sample[4:8] != b"ftyp":
continue
return name
return "unknown binary"
def describe_binary_file(sample: Optional[bytes], file_size: int) -> str:
"""One-line answer for the binary-file refusal.
Naming the dead end: "Binary file" alone sends the model hunting for
'appropriate tools' that may not exist in its toolset. Naming the TYPE
("PNG image data, 4.1 KB") answers what-is-this in a single read.
"""
kind = identify_binary_bytes(sample or b"")
if file_size >= 1024 * 1024:
size = f"{file_size / (1024 * 1024):.1f} MB"
elif file_size >= 1024:
size = f"{file_size / 1024:.1f} KB"
else:
size = f"{file_size} bytes"
return f"Binary file ({kind}, {size}) — cannot display as text."
class FileOperations(ABC):
"""Abstract interface for file operations across terminal backends."""
@@ -1339,7 +1403,7 @@ class ShellFileOperations(FileOperations):
return ReadResult(
is_binary=True,
file_size=file_size,
error="Binary file - cannot display as text. Use appropriate tools to handle this file type."
error=describe_binary_file(sample_bytes, file_size),
)
# Read with pagination using sed, clamping each line to a byte
@@ -1574,7 +1638,7 @@ class ShellFileOperations(FileOperations):
if is_binary:
return ReadResult(
is_binary=True, file_size=file_size,
error="Binary file — cannot display as text."
error=describe_binary_file(sample_bytes, file_size),
)
cat_result = self._exec(f"cat {self._escape_shell_arg(path)}")
if cat_result.exit_code != 0:

View File

@@ -1681,7 +1681,10 @@ def read_file_tool(path: str, offset: int = 1, limit: int = 2000, task_id: str =
return json.dumps(result_dict, ensure_ascii=False)
# ── Binary file guard ─────────────────────────────────────────
# Block binary files by extension (no I/O).
# Block binary files by extension (no I/O). Name what we know:
# the extension is a claim, so keep this branch's message to the
# extension itself — the content-sniffing path below names the
# actual magic-byte type for extension-less/lying files.
if has_binary_extension(str(_resolved)):
_ext = _resolved.suffix.lower()
return tool_error(