"""Tests for tools/file_operations.py — deny list, result dataclasses, helpers.""" import base64 import os import pytest import shutil import subprocess from pathlib import Path from unittest.mock import MagicMock from tests.tools.file_ops_fakes import READ_SENTINEL_RE, compound_read_output from tools.environments.local import _find_bash, LocalEnvironment from agent.file_safety import is_write_denied as _is_write_denied from tools.file_operations_common import SearchMatch from tools.file_operations import ( ReadResult, SearchResult, ShellFileOperations, normalize_read_pagination, ) # ========================================================================= # Write deny list # ========================================================================= class TestIsWriteDenied: def test_ssh_authorized_keys_denied(self): path = os.path.join(str(Path.home()), ".ssh", "authorized_keys") assert _is_write_denied(path) is True def test_netrc_denied(self): path = os.path.join(str(Path.home()), ".netrc") assert _is_write_denied(path) is True @pytest.mark.parametrize("name", [".pgpass", ".npmrc", ".pypirc"]) def test_credential_config_files_denied(self, name): path = os.path.join(str(Path.home()), name) assert _is_write_denied(path) is True def test_aws_prefix_denied(self): path = os.path.join(str(Path.home()), ".aws", "credentials") assert _is_write_denied(path) is True @pytest.mark.parametrize( "path", [ "./.anthropic_oauth.json", ], ) def test_oauth_traversal_denied(self, path): """Path traversal attempts to protected OAuth files must be blocked.""" from hermes_constants import get_hermes_home hermes_home = get_hermes_home() full_path = str(hermes_home / path) assert _is_write_denied(full_path) is True def test_mcp_tokens_dir_protected_in_profile_mode(self, tmp_path, monkeypatch): """mcp-tokens/ under profile AND under root must both be denied.""" root = tmp_path / "hermes" profile = root / "profiles" / "coder" profile.mkdir(parents=True) monkeypatch.setenv("HERMES_HOME", str(profile)) assert _is_write_denied(str(profile / "mcp-tokens" / "tok.json")) is True assert _is_write_denied(str(root / "mcp-tokens" / "tok.json")) is True # The directory itself must also be denied (not just files inside) assert _is_write_denied(str(root / "mcp-tokens")) is True def test_pairing_dir_denied(self, tmp_path, monkeypatch): """Regression: pairing/ must be write-denied under both profile and root. PR #30383 introduced ~/.hermes/pairing/{platform}-approved.json as the gateway access-control list. Without this block, a prompt-injected agent can write arbitrary user IDs into an approved file, granting persistent gateway access without going through the pairing code flow — the same threat class that motivated protecting webhook_subscriptions.json. """ root = tmp_path / "hermes" profile = root / "profiles" / "coder" profile.mkdir(parents=True) monkeypatch.setenv("HERMES_HOME", str(profile)) # Active profile pairing entries assert _is_write_denied(str(profile / "pairing" / "telegram-approved.json")) is True assert _is_write_denied(str(profile / "pairing" / "discord-pending.json")) is True # The directory itself assert _is_write_denied(str(profile / "pairing")) is True # Root pairing entries (profile mode — same shape as mcp-tokens gap) assert _is_write_denied(str(root / "pairing" / "telegram-approved.json")) is True assert _is_write_denied(str(root / "pairing")) is True # ========================================================================= # Result dataclasses # ========================================================================= class TestReadResult: def test_to_dict_omits_defaults(self): r = ReadResult() d = r.to_dict() assert "error" not in d # None omitted assert "similar_files" not in d # empty list omitted class TestSearchResult: def test_truncated_flag_marks_total_as_lower_bound(self): r = SearchResult(total_count=100, truncated=True) d = r.to_dict() assert d["truncated"] is True assert d["total_count_is_lower_bound"] is True def test_untruncated_total_omits_lower_bound_flag(self): r = SearchResult(total_count=100) d = r.to_dict() assert "total_count_is_lower_bound" not in d class TestSearchResultDensify: """Path-grouped densification of content-mode matches (lossless).""" def _matches(self, n, paths=None): # Real ripgrep output is path-ordered: all matches in a file are # consecutive (verified against live search_files corpus). The fixture # mirrors that — group by path, then enumerate lines within each. paths = paths or ["a.py"] out = [] per = max(1, n // len(paths)) ln = 0 for p in paths: for _ in range(per): ln += 1 out.append(SearchMatch(path=p, line_number=ln, content=f"line content {ln}")) # pad remainder onto the last path while len(out) < n: ln += 1 out.append(SearchMatch(path=paths[-1], line_number=ln, content=f"line content {ln}")) return out def test_densify_off_by_default(self): # The model-facing default must be unchanged for callers that don't # opt in: verbose array, no matches_text key. r = SearchResult(matches=self._matches(10), total_count=10) d = r.to_dict() assert "matches" in d assert "matches_text" not in d def test_densify_below_threshold_keeps_verbose(self): # Too few matches: the grouping header would cost more than it saves, # so we fall back to the verbose array even with densify=True. r = SearchResult(matches=self._matches(4), total_count=4) d = r.to_dict(densify=True) assert "matches" in d assert "matches_text" not in d def test_densify_paths_with_spaces(self): matches = [SearchMatch(path="my dir/a b.py", line_number=i + 1, content=f"x{i}") for i in range(6)] text = SearchResult(matches=matches, total_count=6).to_dict(densify=True)["matches_text"] # path with spaces survives as a header line verbatim assert "my dir/a b.py" in text.split("\n")[0] # ========================================================================= # ShellFileOperations helpers # ========================================================================= @pytest.fixture() def mock_env(): """Create a mock terminal environment.""" env = MagicMock() env.cwd = "/tmp/test" env.execute.return_value = {"output": "", "returncode": 0} return env @pytest.fixture() def file_ops(mock_env): return ShellFileOperations(mock_env) def make_real_subprocess_env(cwd: str, include_stderr: bool = False) -> MagicMock: """Mock env whose execute() runs the command in a real subprocess. For tests that need the generated shell scripts to actually run (search fallback, atomic-write permissions) instead of being intercepted by a bare MagicMock. ``include_stderr`` folds stderr into ``output`` for tests that surface shell error text; leave it off for tests that parse structured stdout (e.g. find results). """ env = MagicMock() env.cwd = cwd def execute(command, **kwargs): stdin_data = kwargs.get("stdin_data") is_windows = os.name == "nt" if is_windows: # Match LocalEnvironment: commands are POSIX scripts executed by # Git Bash, and stdin bytes must bypass Windows newline rewriting. command = [_find_bash(), "-c", command] completed = subprocess.run( command, shell=not is_windows, text=not is_windows, capture_output=True, input=(stdin_data.encode("utf-8", "surrogateescape") if is_windows and stdin_data is not None else stdin_data), ) output = ( completed.stdout.decode("utf-8", "replace") if is_windows else completed.stdout ) if include_stderr: output += ( completed.stderr.decode("utf-8", "replace") if is_windows else completed.stderr ) return { "output": output, "returncode": completed.returncode, } env.execute = execute return env class TestShellFileOpsHelpers: def test_normalize_read_pagination_clamps_invalid_values(self): assert normalize_read_pagination(offset=0, limit=0) == (1, 1) assert normalize_read_pagination(offset=-10, limit=-5) == (1, 1) assert normalize_read_pagination(offset="bad", limit="bad") == (1, 2000) assert normalize_read_pagination(offset=2, limit=999999) == (2, 2000) @pytest.mark.platforms("windows") def test_escape_shell_arg_rewrites_forward_slash_native_paths(self, file_ops): """Windows-only: ``_bash_safe_path`` only rewrites drive paths to the Git Bash form on Windows, where the MSYS path mangling it works around actually happens.""" assert file_ops._escape_shell_arg( "C:/Users/alice/notes.txt" ) == "'/c/Users/alice/notes.txt'" def test_is_likely_binary_by_extension(self, file_ops): assert file_ops._is_likely_binary("photo.png") is True assert file_ops._is_likely_binary("data.db") is True assert file_ops._is_likely_binary("code.py") is False assert file_ops._is_likely_binary("readme.md") is False def test_read_file_strips_leaked_terminal_fence_markers(self, mock_env): leaked = ( "'\x07__HERMES_FENCE_a9f7b3__\x1b]0;cat " "'/tmp/test/a.py' 2> /dev/null\x07\n" "print('ok')\n" "__HERMES_FENCE_a9f7b3__\x07'\n" ) def side_effect(command, **kwargs): m = READ_SENTINEL_RE.search(command) if m: return { "output": compound_read_output( m.group(0), size=12, sample=b"print('ok')\n", content=leaked, total_lines=1, ), "returncode": 0, } return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = ShellFileOperations(mock_env) result = ops.read_file("/tmp/test/a.py") assert result.error is None assert "HERMES_FENCE" not in result.content assert "\x1b]" not in result.content assert "\x07" not in result.content assert "1|print('ok')" in result.content def test_newline_terminated_content_has_no_phantom_line(self, file_ops): # A file ending in a newline (the normal, well-formed case) has its # last line terminated, NOT followed by an empty line. The gutter must # match `cat -n`: three lines in, three numbered lines out. result = file_ops._add_line_numbers("line1\nline2\nline3\n") assert result == "1|line1\n2|line2\n3|line3" assert "4|" not in result assert len(result.split("\n")) == 3 def test_non_terminated_content_still_numbered_correctly(self, file_ops): # Content with no trailing newline was already correct; guard it. result = file_ops._add_line_numbers("line1\nline2\nline3") assert result == "1|line1\n2|line2\n3|line3" def test_trailing_blank_line_is_kept(self, file_ops): # "a" then a genuine blank line, then the terminating newline: that is # two lines (a, blank), so only the single terminator is dropped. result = file_ops._add_line_numbers("a\n\n") assert result == "1|a\n2|" assert "3|" not in result def test_newline_terminated_with_offset_has_no_phantom_line(self, file_ops): # A truncated page (offset>1) that ends on a newline must not append a # phantom numbered line at the page boundary. result = file_ops._add_line_numbers("def f():\n return 1\n", start_line=10) assert result == "10|def f():\n11| return 1" assert "12|" not in result class TestSearchPathValidation: """Test that search() returns an error for non-existent paths.""" def test_search_nonexistent_path_returns_error(self, mock_env): """search() should return an error when the path doesn't exist.""" def side_effect(command, **kwargs): if "test -e" in command: return {"output": "not_found", "returncode": 1} if "command -v" in command: return {"output": "yes", "returncode": 0} return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = ShellFileOperations(mock_env) result = ops.search("pattern", path="/nonexistent/path") assert result.error is not None assert "not found" in result.error.lower() or "Path not found" in result.error def test_search_rg_error_exit_code(self, mock_env): """search() should report error when rg returns exit code 2.""" call_count = {"n": 0} def side_effect(command, **kwargs): call_count["n"] += 1 if "test -e" in command: return {"output": "exists", "returncode": 0} if "command -v" in command: return {"output": "yes", "returncode": 0} # rg returns exit 2 (error) with empty output return {"output": "", "returncode": 2} mock_env.execute.side_effect = side_effect ops = ShellFileOperations(mock_env) result = ops.search("pattern", path="/some/path") assert result.error is not None assert "search failed" in result.error.lower() or "Search error" in result.error class TestSearchFilesIncludesDirectories: """``search_files(target='files')`` is documented as an ls replacement and must list matching directories, not just files — empty directories included (#54347).""" @pytest.mark.skipif(shutil.which("rg") is None, reason="ripgrep not installed") def test_rg_target_files_includes_matching_empty_directory(self, tmp_path): root = tmp_path / "repo" empty_dir = root / "vault" empty_dir.mkdir(parents=True) ops = ShellFileOperations(LocalEnvironment("/")) result = ops.search("vault", path=str(root), target="files") assert result.error is None assert str(empty_dir) in result.files @pytest.mark.skipif(shutil.which("rg") is None, reason="ripgrep not installed") @pytest.mark.skipif(shutil.which("git") is None, reason="git not installed") def test_rg_target_files_excludes_gitignored_empty_directory(self, tmp_path): root = tmp_path / "repo" ignored_dir = root / "ignored_dir" visible_dir = root / "visible_dir" ignored_dir.mkdir(parents=True) visible_dir.mkdir() (root / ".gitignore").write_text("ignored_dir/\n") subprocess.run(["git", "init", "-q"], cwd=root, check=True) ops = ShellFileOperations(LocalEnvironment("/")) result = ops.search("*dir", path=str(root), target="files") assert result.error is None assert str(visible_dir) in result.files assert str(ignored_dir) not in result.files @pytest.mark.skipif(shutil.which("rg") is None, reason="ripgrep not installed") def test_rg_sorts_files_and_directories_before_pagination(self, tmp_path): """A newer directory must page ahead of older files and dirs (globally merged mtime order before offset/limit, not dirs appended after).""" root = tmp_path / "repo" older_file = root / "older.log" newer_dir = root / "alpha-newer" older_dirs = [root / "zulu-older", root / "yankee-older"] root.mkdir() older_file.write_text("x") newer_dir.mkdir() for older_dir in older_dirs: older_dir.mkdir() os.utime(older_file, (1_000, 1_000)) for older_dir in older_dirs: os.utime(older_dir, (1_000, 1_000)) os.utime(newer_dir, (2_000, 2_000)) ops = ShellFileOperations(LocalEnvironment("/")) result = ops.search("*", path=str(root), target="files", limit=1) assert result.error is None assert result.files == [str(newer_dir)] assert result.truncated is True def test_find_target_files_includes_matching_empty_directory(self, tmp_path, monkeypatch): root = tmp_path / "repo" empty_dir = root / "vault" regular_file = root / "vault.txt" empty_dir.mkdir(parents=True) regular_file.write_text("x") ops = ShellFileOperations(LocalEnvironment("/")) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") result = ops.search("vault*", path=str(root), target="files") assert result.error is None assert {str(empty_dir), str(regular_file)}.issubset(set(result.files)) def test_target_files_still_excludes_hidden_matching_directories(self, tmp_path, monkeypatch): root = tmp_path / "repo" visible_dir = root / "visible" / "cache" hidden_dir = root / ".hidden" / "cache" visible_dir.mkdir(parents=True) hidden_dir.mkdir(parents=True) ops = ShellFileOperations(LocalEnvironment("/")) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") result = ops.search("cache", path=str(root), target="files") assert result.error is None assert str(visible_dir) in result.files assert str(hidden_dir) not in result.files def test_find_lane_never_returns_the_search_root_itself(self, tmp_path, monkeypatch): """The root is excluded by path identity — not by ``-mindepth 1``, which would also exclude the depth-0 operand match a symlinked root depends on (#116270) — and a same-named descendant still matches (#54347).""" root = tmp_path / "vault" child = root / "vault" child.mkdir(parents=True) ops = ShellFileOperations(LocalEnvironment("/")) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") result = ops.search("vault", path=str(root), target="files") assert result.error is None assert str(child) in result.files assert str(root) not in result.files def test_target_files_paginates_combined_files_and_directories(self, tmp_path, monkeypatch): """Files and directories share one ordering: page slices must compose.""" root = tmp_path / "repo" empty_dir = root / "alpha" regular_file = root / "bravo" empty_dir.mkdir(parents=True) regular_file.write_text("x") ops = ShellFileOperations(LocalEnvironment("/")) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") first = ops.search("*", path=str(root), target="files", limit=1, offset=0) second = ops.search("*", path=str(root), target="files", limit=1, offset=1) combined = ops.search("*", path=str(root), target="files", limit=2, offset=0) assert first.error is None assert second.error is None assert combined.error is None assert first.files + second.files == combined.files assert set(combined.files) == {str(empty_dir), str(regular_file)} class TestSearchFilesFallbackHiddenPaths: def _make_env(self): return LocalEnvironment("/") def test_hidden_root_with_hidden_ancestor_includes_files(self, tmp_path, monkeypatch): """Fallback find should include visible files when path is inside hidden root.""" root = tmp_path / ".hermes" / "logs" root.mkdir(parents=True) visible_file = root / "agent.log" hidden_dir_file = root / ".hidden" / "secret.log" nested_hidden_file = root / "nested" / ".secret.log" visible_nested_file = root / "nested" / "visible.log" for p in [visible_file, nested_hidden_file, visible_nested_file, hidden_dir_file]: p.parent.mkdir(parents=True, exist_ok=True) p.write_text("x") ops = ShellFileOperations(self._make_env()) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") result = ops._search_files("*.log", str(root), limit=50, offset=0) assert result.error is None assert set(result.files) == {str(visible_file), str(visible_nested_file)} def test_normal_root_still_excludes_hidden_descendants(self, tmp_path, monkeypatch): """Fallback find should still exclude hidden descendant paths for normal roots.""" root = tmp_path / "repo" root.mkdir() visible_file = root / "agent.log" visible_nested_file = root / "nested" / "visible.log" hidden_dir_file = root / ".hidden" / "secret.log" for p in [visible_file, visible_nested_file, hidden_dir_file]: p.parent.mkdir(parents=True, exist_ok=True) p.write_text("x") ops = ShellFileOperations(self._make_env()) monkeypatch.setattr(ops, "_has_command", lambda command: command == "find") result = ops._search_files("*.log", str(root), limit=50, offset=0) assert result.error is None assert set(result.files) == {str(visible_file), str(visible_nested_file)} class TestShellFileOpsWriteDenied: def test_write_file_denied_path(self, file_ops): result = file_ops.write_file("~/.ssh/authorized_keys", "evil key") assert result.error is not None assert "denied" in result.error.lower() def test_move_file_failure_path(self, mock_env): mock_env.execute.return_value = {"output": "No such file or directory", "returncode": 1} ops = ShellFileOperations(mock_env) result = ops.move_file("/tmp/nonexistent.txt", "/tmp/dest.txt") assert result.error is not None assert "Failed to move" in result.error def _fenced_base64_reply(command: str, payload: bytes, rc: int = 0) -> str: """The reply shape ``_read_exact_bytes`` asks for: its per-call sentinel around the base64 payload (and the file's ``wc -c`` when the command asks for it), then the read's exit status. Mirrors what the real shell emits, so a double stays honest about the fence the transport relies on.""" import base64 as _b64 import re as _re sentinel = _re.search(r"__HERMES_RB_[0-9a-f]+__", command).group(0) body = _b64.b64encode(payload).decode() if rc == 0 else "" size = f"{len(payload)}\n{sentinel}\n" if "wc -c <" in command else "" return f"{sentinel}\n{body}\n{sentinel}\n{size}{rc}\n" class TestPatchReplacePostWriteVerification: """Tests for the post-write verification added in patch_replace. Confirms that a silent persistence failure (where write_file's command appears to succeed but the bytes on disk don't match new_content) is surfaced as an error instead of being reported as a successful patch. """ def test_patch_replace_fails_when_file_not_persisted(self, mock_env): """write_file reports success but the re-read returns old content: patch_replace must return an error, not success-with-diff.""" file_contents = {"/tmp/test/a.py": "hello world\n"} def side_effect(command, **kwargs): # the byte-exact read (base64 over the transport) — both the initial read and the verify read if "base64 < " in command: for path in file_contents: if path in command: return {"output": _fenced_base64_reply(command, file_contents[path].encode()), "returncode": 0} return {"output": "", "returncode": 1} # mkdir for parent dir if command.startswith("mkdir "): return {"output": "", "returncode": 0} # wc -c for byte count after write if command.startswith("if [ -f ") or command.startswith("wc -c"): for path in file_contents: if path in command: return {"output": str(len(file_contents[path].encode())), "returncode": 0} return {"output": "0", "returncode": 0} # Everything else (including the write itself) pretends to succeed # but DOESN'T update file_contents — simulates silent failure return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = ShellFileOperations(mock_env) result = ops.patch_replace("/tmp/test/a.py", "hello", "hi") assert result.error is not None, ( "Silent persistence failure must surface as error, got: " f"success={result.success}, diff={result.diff}" ) assert "verification failed" in result.error.lower() assert "did not persist" in result.error.lower() def test_patch_replace_fails_when_verify_read_errors(self, mock_env): """If the verify-read step itself fails (exit code != 0), return an error.""" call_count = {"read": 0} state = {"content": "hello world\n"} def side_effect(command, stdin_data=None, **kwargs): if stdin_data is not None: # write (atomic temp-file + mv script) state["content"] = stdin_data return {"output": "", "returncode": 0} if "base64 < " in command: # byte-exact read call_count["read"] += 1 # First read (initial fetch) succeeds; second read (verify) fails if call_count["read"] == 1: return {"output": _fenced_base64_reply(command, state["content"].encode()), "returncode": 0} return {"output": "", "returncode": 1} if command.startswith("mkdir "): return {"output": "", "returncode": 0} if command.startswith("if [ -f ") or command.startswith("wc -c"): return {"output": str(len(state["content"].encode())), "returncode": 0} return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = ShellFileOperations(mock_env) result = ops.patch_replace("/tmp/test/a.py", "hello", "hi") assert result.error is not None assert "could not re-read" in result.error.lower() # ========================================================================= # Atomic write: umask-default permissions for new files # ========================================================================= class TestAtomicWriteNewFilePermissions: """_atomic_write should apply umask-default perms to new files (not 0600).""" @pytest.mark.parametrize("test_umask", [0o022, 0o002, 0o077]) def test_new_file_gets_umask_default_permissions(self, tmp_path, test_umask): """Newly created file should get umask-computed perms, not mktemp's 0600. Uses a real subprocess so the shell script actually runs. """ ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) dest = tmp_path / "new_file.txt" assert not dest.exists() old_umask = os.umask(test_umask) try: result = ops.write_file(str(dest), "test content\n") finally: os.umask(old_umask) assert result.error is None, f"write failed: {result.error}" assert dest.read_text() == "test content\n" expected_mode = 0o666 & ~test_umask actual_mode = dest.stat().st_mode & 0o777 assert actual_mode == expected_mode, ( f"Expected mode {expected_mode:04o} (umask {test_umask:04o}), " f"got {actual_mode:04o}" ) def test_overwrite_still_preserves_existing_mode(self, tmp_path): """The new-file branch must not disturb the overwrite path's mode preservation (e.g. an executable script stays 0755).""" ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) dest = tmp_path / "existing.sh" dest.write_text("#!/bin/sh\n") dest.chmod(0o755) result = ops.write_file(str(dest), "#!/bin/sh\necho updated\n") assert result.error is None, f"write failed: {result.error}" assert dest.read_text() == "#!/bin/sh\necho updated\n" assert dest.stat().st_mode & 0o777 == 0o755 class TestAtomicWriteThroughSymlink: """_atomic_write must edit a symlink's target, not replace the link. Regression: the temp-file + ``mv`` swap replaced the symlink itself with a plain file, orphaning the real target and destroying the link (data-loss). """ def test_write_follows_symlink_and_preserves_link(self, tmp_path): ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) real = tmp_path / "real.txt" link = tmp_path / "link.txt" real.write_text("original\n") link.symlink_to(real) result = ops.write_file(str(link), "newcontent\n") assert result.error is None, f"write failed: {result.error}" # The link must survive as a symlink... assert link.is_symlink(), "symlink was replaced by a plain file" # ...and the real target must carry the new content. assert real.read_text() == "newcontent\n" assert os.path.realpath(link) == str(real) def test_write_through_broken_symlink_falls_back(self, tmp_path): """A broken link resolves through readlink -f and creates the target.""" ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) target = tmp_path / "target.txt" link = tmp_path / "broken.lnk" link.symlink_to(target) # target does not exist yet result = ops.write_file(str(link), "data\n") assert result.error is None, f"write failed: {result.error}" assert target.exists() assert target.read_text() == "data\n" class TestReadNonUtf8IsBinary: """Non-UTF-8 content must be flagged binary, not returned as lossy text. Regression: the terminal env decodes stdout with errors="replace", turning every non-UTF-8 byte into U+FFFD before _is_likely_binary sees it. U+FFFD is "printable", so the non-printable ratio never caught it, and a read→edit→write round-trip would overwrite the original bytes with mojibake. """ def test_replacement_char_sample_flagged_binary(self, tmp_path): ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) # A latin-1 file decoded with errors="replace" yields U+FFFD chars. lossy_sample = "caf\ufffd r\ufffdsum\ufffd\n" assert ops._is_likely_binary("notes.txt", lossy_sample) is True def test_plain_utf8_text_not_flagged(self, tmp_path): ops = ShellFileOperations(make_real_subprocess_env(str(tmp_path))) # Proper UTF-8 (including non-ASCII) must still read as text. assert ops._is_likely_binary("notes.txt", "café résumé\nsecond\n") is False # ========================================================================= # Byte-layer binary detection (#80308 class: CJK/multibyte text flagged # binary because the byte-boundary sample manufactured U+FFFD in transit) # ========================================================================= class TestByteLayerBinaryDetection: """Regression suite for the misclassification class behind #80308. Fragment reports/fixes each caught one member: #80261, #80250, #80188, #80349, #79834, #79534, #79408. The boundary contract: text = valid UTF-8 allowing one incomplete multibyte sequence at the sample's end; NUL or mid-stream invalid UTF-8 = read-only. """ # --- unit: _is_likely_binary_bytes ----------------------------------- def test_cjk_text_cut_mid_character_is_text(self, file_ops): # 999 ASCII bytes + a 3-byte CJK char cut after its first byte — # exactly what `head -c 1000` does to a CJK file. sample = (b"a" * 999 + "中".encode("utf-8"))[:1000] assert sample[-1:] != b"a" # the cut really is mid-character assert file_ops._is_likely_binary_bytes(sample) is False def test_emoji_cut_at_boundary_is_text(self, file_ops): # 4-byte sequence cut after 2 bytes. sample = (b"x" * 998 + "🎉".encode("utf-8"))[:1000] assert file_ops._is_likely_binary_bytes(sample) is False def test_utf8_bom_is_text(self, file_ops): assert file_ops._is_likely_binary_bytes(b"\xef\xbb\xbfhello") is False def test_file_containing_real_replacement_char_is_text(self, file_ops): # A log file that legitimately stores U+FFFD is valid UTF-8. The old # text-layer check could not tell it from transport damage. assert file_ops._is_likely_binary_bytes("log: \ufffd bad byte\n".encode("utf-8")) is False def test_nul_byte_is_binary(self, file_ops): assert file_ops._is_likely_binary_bytes(b"MZ\x00\x01text") is True def test_latin1_text_stays_read_only(self, file_ops): # Mid-stream invalid UTF-8 (0xE9 = latin-1 é). Reading it through the # replace-decoding transport would mojibake a read→edit→write # round-trip, so it must stay flagged (the old check's guarantee). assert file_ops._is_likely_binary_bytes(b"caf\xe9 au lait, plus padding") is True def test_empty_sample_is_text(self, file_ops): assert file_ops._is_likely_binary_bytes(b"") is False def test_truncated_garbage_tail_after_invalid_prefix_is_binary(self, file_ops): # Error near the end but the prefix itself is not clean UTF-8. assert file_ops._is_likely_binary_bytes(b"\xff\xfe" + b"a" * 10 + b"\xe4") is True # --- transport: _sample_file_bytes ------------------------------------ def test_sample_decodes_base64_transport(self, mock_env): payload = ("汉字" * 400).encode("utf-8")[:1000] mock_env.execute.side_effect = lambda command, **kwargs: { "output": _fenced_base64_reply(command, payload), "returncode": 0} ops = ShellFileOperations(mock_env) assert ops._sample_file_bytes("/tmp/x.txt") == payload def test_sample_falls_back_on_non_base64_output(self, mock_env): mock_env.execute.return_value = {"output": "not base64 at all!!", "returncode": 0} ops = ShellFileOperations(mock_env) assert ops._sample_file_bytes("/tmp/x.txt") is None def test_sample_falls_back_on_nonzero_exit(self, mock_env): mock_env.execute.return_value = {"output": "", "returncode": 127} ops = ShellFileOperations(mock_env) assert ops._sample_file_bytes("/tmp/x.txt") is None # --- integration: read_file over the mocked terminal ------------------ def _dispatch(self, cjk_bytes): def side_effect(command, **kwargs): m = READ_SENTINEL_RE.search(command) if m: return { "output": compound_read_output( m.group(0), size=len(cjk_bytes), sample=cjk_bytes[:1000], content=cjk_bytes.decode("utf-8", errors="replace"), total_lines=1, ), "returncode": 0, } return {"output": "", "returncode": 0} return side_effect def test_read_file_returns_cjk_content_instead_of_binary_error(self, mock_env): content = ("汉字测试" * 300).encode("utf-8") # > 1000 bytes, cut mid-char mock_env.execute.side_effect = self._dispatch(content) ops = ShellFileOperations(mock_env) result = ops.read_file("/tmp/notes-中文.txt") assert result.is_binary is False assert result.error is None assert "汉字测试" in (result.content or "") def test_read_file_still_blocks_nul_binaries(self, mock_env): content = b"\x7fELF\x00\x00binarybinary" + b"\x00" * 100 mock_env.execute.side_effect = self._dispatch(content) ops = ShellFileOperations(mock_env) result = ops.read_file("/tmp/a.out") assert result.is_binary is True class TestEscapeNativeToolArg: """Regression tests for _escape_native_tool_arg (Windows native-binary paths). Live failure (Windows, Aug 2026): search_files passed rg the MSYS form (/c/Users/...) that _escape_shell_arg produces, but Hermes sets MSYS_NO_PATHCONV=1 / MSYS2_ARG_CONV_EXCL=* for its bash subprocesses, so nothing converted the path back for the native (winget) ripgrep binary — every search on a drive-letter path failed with "The system cannot find the path specified. (os error 3)". Native Windows binaries need C:/... (forward-slash native), which bash also passes through untouched. """ def _ops(self, mock_env): return ShellFileOperations(mock_env) @pytest.mark.platforms("windows") def test_windows_native_path_kept_native(self, mock_env): ops = self._ops(mock_env) out = ops._escape_native_tool_arg(r"C:\Users\alice\project") assert out == "'C:/Users/alice/project'" @pytest.mark.platforms("windows") def test_msys_path_translated_back_to_native(self, mock_env): ops = self._ops(mock_env) out = ops._escape_native_tool_arg("/c/Users/alice/project") assert out == "'C:/Users/alice/project'" @pytest.mark.platforms("windows") def test_posix_path_untouched_on_windows(self, mock_env): """Multi-segment POSIX paths (/home/x, /tmp/y) are not drive paths.""" ops = self._ops(mock_env) assert ops._escape_native_tool_arg("/tmp/workdir") == "'/tmp/workdir'" @pytest.mark.platforms("windows") def test_rg_content_search_uses_native_form(self, mock_env): """The call site, not just the helper: search must hand the native rg binary C:/..., never the MSYS /c/... form (the live os-error-3 failure).""" commands = [] def side_effect(command, **kwargs): commands.append(command) if "test -e" in command: return {"output": "exists", "returncode": 0} if "command -v" in command: return {"output": "yes", "returncode": 0} return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = self._ops(mock_env) ops.search("needle", path=r"C:\Users\alice\project") rg_cmds = [c for c in commands if "rg " in c or c.startswith("rg")] assert rg_cmds, f"no rg command captured in: {commands}" assert any("'C:/Users/alice/project'" in c for c in rg_cmds), rg_cmds assert all("/c/Users" not in c for c in rg_cmds), rg_cmds @pytest.mark.platforms("windows") def test_shell_linter_uses_native_form(self, mock_env): """_check_lint must hand node/python/etc. the native C:/ path. Regression for the double-prefix failure (#84303): node given the MSYS /c/Users/... form resolves it as C:\\c\\Users\\... and every .js write reports a phantom ENOENT lint error. """ commands = [] def side_effect(command, **kwargs): commands.append(command) if "command -v" in command: return {"output": "yes", "returncode": 0} return {"output": "", "returncode": 0} mock_env.execute.side_effect = side_effect ops = self._ops(mock_env) result = ops._check_lint(r"C:\Users\alice\app\main.js") assert result.skipped is False node_cmds = [c for c in commands if "node --check" in c] assert node_cmds, f"no node command captured in: {commands}" assert "'C:/Users/alice/app/main.js'" in node_cmds[0] assert "/c/Users" not in node_cmds[0]