diff --git a/agent/search_policy.py b/agent/search_policy.py new file mode 100644 index 0000000000..2972d5cae8 --- /dev/null +++ b/agent/search_policy.py @@ -0,0 +1,30 @@ +"""Shared directory pruning policy for broad recursive scans. + +These names identify version-control internals, dependency trees, generated +artifacts, caches, and backup copies that are not useful results for broad +agent-facing discovery. Ordinary search callers may still target an explicit +path; broad diagnostic probes should apply this policy to recursive walks. +""" + +from __future__ import annotations + + +# Keep this policy conservative and name-based so it works for local and remote +# shell backends alike. The same set is used by context discovery and search +# probes; adding a directory here protects every broad recursive consumer. +SEARCH_PRUNE_DIR_NAMES = frozenset({ + # Version-control internals. + ".git", ".hg", ".svn", + # Dependency and vendored trees. + "node_modules", "venv", ".venv", "site-packages", "dist-packages", + "vendor", "third_party", + # Generated/build output. + "build", "dist", "target", "out", "coverage", + ".next", ".turbo", ".parcel-cache", ".nuxt", ".svelte-kit", + # Python and package-manager caches. + "__pycache__", ".cache", ".Trash", ".tox", ".nox", ".mypy_cache", + ".pytest_cache", ".ruff_cache", ".npm", ".yarn", ".pnpm-store", + ".gradle", ".m2", ".nuget", + # Backup copies. + "backups", "backup", ".backups", +}) diff --git a/agent/subdirectory_hints.py b/agent/subdirectory_hints.py index 41fb68ba88..fec41517c2 100644 --- a/agent/subdirectory_hints.py +++ b/agent/subdirectory_hints.py @@ -21,6 +21,7 @@ from pathlib import Path from typing import Dict, Any, Optional, Set from agent.prompt_builder import _read_text_with_timeout, _scan_context_content +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES logger = logging.getLogger(__name__) @@ -47,17 +48,9 @@ _COMMAND_TOOLS = {"terminal"} # Prevents scanning all the way to / for deeply nested paths. _MAX_ANCESTOR_WALK = 5 -# Directory names that never contain authoritative project context. -# Backups, vendored deps, VCS internals, and caches routinely hold *copies* of -# AGENTS.md; loading those duplicates real context and inflates the prompt. -_EXCLUDED_DIR_NAMES = frozenset({ - "node_modules", "venv", ".venv", "__pycache__", - ".git", ".hg", ".svn", - ".Trash", ".cache", ".tox", ".mypy_cache", ".pytest_cache", - "site-packages", "dist-packages", - "backups", "backup", ".backups", - "vendor", "third_party", -}) +# Shared with broad recursive search probes so context discovery and search do +# not drift into different dependency/cache/build trees. +_EXCLUDED_DIR_NAMES = SEARCH_PRUNE_DIR_NAMES def _is_ancestor_or_same(a: Path, b: Path) -> bool: diff --git a/tests/agent/test_subdirectory_hints.py b/tests/agent/test_subdirectory_hints.py index 3b1e8b69fb..8972027adc 100644 --- a/tests/agent/test_subdirectory_hints.py +++ b/tests/agent/test_subdirectory_hints.py @@ -6,6 +6,7 @@ import pytest from pathlib import Path from unittest.mock import patch +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES from agent.subdirectory_hints import SubdirectoryHintTracker @@ -281,7 +282,7 @@ class TestExcludedDirectories: @pytest.mark.parametrize( "excluded", - ["backups", "node_modules", ".git", "venv", "site-packages", ".Trash", "vendor"], + sorted(SEARCH_PRUNE_DIR_NAMES), ) def test_excluded_directory_skipped(self, tmp_path, excluded): target = tmp_path / excluded / "snapshot" diff --git a/tests/tools/test_search_zero_match_and_multipath.py b/tests/tools/test_search_zero_match_and_multipath.py index a001447051..cc6733d36c 100644 --- a/tests/tools/test_search_zero_match_and_multipath.py +++ b/tests/tools/test_search_zero_match_and_multipath.py @@ -58,6 +58,86 @@ class TestZeroMatchProbe: # Same class as the casing probe: the path must be in the hint. assert "conf.cfg" in r.get("warning", "") + def test_hidden_probe_prunes_dependency_trees_and_keeps_local_ignored(self, proj, monkeypatch): + d = proj / "proj" + dependency = d / "node_modules" / "package" + dependency.mkdir(parents=True) + dependency_file = dependency / "dependency.js" + dependency_file.write_text("BOUNDED_HIDDEN_TOKEN = true\n") + local = d / ".project-local" + local.mkdir() + local_file = local / "settings.cfg" + local_file.write_text("BOUNDED_HIDDEN_TOKEN = true\n") + (d / ".gitignore").write_text("node_modules/\n.project-local/\n") + + # Drive the public search seam while recording the commands that the + # zero-match probe actually executes. The real rg calls still run. + from tools.file_tools import _get_file_ops + + task_id = "t-zm-pruned-hidden" + ops = _get_file_ops(task_id=task_id) + commands = [] + real_exec = ops._exec + + def recording_exec(command, *args, **kwargs): + commands.append(command) + return real_exec(command, *args, **kwargs) + + monkeypatch.setattr(ops, "_exec", recording_exec) + r = json.loads(search_tool("BOUNDED_HIDDEN_TOKEN", path=str(d), task_id=task_id)) + warning = r.get("warning", "") + + assert r["total_count"] == 0 + assert "hidden or gitignored" in warning + assert local_file.name in warning + assert dependency_file.name not in warning + + hidden_probe_commands = [ + command for command in commands + if "--hidden" in command and "--no-ignore" in command + ] + assert len(hidden_probe_commands) == 1 + hidden_probe = hidden_probe_commands[0] + assert "--glob" in hidden_probe + assert "'!node_modules/**'" in hidden_probe + assert "'!**/node_modules/**'" in hidden_probe + + def test_hidden_probe_prunes_explicit_dependency_root(self, proj, monkeypatch): + d = proj / "proj" + dependency = d / "node_modules" / "package" / ".hidden" + dependency.mkdir(parents=True) + (dependency / "dependency.js").write_text("EXPLICIT_ROOT_TOKEN = true\n") + (d / ".gitignore").write_text("node_modules/\n") + + from tools.file_tools import _get_file_ops + + task_id = "t-zm-explicit-pruned-root" + ops = _get_file_ops(task_id=task_id) + commands = [] + real_exec = ops._exec + + def recording_exec(command, *args, **kwargs): + commands.append(command) + return real_exec(command, *args, **kwargs) + + monkeypatch.setattr(ops, "_exec", recording_exec) + r = json.loads(search_tool( + "EXPLICIT_ROOT_TOKEN", + path=str(d / "node_modules"), + task_id=task_id, + )) + + assert r["total_count"] == 0 + assert "warning" not in r + hidden_probe_commands = [ + command for command in commands + if "--hidden" in command and "--no-ignore" in command + ] + assert len(hidden_probe_commands) == 1 + hidden_probe = hidden_probe_commands[0] + assert "'!node_modules/**'" in hidden_probe + assert "'!**/node_modules/**'" in hidden_probe + def test_probe_path_list_is_capped(self, proj): d = proj / "proj" for i in range(8): diff --git a/tools/file_operations.py b/tools/file_operations.py index b67299f091..a349e9a2c7 100644 --- a/tools/file_operations.py +++ b/tools/file_operations.py @@ -50,6 +50,7 @@ from agent.file_safety import ( get_write_denied_error, is_write_denied as _shared_is_write_denied, ) +from agent.search_policy import SEARCH_PRUNE_DIR_NAMES from tools import interrupt as tool_interrupt logger = logging.getLogger(__name__) @@ -3654,15 +3655,31 @@ class ShellFileOperations(FileOperations): merged.warning = " ".join(warning_parts) return merged + def _search_prune_glob_args(self) -> str: + """Return rg globs that prune known heavyweight recursive subtrees. + + The two forms cover both a root whose basename is a protected name and + protected descendants. Globs are relative to each rg search root, so a + single ``**/name/**`` pattern does not cover an explicitly selected + ``name/`` root. The directory names come from the shared scan policy; + this method deliberately does not maintain a second search-only list. + """ + globs = [] + for dirname in sorted(SEARCH_PRUNE_DIR_NAMES): + for prefix in ("", "**/"): + pattern = f"!{prefix}{dirname}/**" + globs.extend(("--glob", self._escape_shell_arg(pattern))) + return " ".join(globs) + def _zero_match_probe(self, pattern: str, path: str, file_glob: Optional[str]) -> Optional[str]: """Return a hint for a 0-match content search, or None. 13.9% of production content searches return zero matches and give - the model nothing to steer by. Run ONE cheap case-insensitive count - probe; if it hits, say so. If the pattern contains regex - metacharacters, also probe it as a fixed string. Bounded: two rg - invocations max, count-only output. + the model nothing to steer by. Run cheap count-only probes for near + misses (wrong casing, hidden-only matches, unescaped regex + metacharacters). The hidden/ignored probe is bounded with the shared + dependency, cache, VCS, vendor, and build-tree pruning policy. """ rg_executable = self._resolve_command('rg') if not rg_executable: @@ -3702,9 +3719,12 @@ class ShellFileOperations(FileOperations): # Hidden/ignored probe: rg skips dotdirs and .gitignore'd files by # default. When the pattern exists only there, say so instead of # returning a bare zero (bench case: match in .hidden/ silently - # missing from results). + # missing from results). Keep --no-ignore so project-local ignored + # files remain diagnosable, but prune heavyweight trees before rg can + # recurse into them. hidden = self._exec( - f"{rg} --hidden --no-ignore --count-matches{glob_expr} " + f"{rg} --hidden --no-ignore --count-matches{glob_expr}" + f" {self._search_prune_glob_args()} " f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} " f"2>/dev/null | head -50", timeout=30,