test: exempt evals/ from the managed-runtime which() guard; make the OS-marker lister tolerate vanishing dirs

Two CI reds from the previous commits.

benchmark_browser_eval.py moved from scripts/ (exempt) to evals/ and its
bare shutil.which("npx") tripped test_no_unreviewed_bare_managed_runtime_
lookups. evals/ are standalone user-invoked benchmark programs, the same
class as scripts/ and skills/, and three other eval files already call
which() the same way; the guard just never saw them because the harness
that carried them lived under scripts/. Exempt the directory.

list_os_marked_tests.py used Path.rglob, which raises FileNotFoundError
when a __pycache__ directory disappears mid-scan (a sibling job in the
same workspace). The managed-runtime guard already switched to os.walk
for the identical TOCTOU; do the same here. Same output, sorted.
This commit is contained in:
teknium1
2026-09-13 05:59:30 -07:00
committed by Teknium
parent 0b40f5a790
commit 7fbe16aed5
2 changed files with 18 additions and 10 deletions

View File

@@ -26,6 +26,7 @@ Prints one path per line (POSIX separators, repo-relative), sorted.
from __future__ import annotations
import os
import re
import sys
from pathlib import Path
@@ -43,14 +44,20 @@ def find_marked_files(marker: str, root: Path) -> list[Path]:
"""
pattern = re.compile(rf"\b{re.escape(marker)}\b")
hits: list[Path] = []
for path in sorted(root.rglob("test_*.py")):
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
if pattern.search(text):
hits.append(path)
return hits
# os.walk, not Path.rglob: rglob raises FileNotFoundError when a directory
# (a sibling job's __pycache__) vanishes mid-scan; os.walk skips it.
for dirpath, _dirnames, filenames in os.walk(root):
for fname in filenames:
if not (fname.startswith("test_") and fname.endswith(".py")):
continue
path = Path(dirpath) / fname
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
if pattern.search(text):
hits.append(path)
return sorted(hits)
def main(argv: list[str]) -> int:

View File

@@ -39,14 +39,15 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
_MANAGED_COMMANDS = frozenset({"uv", "node", "npm", "npx"})
# Directories that are not Hermes-owned subprocess code: plugins ship their own
# resolution policy, tests assert against PATH deliberately, and skills/scripts
# run as standalone user-invoked programs.
# resolution policy, tests assert against PATH deliberately, and skills/scripts/
# evals run as standalone user-invoked programs.
_EXEMPT_DIRS = (
"tests",
"plugins",
"skills",
"optional-skills",
"scripts",
"evals",
"website",
"node_modules",
".git",