From 7fbe16aed5cdb59b59eea31aca076c1e5213beb2 Mon Sep 17 00:00:00 2001
From: teknium1 <127238744+teknium1@users.noreply.github.com>
Date: Sun, 13 Sep 2026 05:59:30 -0700
Subject: [PATCH] test: exempt evals/ from the managed-runtime which() guard;
 make the OS-marker lister tolerate vanishing dirs

Two CI reds from the previous commits.

benchmark_browser_eval.py moved from scripts/ (exempt) to evals/ and its
bare shutil.which("npx") tripped test_no_unreviewed_bare_managed_runtime_
lookups. evals/ are standalone user-invoked benchmark programs, the same
class as scripts/ and skills/, and three other eval files already call
which() the same way; the guard just never saw them because the harness
that carried them lived under scripts/. Exempt the directory.

list_os_marked_tests.py used Path.rglob, which raises FileNotFoundError
when a __pycache__ directory disappears mid-scan (a sibling job in the
same workspace). The managed-runtime guard already switched to os.walk
for the identical TOCTOU; do the same here. Same output, sorted.
---
 scripts/ci/list_os_marked_tests.py       | 23 +++++++++++++++--------
 tests/test_managed_runtime_resolution.py |  5 +++--
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/scripts/ci/list_os_marked_tests.py b/scripts/ci/list_os_marked_tests.py
index bfbc0a60c1..f3eedf87fe 100644
--- a/scripts/ci/list_os_marked_tests.py
+++ b/scripts/ci/list_os_marked_tests.py
@@ -26,6 +26,7 @@ Prints one path per line (POSIX separators, repo-relative), sorted.
 
 from __future__ import annotations
 
+import os
 import re
 import sys
 from pathlib import Path
@@ -43,14 +44,20 @@ def find_marked_files(marker: str, root: Path) -> list[Path]:
     """
     pattern = re.compile(rf"\b{re.escape(marker)}\b")
     hits: list[Path] = []
-    for path in sorted(root.rglob("test_*.py")):
-        try:
-            text = path.read_text(encoding="utf-8", errors="replace")
-        except OSError:
-            continue
-        if pattern.search(text):
-            hits.append(path)
-    return hits
+    # os.walk, not Path.rglob: rglob raises FileNotFoundError when a directory
+    # (a sibling job's __pycache__) vanishes mid-scan; os.walk skips it.
+    for dirpath, _dirnames, filenames in os.walk(root):
+        for fname in filenames:
+            if not (fname.startswith("test_") and fname.endswith(".py")):
+                continue
+            path = Path(dirpath) / fname
+            try:
+                text = path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            if pattern.search(text):
+                hits.append(path)
+    return sorted(hits)
 
 
 def main(argv: list[str]) -> int:
diff --git a/tests/test_managed_runtime_resolution.py b/tests/test_managed_runtime_resolution.py
index f5406974e9..410ab80255 100644
--- a/tests/test_managed_runtime_resolution.py
+++ b/tests/test_managed_runtime_resolution.py
@@ -39,14 +39,15 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
 _MANAGED_COMMANDS = frozenset({"uv", "node", "npm", "npx"})
 
 # Directories that are not Hermes-owned subprocess code: plugins ship their own
-# resolution policy, tests assert against PATH deliberately, and skills/scripts
-# run as standalone user-invoked programs.
+# resolution policy, tests assert against PATH deliberately, and skills/scripts/
+# evals run as standalone user-invoked programs.
 _EXEMPT_DIRS = (
     "tests",
     "plugins",
     "skills",
     "optional-skills",
     "scripts",
+    "evals",
     "website",
     "node_modules",
     ".git",