diff --git a/scripts/ci/list_os_marked_tests.py b/scripts/ci/list_os_marked_tests.py
index bfbc0a60c1..f3eedf87fe 100644
--- a/scripts/ci/list_os_marked_tests.py
+++ b/scripts/ci/list_os_marked_tests.py
@@ -26,6 +26,7 @@ Prints one path per line (POSIX separators, repo-relative), sorted.
 
 from __future__ import annotations
 
+import os
 import re
 import sys
 from pathlib import Path
@@ -43,14 +44,20 @@ def find_marked_files(marker: str, root: Path) -> list[Path]:
     """
     pattern = re.compile(rf"\b{re.escape(marker)}\b")
     hits: list[Path] = []
-    for path in sorted(root.rglob("test_*.py")):
-        try:
-            text = path.read_text(encoding="utf-8", errors="replace")
-        except OSError:
-            continue
-        if pattern.search(text):
-            hits.append(path)
-    return hits
+    # os.walk, not Path.rglob: rglob raises FileNotFoundError when a directory
+    # (a sibling job's __pycache__) vanishes mid-scan; os.walk skips it.
+    for dirpath, _dirnames, filenames in os.walk(root):
+        for fname in filenames:
+            if not (fname.startswith("test_") and fname.endswith(".py")):
+                continue
+            path = Path(dirpath) / fname
+            try:
+                text = path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            if pattern.search(text):
+                hits.append(path)
+    return sorted(hits)
 
 
 def main(argv: list[str]) -> int:
diff --git a/tests/test_managed_runtime_resolution.py b/tests/test_managed_runtime_resolution.py
index f5406974e9..410ab80255 100644
--- a/tests/test_managed_runtime_resolution.py
+++ b/tests/test_managed_runtime_resolution.py
@@ -39,14 +39,15 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
 _MANAGED_COMMANDS = frozenset({"uv", "node", "npm", "npx"})
 
 # Directories that are not Hermes-owned subprocess code: plugins ship their own
-# resolution policy, tests assert against PATH deliberately, and skills/scripts
-# run as standalone user-invoked programs.
+# resolution policy, tests assert against PATH deliberately, and skills/scripts/
+# evals run as standalone user-invoked programs.
 _EXEMPT_DIRS = (
     "tests",
     "plugins",
     "skills",
     "optional-skills",
     "scripts",
+    "evals",
     "website",
     "node_modules",
     ".git",