16fba82ce3
_Binder.visit_Global marked the name ambiguous in the module, so `global execute` in a function hid `from subprocess import run as execute` and HX006 (missing timeout) never fired on `execute(cmd)`; nonlocal did the same for an enclosing function's import. A declaration selects the outer binding, it does not rebind it: declared names now resolve in the scope they point at, and only an assignment (or import) through the declaration rebinds that outer name.
930 lines
39 KiB
Python
930 lines
39 KiB
Python
"""Hermes-specific AST rules (the ``HX`` ids in ``config.RULES``).
|
|
|
|
Each checker is a small function ``(tree, ctx) -> iterable of line numbers``; the table at the
|
|
bottom maps rule ids to checkers. Checkers favour precision: a rule that cries wolf gets an
|
|
allow comment on every hit and stops meaning anything.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import re
|
|
from collections.abc import Callable, Iterable, Iterator
|
|
from dataclasses import dataclass, field
|
|
|
|
_FUNCS = (ast.FunctionDef, ast.AsyncFunctionDef)
|
|
_CAPTURE_CALLS = {
|
|
"get_hermes_home",
|
|
"display_hermes_home",
|
|
"load_config",
|
|
"load_config_readonly",
|
|
"read_raw_config",
|
|
"get_secret",
|
|
"getcwd",
|
|
"expanduser",
|
|
"_float_env",
|
|
"_int_env",
|
|
"_bool_env",
|
|
"_env_float",
|
|
"_env_int",
|
|
"_env_bool",
|
|
}
|
|
_SYNC_CONFIG_CALLS = {"load_config", "save_config", "read_raw_config", "load_config_readonly"}
|
|
_SUBPROCESS_WAITS = {"run", "call", "check_call", "check_output"}
|
|
# health: allow HX003 -- the detector's own pattern list
|
|
_SHELL_IDENTITY = ("pgrep -f", "ps aux", "ps -ef", "ps -eo")
|
|
|
|
|
|
@dataclass
|
|
class Ctx:
|
|
known_env: set[str] = field(default_factory=set)
|
|
|
|
|
|
@dataclass
|
|
class _Scope:
|
|
"""One lexical scope's bindings: names an import binds, and names anything else binds."""
|
|
|
|
parent: _Scope | None = None
|
|
kind: str = "function" # "module" | "class" | "function" | "comprehension"
|
|
imports: dict[str, str] = field(default_factory=dict)
|
|
others: set[str] = field(default_factory=set)
|
|
# Names a `global` / `nonlocal` statement hands to an outer scope: they bind nothing here.
|
|
declared: dict[str, str] = field(default_factory=dict)
|
|
|
|
def owner(self, name: str) -> _Scope:
|
|
"""The scope that binds ``name`` for code in this one, following `global`/`nonlocal`."""
|
|
scope = self
|
|
while name in scope.declared:
|
|
if scope.declared[name] == "global":
|
|
while scope.parent is not None:
|
|
scope = scope.parent
|
|
return scope
|
|
outer = scope.parent # nonlocal: the nearest enclosing function that has it
|
|
while outer is not None and outer.parent is not None and (
|
|
outer.kind == "class" or not outer.binds(name)):
|
|
outer = outer.parent
|
|
if outer is None or outer.parent is None:
|
|
return scope
|
|
scope = outer
|
|
return scope
|
|
|
|
def binds(self, name: str) -> bool:
|
|
return name in self.others or name in self.imports or name in self.declared
|
|
|
|
def resolve(self, name: str) -> str | None:
|
|
"""Import target ``name`` means here: the innermost scope binding it decides, and a
|
|
scope that binds it any other way (or by two imports) makes it a plain local. Class
|
|
bodies are invisible to the functions and comprehensions nested in them."""
|
|
scope: _Scope | None = self
|
|
while scope is not None:
|
|
if scope is self or scope.kind != "class":
|
|
if name in scope.declared:
|
|
if name in scope.others: # rebound before its outer binding was seen
|
|
return None
|
|
scope = scope.owner(name)
|
|
return scope.imports.get(name) if name not in scope.others else None
|
|
if name in scope.others:
|
|
return None
|
|
if name in scope.imports:
|
|
return scope.imports[name]
|
|
scope = scope.parent
|
|
return None
|
|
|
|
|
|
class _Binder(ast.NodeVisitor):
|
|
"""Records each scope's bindings and the scope every loaded name is read in."""
|
|
|
|
def __init__(self) -> None:
|
|
self.module = self.scope = _Scope(kind="module")
|
|
self.loads: dict[int, _Scope] = {}
|
|
|
|
def _bind(self, name: str, scope: _Scope | None = None) -> None:
|
|
(scope or self.scope).owner(name).others.add(name)
|
|
|
|
def _import(self, name: str, target: str) -> None:
|
|
scope = self.scope.owner(name)
|
|
if scope.imports.get(name, target) != target:
|
|
self._bind(name)
|
|
scope.imports[name] = target
|
|
|
|
def _enter(self, kind: str, nodes: Iterable[ast.AST]) -> None:
|
|
outer, self.scope = self.scope, _Scope(self.scope, kind)
|
|
for node in nodes:
|
|
self.visit(node)
|
|
self.scope = outer
|
|
|
|
def visit_Import(self, node: ast.Import) -> None:
|
|
for alias in node.names:
|
|
top = alias.name.split(".")[0]
|
|
self._import(alias.asname or top, alias.name if alias.asname else top)
|
|
|
|
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
|
# A relative import keeps its module path minus the dots: rules match the leaf.
|
|
for alias in node.names:
|
|
if alias.name != "*":
|
|
target = f"{node.module}.{alias.name}" if node.module else alias.name
|
|
self._import(alias.asname or alias.name, target)
|
|
|
|
def visit_Name(self, node: ast.Name) -> None:
|
|
if isinstance(node.ctx, ast.Load):
|
|
self.loads[id(node)] = self.scope
|
|
else:
|
|
self._bind(node.id)
|
|
|
|
# A declaration selects the outer binding; only an assignment through it rebinds that.
|
|
def visit_Global(self, node: ast.Global) -> None:
|
|
self.scope.declared.update(dict.fromkeys(node.names, "global"))
|
|
|
|
def visit_Nonlocal(self, node: ast.Nonlocal) -> None:
|
|
self.scope.declared.update(dict.fromkeys(node.names, "nonlocal"))
|
|
|
|
def generic_visit(self, node: ast.AST) -> None:
|
|
# except-as, match captures and **rest bind a plain string attribute.
|
|
for attr in ("name", "rest"):
|
|
value = getattr(node, attr, None)
|
|
if isinstance(value, str) and not isinstance(node, ast.alias):
|
|
self._bind(value)
|
|
super().generic_visit(node)
|
|
|
|
def _visit_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda) -> None:
|
|
"""Decorators, defaults and annotations run in the enclosing scope; the body does not."""
|
|
args = node.args
|
|
every = [*args.posonlyargs, *args.args, *args.kwonlyargs, args.vararg, args.kwarg]
|
|
params = [a for a in every if a is not None]
|
|
returns = getattr(node, "returns", None)
|
|
outer = [*getattr(node, "decorator_list", []), *args.defaults,
|
|
*(d for d in args.kw_defaults if d is not None),
|
|
*(a.annotation for a in params if a.annotation), *([returns] if returns else [])]
|
|
for expr in outer:
|
|
self.visit(expr)
|
|
if not isinstance(node, ast.Lambda):
|
|
self._bind(node.name)
|
|
self._enter("function", [*params, *(node.body if isinstance(node.body, list) else [node.body])])
|
|
|
|
visit_FunctionDef = visit_AsyncFunctionDef = visit_Lambda = _visit_function
|
|
|
|
def visit_arg(self, node: ast.arg) -> None:
|
|
self._bind(node.arg)
|
|
|
|
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
for expr in (*node.decorator_list, *node.bases, *node.keywords):
|
|
self.visit(expr)
|
|
self._bind(node.name)
|
|
self._enter("class", node.body)
|
|
|
|
def _visit_comprehension(self, node: ast.ListComp | ast.SetComp | ast.DictComp | ast.GeneratorExp) -> None:
|
|
"""The first iterable runs outside; targets, conditions and elements run inside."""
|
|
first, *rest = node.generators
|
|
self.visit(first.iter)
|
|
elements = [node.key, node.value] if isinstance(node, ast.DictComp) else [node.elt]
|
|
inner = [first.target, *first.ifs, *(n for g in rest for n in (g.iter, g.target, *g.ifs))]
|
|
self._enter("comprehension", [*inner, *elements])
|
|
|
|
visit_ListComp = visit_SetComp = visit_DictComp = visit_GeneratorExp = _visit_comprehension
|
|
|
|
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
|
|
# A walrus in a comprehension binds in the enclosing function (PEP 572).
|
|
scope = self.scope
|
|
while scope.kind == "comprehension" and scope.parent is not None:
|
|
scope = scope.parent
|
|
self._bind(node.target.id, scope)
|
|
self.visit(node.value)
|
|
|
|
|
|
def canonical_tree(tree: ast.Module) -> ast.Module:
|
|
"""``tree`` with every import-bound name spelled out: ``sp.run`` -> ``subprocess.run``,
|
|
``execute`` (``from subprocess import run as execute``) -> ``subprocess.run``.
|
|
|
|
Rules then match canonical API names, so an alias neither hides a call nor lets a local
|
|
helper that merely shares a leaf name (``def wait_for``) pass as the real API. Each read is
|
|
resolved in its own scope: a parameter or local of the same name shadows the import only
|
|
in the function that binds it, and a scope that rebinds the name itself (def, class,
|
|
assignment, parameter) leaves it alone there, since the name is ambiguous.
|
|
"""
|
|
binder = _Binder()
|
|
binder.visit(tree)
|
|
|
|
class _Spell(ast.NodeTransformer):
|
|
def visit_Name(self, node: ast.Name) -> ast.AST:
|
|
scope = binder.loads.get(id(node))
|
|
target = scope.resolve(node.id) if scope else None
|
|
if target is None or target == node.id:
|
|
return node
|
|
parts = target.split(".")
|
|
expr: ast.expr = ast.Name(parts[0], ast.Load())
|
|
for part in parts[1:]:
|
|
expr = ast.Attribute(expr, part, ast.Load())
|
|
return ast.copy_location(expr, node)
|
|
|
|
return ast.fix_missing_locations(_Spell().visit(tree))
|
|
|
|
|
|
def _dotted(node: ast.AST) -> str:
|
|
"""``os.environ.get`` for an Attribute chain, ``""`` for anything else."""
|
|
parts: list[str] = []
|
|
while isinstance(node, ast.Attribute):
|
|
parts.append(node.attr)
|
|
node = node.value
|
|
if isinstance(node, ast.Name):
|
|
parts.append(node.id)
|
|
return ".".join(reversed(parts))
|
|
if isinstance(node, ast.Call):
|
|
inner = _dotted(node.func)
|
|
return ".".join([f"{inner}()", *reversed(parts)]) if inner else ""
|
|
return ""
|
|
|
|
|
|
def _call_name(call: ast.Call) -> str:
|
|
return _dotted(call.func)
|
|
|
|
|
|
def _str_arg(call: ast.Call, index: int = 0) -> str | None:
|
|
if len(call.args) > index:
|
|
arg = call.args[index]
|
|
if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
|
|
return arg.value
|
|
return None
|
|
|
|
|
|
def _env_read_name(node: ast.AST) -> str | None:
|
|
"""Env var name for ``os.getenv("X")`` / ``os.environ.get("X")`` / ``os.environ["X"]``."""
|
|
if isinstance(node, ast.Call):
|
|
if _call_name(node) in ("os.getenv", "os.environ.get", "environ.get", "getenv"):
|
|
return _str_arg(node)
|
|
return None
|
|
if isinstance(node, ast.Subscript) and isinstance(node.ctx, ast.Load):
|
|
if _dotted(node.value) in ("os.environ", "environ"):
|
|
key = node.slice
|
|
if isinstance(key, ast.Constant) and isinstance(key.value, str):
|
|
return key.value
|
|
if isinstance(node, ast.Compare) and len(node.ops) == 1:
|
|
if isinstance(node.ops[0], (ast.In, ast.NotIn)):
|
|
if _dotted(node.comparators[0]) in ("os.environ", "environ"):
|
|
left = node.left
|
|
if isinstance(left, ast.Constant) and isinstance(left.value, str):
|
|
return left.value
|
|
return None
|
|
|
|
|
|
def _env_write_names(node: ast.AST) -> list[str]:
|
|
"""Env var names a node SETS: ``os.environ["X"] = ...``, ``setdefault``, ``update``, ``putenv``."""
|
|
if isinstance(node, ast.Subscript) and not isinstance(node.ctx, ast.Load):
|
|
key = node.slice
|
|
if _dotted(node.value) == "os.environ" and isinstance(key, ast.Constant) and isinstance(key.value, str):
|
|
return [key.value]
|
|
return []
|
|
if not isinstance(node, ast.Call):
|
|
return []
|
|
name = _call_name(node)
|
|
if name in ("os.environ.setdefault", "os.putenv"):
|
|
key = _str_arg(node)
|
|
return [key] if key else []
|
|
if name == "os.environ.update":
|
|
keys = [kw.arg for kw in node.keywords if kw.arg]
|
|
for arg in node.args:
|
|
if isinstance(arg, ast.Dict):
|
|
keys += [k.value for k in arg.keys if isinstance(k, ast.Constant) and isinstance(k.value, str)]
|
|
return keys
|
|
return []
|
|
|
|
|
|
def _deferred_parts(node: ast.AST) -> list[ast.AST] | None:
|
|
"""For a node whose evaluation defers part of itself, the parts that run NOW; else None.
|
|
|
|
A lambda/def runs only its defaults and decorators when evaluated; a generator expression
|
|
runs only its first iterable (the element, conditions and later loops run on consumption).
|
|
Comprehensions and class bodies run immediately, so they are not deferred.
|
|
"""
|
|
if isinstance(node, ast.Lambda):
|
|
return [d for d in (*node.args.defaults, *node.args.kw_defaults) if d is not None]
|
|
if isinstance(node, _FUNCS):
|
|
defaults = [d for d in (*node.args.defaults, *node.args.kw_defaults) if d is not None]
|
|
return [*node.decorator_list, *defaults]
|
|
if isinstance(node, ast.GeneratorExp):
|
|
return [node.generators[0].iter]
|
|
return None
|
|
|
|
|
|
def _eager(node: ast.AST) -> Iterator[ast.AST]:
|
|
"""Every node evaluated when ``node`` is evaluated (or a statement executes), root included."""
|
|
stack = [node]
|
|
while stack:
|
|
current = stack.pop()
|
|
parts = _deferred_parts(current)
|
|
if parts is not None:
|
|
stack.extend(parts)
|
|
continue
|
|
yield current
|
|
stack.extend(ast.iter_child_nodes(current))
|
|
|
|
|
|
def _hermes_home_path(text: str, prefix: str = "") -> bool:
|
|
"""``text`` (after ``prefix``) starts with the exact ``.hermes`` path component, so
|
|
``.hermes/x`` matches and ``.hermes-profile-exports`` does not."""
|
|
if not text.startswith(prefix):
|
|
return False
|
|
return re.split(r"[/\\]", text[len(prefix):].lstrip("/\\"), maxsplit=1)[0] == ".hermes"
|
|
|
|
|
|
def hardcoded_home(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div):
|
|
right = node.right
|
|
if (
|
|
isinstance(node.left, ast.Call)
|
|
and _call_name(node.left).endswith("Path.home")
|
|
and isinstance(right, ast.Constant)
|
|
and isinstance(right.value, str)
|
|
and _hermes_home_path(right.value)
|
|
):
|
|
yield node.lineno
|
|
elif isinstance(node, ast.Call):
|
|
name = _call_name(node)
|
|
arg = _str_arg(node)
|
|
if arg and _hermes_home_path(arg, "~/") and name.endswith(("expanduser", "Path")):
|
|
yield node.lineno
|
|
|
|
|
|
def new_env_var(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
# Setting a variable introduces it as surely as reading one does.
|
|
for node in ast.walk(tree):
|
|
read = _env_read_name(node)
|
|
names = [read] if read else _env_write_names(node)
|
|
if any(n.startswith("HERMES_") and n not in ctx.known_env for n in names):
|
|
yield getattr(node, "lineno", 0)
|
|
|
|
|
|
def argv_identity(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
# A docstring or bare string statement is prose about the pattern, not a command.
|
|
prose = {id(n.value) for n in ast.walk(tree)
|
|
if isinstance(n, ast.Expr) and isinstance(n.value, ast.Constant)}
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Compare) and len(node.ops) == 1:
|
|
if isinstance(node.ops[0], (ast.In, ast.NotIn)):
|
|
left = node.left
|
|
target = ast.unparse(node.comparators[0])
|
|
if isinstance(left, ast.Constant) and isinstance(left.value, str):
|
|
if "cmdline" in target:
|
|
yield node.lineno
|
|
elif isinstance(node, ast.Constant) and isinstance(node.value, str) and id(node) not in prose:
|
|
if any(cmd in node.value for cmd in _SHELL_IDENTITY):
|
|
yield node.lineno
|
|
|
|
|
|
def unscoped_secret_fallback(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.ExceptHandler) or node.type is None:
|
|
continue
|
|
if "UnscopedSecretError" not in ast.unparse(node.type):
|
|
continue
|
|
for stmt in node.body:
|
|
if any(_env_read_name(inner) for inner in ast.walk(stmt)):
|
|
yield node.lineno
|
|
break
|
|
|
|
|
|
def _is_capture(node: ast.AST) -> bool:
|
|
if isinstance(node, ast.Call):
|
|
name = _call_name(node)
|
|
if name.rsplit(".", 1)[-1] in _CAPTURE_CALLS or name.endswith("Path.home"):
|
|
return True
|
|
return bool(_env_read_name(node))
|
|
|
|
|
|
def _capture_lines(expr: ast.AST) -> Iterator[int]:
|
|
"""Every independent capture ``expr`` evaluates: each dict entry or list item is its own
|
|
occurrence, so one added next to an existing one is new debt. A capture nested inside
|
|
another (``expanduser(getenv(...))``) is the same occurrence and is not entered.
|
|
Deferred bodies (lambda, def, generator element) read at call time: that is the fix."""
|
|
stack = [expr]
|
|
while stack:
|
|
node = stack.pop()
|
|
parts = _deferred_parts(node)
|
|
if parts is not None:
|
|
stack.extend(parts)
|
|
elif _is_capture(node):
|
|
yield getattr(node, "lineno", 0)
|
|
else:
|
|
stack.extend(ast.iter_child_nodes(node))
|
|
|
|
|
|
def _main_guard(stmt: ast.stmt) -> str | None:
|
|
"""``"=="``/``"!="`` for ``if __name__ <op> "__main__":``, else None."""
|
|
test = stmt.test if isinstance(stmt, ast.If) else None
|
|
if not (isinstance(test, ast.Compare) and len(test.ops) == 1 and len(test.comparators) == 1):
|
|
return None
|
|
sides = {ast.unparse(test.left), ast.unparse(test.comparators[0])}
|
|
if sides != {"__name__", "'__main__'"}:
|
|
return None
|
|
return {ast.Eq: "==", ast.NotEq: "!="}.get(type(test.ops[0]))
|
|
|
|
|
|
# Statement blocks that execute when the enclosing block does (``match`` cases via ``cases``).
|
|
_BLOCKS = ("body", "orelse", "finalbody")
|
|
_COMPOUND = (ast.If, ast.Try, ast.TryStar, ast.With, ast.For, ast.While, ast.Match)
|
|
|
|
|
|
def _import_time_blocks(stmt: ast.stmt) -> Iterator[list[ast.stmt]]:
|
|
guard = _main_guard(stmt)
|
|
if isinstance(stmt, ast.If) and guard is not None:
|
|
# Only the branch that runs on import: ``else`` of ``==``, body of ``!=``.
|
|
yield stmt.orelse if guard == "==" else stmt.body
|
|
return
|
|
for name in _BLOCKS:
|
|
yield getattr(stmt, name, None) or []
|
|
for handler in getattr(stmt, "handlers", None) or []:
|
|
yield handler.body
|
|
for case in getattr(stmt, "cases", None) or []:
|
|
yield case.body
|
|
|
|
|
|
def _import_time_statements(body: list[ast.stmt]) -> Iterator[ast.stmt]:
|
|
"""Statements that run at import: module/class bodies and every compound block in them."""
|
|
for stmt in body:
|
|
yield stmt
|
|
if isinstance(stmt, ast.ClassDef):
|
|
yield from _import_time_statements(stmt.body)
|
|
elif isinstance(stmt, _COMPOUND):
|
|
for block in _import_time_blocks(stmt):
|
|
yield from _import_time_statements(block)
|
|
|
|
|
|
def _postponed_annotations(tree: ast.Module) -> bool:
|
|
return any(isinstance(stmt, ast.ImportFrom) and stmt.module == "__future__"
|
|
and any(alias.name == "annotations" for alias in stmt.names) for stmt in tree.body)
|
|
|
|
|
|
def _annotations(func: ast.FunctionDef | ast.AsyncFunctionDef) -> list[ast.expr]:
|
|
args = func.args
|
|
params = [*args.posonlyargs, *args.args, *args.kwonlyargs, args.vararg, args.kwarg]
|
|
return [n for n in (*(p.annotation for p in params if p is not None), func.returns) if n is not None]
|
|
|
|
|
|
def _import_time_exprs(stmt: ast.stmt, postponed: bool) -> Iterator[ast.AST]:
|
|
"""What a statement evaluates at import, besides its nested blocks (walked separately).
|
|
|
|
Annotations of a def and of a module/class variable run at definition time on the
|
|
supported interpreters (3.11-3.13; 3.14 defers them, PEP 649) unless the module has
|
|
``from __future__ import annotations``; an annotated assignment's value always runs."""
|
|
if isinstance(stmt, ast.ClassDef):
|
|
yield from (*stmt.decorator_list, *stmt.bases, *(kw.value for kw in stmt.keywords))
|
|
elif isinstance(stmt, ast.AnnAssign):
|
|
yield from (n for n in (stmt.target, stmt.value) if n is not None)
|
|
if not postponed:
|
|
yield stmt.annotation
|
|
elif isinstance(stmt, (ast.Assign, ast.AugAssign, *_FUNCS)):
|
|
yield stmt # _eager keeps only a def's decorators and defaults
|
|
if isinstance(stmt, _FUNCS) and not postponed:
|
|
yield from _annotations(stmt)
|
|
elif isinstance(stmt, ast.Expr):
|
|
# A bare call is an action, not a capture; a walrus inside it binds a module name.
|
|
yield from (n for n in _eager(stmt.value) if isinstance(n, ast.NamedExpr))
|
|
elif isinstance(stmt, (ast.If, ast.While)):
|
|
yield stmt.test
|
|
elif isinstance(stmt, ast.For):
|
|
yield stmt.iter
|
|
elif isinstance(stmt, ast.With):
|
|
yield from (item.context_expr for item in stmt.items)
|
|
elif isinstance(stmt, ast.Match):
|
|
yield stmt.subject
|
|
yield from (case.guard for case in stmt.cases if case.guard is not None)
|
|
|
|
|
|
def import_time_capture(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
postponed = _postponed_annotations(tree)
|
|
for stmt in _import_time_statements(tree.body):
|
|
for expr in _import_time_exprs(stmt, postponed):
|
|
yield from _capture_lines(expr)
|
|
|
|
|
|
_INFINITE = {"inf", "+inf", "infinity", "+infinity"}
|
|
|
|
|
|
def _finite(node: ast.AST | None) -> bool:
|
|
"""A deadline that is present and not statically disabled: ``None``, ``math.inf``,
|
|
``float("inf")`` and an overflowing literal all wait forever. An unknown expression is
|
|
trusted (it is usually a configured value)."""
|
|
if node is None:
|
|
return False
|
|
if isinstance(node, ast.Constant):
|
|
return node.value is not None and node.value != float("inf")
|
|
if _dotted(node).rpartition(".")[2] == "inf":
|
|
return False
|
|
if isinstance(node, ast.Call) and _call_name(node) == "float":
|
|
arg = _str_arg(node)
|
|
return arg is None or arg.strip().lower() not in _INFINITE
|
|
return True
|
|
|
|
|
|
def _deadline(call: ast.Call, name: str = "timeout", position: int | None = None) -> bool:
|
|
"""True when ``call`` passes a finite deadline: ``name=<not None>``, the positional slot,
|
|
or ``**opts`` (an unknown mapping is trusted; a literal one must name the deadline)."""
|
|
if position is not None and len(call.args) > position:
|
|
return _finite(call.args[position])
|
|
for kw in call.keywords:
|
|
if kw.arg == name:
|
|
return _finite(kw.value)
|
|
if kw.arg is None:
|
|
if not isinstance(kw.value, ast.Dict):
|
|
return True
|
|
for key, value in zip(kw.value.keys, kw.value.values, strict=True):
|
|
if isinstance(key, ast.Constant) and key.value == name:
|
|
return _finite(value)
|
|
return False
|
|
|
|
|
|
def _awaited_calls(body: list[ast.stmt]) -> Iterator[ast.Call]:
|
|
"""Calls awaited while ``body`` runs; a coroutine defined there runs later, unbounded."""
|
|
for stmt in body:
|
|
for node in _eager(stmt):
|
|
if isinstance(node, ast.Await) and isinstance(node.value, ast.Call):
|
|
yield node.value
|
|
|
|
|
|
_WAIT_FOR = {"asyncio.wait_for"}
|
|
# deadline context manager -> its deadline parameter (positional slot 0)
|
|
_TIMEOUT_CMS = {
|
|
"asyncio.timeout": "delay",
|
|
"asyncio.timeout_at": "when",
|
|
"async_timeout.timeout": "delay",
|
|
"async_timeout.timeout_at": "deadline",
|
|
}
|
|
_SPAWNS = {
|
|
"subprocess.Popen": "sync",
|
|
"asyncio.create_subprocess_exec": "async",
|
|
"asyncio.create_subprocess_shell": "async",
|
|
}
|
|
|
|
|
|
def _bounded_calls(tree: ast.Module) -> set[int]:
|
|
"""ids of calls an asyncio deadline bounds: the awaitable passed to ``asyncio.wait_for(x,
|
|
<finite>)`` and every call awaited directly inside ``async with asyncio.timeout(<finite>):``.
|
|
Only the real APIs count (names are canonical); a same-named local helper bounds nothing."""
|
|
bounded: set[int] = set()
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Call) and _call_name(node) in _WAIT_FOR:
|
|
awaitable = node.args[0] if node.args else next(
|
|
(kw.value for kw in node.keywords if kw.arg in ("fut", "aw")), None)
|
|
if awaitable is not None and _deadline(node, "timeout", 1):
|
|
bounded.add(id(awaitable))
|
|
elif isinstance(node, ast.AsyncWith):
|
|
for item in node.items:
|
|
ctx_call = item.context_expr
|
|
if not isinstance(ctx_call, ast.Call):
|
|
continue
|
|
slot = _TIMEOUT_CMS.get(_call_name(ctx_call))
|
|
if slot and _deadline(ctx_call, slot, 0):
|
|
bounded.update(id(call) for call in _awaited_calls(node.body))
|
|
return bounded
|
|
|
|
|
|
def _spawn_kind(value: ast.AST | None) -> str | None:
|
|
if isinstance(value, ast.Await):
|
|
value = value.value
|
|
return _SPAWNS.get(_call_name(value)) if isinstance(value, ast.Call) else None
|
|
|
|
|
|
_SCOPES = (*_FUNCS, ast.Lambda, ast.ClassDef)
|
|
|
|
|
|
def _own_nodes(scope: ast.AST) -> Iterator[ast.AST]:
|
|
"""Nodes of ``scope`` itself: a nested def/lambda/class is yielded but not entered."""
|
|
stack = list(ast.iter_child_nodes(scope))
|
|
while stack:
|
|
node = stack.pop()
|
|
yield node
|
|
if not isinstance(node, _SCOPES):
|
|
stack.extend(ast.iter_child_nodes(node))
|
|
|
|
|
|
def _scopes(tree: ast.Module) -> Iterator[tuple[ast.AST, ast.AST | None, list[ast.AST]]]:
|
|
"""``(scope, enclosing scope, its own nodes)`` for the module and every def/lambda/class."""
|
|
todo: list[tuple[ast.AST, ast.AST | None]] = [(tree, None)]
|
|
while todo:
|
|
scope, parent = todo.pop()
|
|
own = list(_own_nodes(scope))
|
|
todo.extend((node, scope) for node in own if isinstance(node, _SCOPES))
|
|
yield scope, parent, own
|
|
|
|
|
|
def _binding_pairs(node: ast.AST) -> list[tuple[ast.AST, ast.AST | None]]:
|
|
if isinstance(node, ast.Assign):
|
|
return [(t, node.value) for t in node.targets]
|
|
if isinstance(node, (ast.AnnAssign, ast.NamedExpr)):
|
|
return [(node.target, node.value)]
|
|
if isinstance(node, (ast.With, ast.AsyncWith)):
|
|
return [(i.optional_vars, i.context_expr) for i in node.items if i.optional_vars]
|
|
return []
|
|
|
|
|
|
def _pos(node: ast.AST, end: bool = False) -> tuple[int, int]:
|
|
if end:
|
|
return (getattr(node, "end_lineno", 0) or 0, getattr(node, "end_col_offset", 0) or 0)
|
|
return (getattr(node, "lineno", 0), getattr(node, "col_offset", 0))
|
|
|
|
|
|
@dataclass
|
|
class _HandleScope:
|
|
parent: _HandleScope | None
|
|
is_class: bool
|
|
owner: int # id of the class (or module) whose methods share ``self._proc``-style handles
|
|
# name -> [(position the binding takes effect, "sync"/"async"/None for anything else)]
|
|
names: dict[str, list[tuple[tuple[int, int], str | None]]] = field(default_factory=dict)
|
|
|
|
def kind(self, name: str, at: tuple[int, int]) -> str | None:
|
|
"""What ``name`` holds at ``at``: the innermost scope that binds it decides (a class
|
|
body is invisible to its methods), and in it the latest binding before ``at``."""
|
|
scope: _HandleScope | None = self
|
|
while scope is not None:
|
|
if (scope is self or not scope.is_class) and name in scope.names:
|
|
before = [k for pos, k in sorted(scope.names[name], key=lambda b: b[0]) if pos <= at]
|
|
return before[-1] if before else None
|
|
scope = scope.parent
|
|
return None
|
|
|
|
|
|
# A parameter annotated as a process is one (``proc: subprocess.Popen[str]``).
|
|
_PROCESS_TYPES = {"subprocess.Popen": "sync", "asyncio.subprocess.Process": "async"}
|
|
|
|
|
|
def _annotated_kind(annotation: ast.expr | None) -> str | None:
|
|
if isinstance(annotation, ast.Subscript):
|
|
annotation = annotation.value
|
|
return _PROCESS_TYPES.get(_dotted(annotation)) if annotation is not None else None
|
|
|
|
|
|
def _record_bindings(scope: _HandleScope, node: ast.AST, attrs: dict[tuple[int, str], str]) -> None:
|
|
"""Any binding of a plain name clears it (a loop variable, ``proc = None``, a parameter
|
|
not annotated as a process); a spawn makes it a handle once the value is computed. An attribute handle is shared by
|
|
the class's methods, so it is keyed by the class, not by the function that assigns it."""
|
|
if isinstance(node, ast.Name) and not isinstance(node.ctx, ast.Load):
|
|
scope.names.setdefault(node.id, []).append((_pos(node), None))
|
|
elif isinstance(node, ast.arg):
|
|
scope.names.setdefault(node.arg, []).append((_pos(node), _annotated_kind(node.annotation)))
|
|
for target, value in _binding_pairs(node):
|
|
kind = _spawn_kind(value)
|
|
if kind is None or value is None:
|
|
continue
|
|
if isinstance(target, ast.Name):
|
|
# takes effect after both sides: `with Popen() as proc` binds after the `as`
|
|
at = max(_pos(target, end=True), _pos(value, end=True))
|
|
scope.names.setdefault(target.id, []).append((at, kind))
|
|
elif _dotted(target):
|
|
attrs[(scope.owner, _dotted(target))] = kind
|
|
|
|
|
|
def _process_kinds(tree: ast.Module) -> dict[int, str]:
|
|
"""id of each ``<receiver>.<method>()`` call whose receiver holds a child process ->
|
|
``sync``/``async``. Bindings are per lexical scope, so a ``proc`` in one function never
|
|
classifies a same-spelled ``proc`` in another, and a reassignment re-classifies it."""
|
|
built: dict[int, _HandleScope] = {}
|
|
attrs: dict[tuple[int, str], str] = {}
|
|
calls: list[tuple[ast.Call, ast.expr, _HandleScope]] = []
|
|
for node, parent, own in _scopes(tree):
|
|
outer = built.get(id(parent)) if parent is not None else None
|
|
owner = id(node) if outer is None or isinstance(node, ast.ClassDef) else outer.owner
|
|
scope = built[id(node)] = _HandleScope(outer, isinstance(node, ast.ClassDef), owner)
|
|
for child in own:
|
|
_record_bindings(scope, child, attrs)
|
|
if isinstance(child, ast.Call) and isinstance(child.func, ast.Attribute):
|
|
calls.append((child, child.func.value, scope))
|
|
kinds: dict[int, str] = {}
|
|
for call, receiver, scope in calls:
|
|
if isinstance(receiver, ast.Name):
|
|
kind = scope.kind(receiver.id, _pos(call))
|
|
else:
|
|
kind = attrs.get((scope.owner, _dotted(receiver)))
|
|
if kind:
|
|
kinds[id(call)] = kind
|
|
return kinds
|
|
|
|
|
|
# Process methods that wait for the child, with the slot of their ``timeout`` (sync Popen);
|
|
# the asyncio Process versions take no timeout and must be bounded by asyncio.
|
|
_PROCESS_WAITS = {"communicate": 1, "wait": 0}
|
|
# A reaping wait's value may be dropped, assigned (annotated or not) or returned.
|
|
_WAIT_STATEMENTS = (ast.Expr, ast.Assign, ast.AnnAssign, ast.Return)
|
|
|
|
|
|
def _reaped_after_kill(tree: ast.Module, kinds: dict[int, str]) -> set[int]:
|
|
"""ids of a sync ``proc.wait()`` that directly follows ``proc.kill()``: SIGKILL bounds it.
|
|
Not ``communicate()`` (it reads until every grandchild holding the pipe exits) and not the
|
|
asyncio ``Process.wait()`` (it waits for the pipe transports too); both measured to hang."""
|
|
reaped: set[int] = set()
|
|
for node in ast.walk(tree):
|
|
for field in ("body", "orelse", "finalbody"):
|
|
stmts = getattr(node, field, None)
|
|
if not isinstance(stmts, list):
|
|
continue
|
|
for first, second in zip(stmts, stmts[1:]):
|
|
kill = first.value if isinstance(first, ast.Expr) else None
|
|
wait = second.value if isinstance(second, _WAIT_STATEMENTS) else None
|
|
if not (isinstance(kill, ast.Call) and isinstance(wait, ast.Call)):
|
|
continue
|
|
head, _, leaf = _call_name(kill).rpartition(".")
|
|
if (leaf == "kill" and kinds.get(id(kill)) == "sync"
|
|
and _call_name(wait) == f"{head}.wait"):
|
|
reaped.add(id(wait))
|
|
return reaped
|
|
|
|
|
|
def missing_timeout(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
kinds = _process_kinds(tree)
|
|
bounded = _bounded_calls(tree) | _reaped_after_kill(tree, kinds)
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.Call) or id(node) in bounded:
|
|
continue
|
|
head, _, leaf = _call_name(node).rpartition(".")
|
|
if head == "subprocess" and leaf in _SUBPROCESS_WAITS and not _deadline(node):
|
|
yield node.lineno
|
|
elif leaf == "urlopen" and not _deadline(node, "timeout", 2):
|
|
yield node.lineno
|
|
elif leaf in _PROCESS_WAITS and id(node) in kinds:
|
|
if kinds[id(node)] == "async" or not _deadline(node, "timeout", _PROCESS_WAITS[leaf]):
|
|
yield node.lineno
|
|
|
|
|
|
def sync_config_in_async(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for func in ast.walk(tree):
|
|
if not isinstance(func, ast.AsyncFunctionDef):
|
|
continue
|
|
for stmt in func.body:
|
|
for node in _eager(stmt):
|
|
if isinstance(node, ast.Call):
|
|
if _call_name(node).rsplit(".", 1)[-1] in _SYNC_CONFIG_CALLS:
|
|
yield node.lineno
|
|
|
|
|
|
def get_event_loop(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Call) and _call_name(node) == "asyncio.get_event_loop":
|
|
yield node.lineno
|
|
|
|
|
|
def _is_exc_gather(node: ast.AST | None) -> bool:
|
|
if isinstance(node, ast.Await):
|
|
node = node.value
|
|
if not (isinstance(node, ast.Call) and _call_name(node) == "asyncio.gather"):
|
|
return False
|
|
return any(kw.arg == "return_exceptions" and getattr(kw.value, "value", False) is True
|
|
for kw in node.keywords)
|
|
|
|
|
|
def _names(target: ast.AST) -> set[str]:
|
|
return {n.id for n in ast.walk(target) if isinstance(n, ast.Name)}
|
|
|
|
|
|
def _from_results(node: ast.AST | None, results: set[str]) -> bool:
|
|
"""``node`` is a gather-with-exceptions call, a results name, or built from one
|
|
(``results[i]``, ``zip(xs, results)``, ``enumerate(results)``)."""
|
|
if node is None:
|
|
return False
|
|
if _is_exc_gather(node):
|
|
return True
|
|
if isinstance(node, ast.Name):
|
|
return node.id in results
|
|
if isinstance(node, ast.Subscript):
|
|
return _from_results(node.value, results)
|
|
if isinstance(node, ast.Call) and _call_name(node) in ("zip", "enumerate", "list", "reversed"):
|
|
return any(_from_results(arg, results) for arg in node.args)
|
|
return False
|
|
|
|
|
|
def _projected(target: ast.AST, iterable: ast.AST, results: set[str]) -> set[str]:
|
|
"""Names in ``target`` that receive a result when it takes one item of ``iterable``.
|
|
|
|
``for payload, result in zip(payloads, results)`` taints only ``result`` and
|
|
``for i, r in enumerate(results)`` only ``r``: each position follows its own source."""
|
|
if isinstance(target, (ast.Tuple, ast.List)) and isinstance(iterable, ast.Call):
|
|
name, elts, args = _call_name(iterable), target.elts, iterable.args
|
|
if name == "zip" and len(elts) == len(args):
|
|
return set().union(*(_projected(e, a, results) for e, a in zip(elts, args, strict=True)))
|
|
if name == "enumerate" and len(elts) == 2 and args:
|
|
return _projected(elts[1], args[0], results)
|
|
return _names(target) if _from_results(iterable, results) else set()
|
|
|
|
|
|
def _result_names(func: ast.AST) -> set[str]:
|
|
"""Names bound to gather(return_exceptions=True) results in ``func``'s own body."""
|
|
results: set[str] = set()
|
|
nodes = [n for stmt in func.body for n in _eager(stmt)]
|
|
for _ in range(3): # results -> loop vars -> unpacked loop vars
|
|
for node in nodes:
|
|
if isinstance(node, (ast.Assign, ast.AnnAssign)) and _from_results(node.value, results):
|
|
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
|
results |= set().union(*(_names(t) for t in targets))
|
|
elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)):
|
|
results |= _projected(node.target, node.iter, results)
|
|
return results
|
|
|
|
|
|
def gather_exception_check(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for func in ast.walk(tree):
|
|
if not isinstance(func, _FUNCS):
|
|
continue
|
|
results = _result_names(func)
|
|
if not results:
|
|
continue
|
|
for stmt in func.body:
|
|
for node in _eager(stmt):
|
|
if (
|
|
isinstance(node, ast.Call)
|
|
and _call_name(node) == "isinstance"
|
|
and len(node.args) == 2
|
|
and _dotted(node.args[1]) == "Exception"
|
|
and _from_results(node.args[0], results)
|
|
):
|
|
yield node.lineno
|
|
|
|
|
|
def bool_of_env(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Call) and _call_name(node) == "bool" and len(node.args) == 1:
|
|
arg = node.args[0]
|
|
if isinstance(arg, (ast.Call, ast.Subscript)) and _env_read_name(arg):
|
|
yield node.lineno
|
|
|
|
|
|
def _ladder_key(test: ast.expr) -> str | None:
|
|
if isinstance(test, ast.Compare) and len(test.ops) == 1:
|
|
if isinstance(test.ops[0], (ast.Eq, ast.In, ast.Is)):
|
|
right = test.comparators[0]
|
|
if isinstance(right, (ast.Constant, ast.Tuple, ast.Set, ast.List)):
|
|
return ast.dump(test.left)
|
|
return None
|
|
|
|
|
|
def elif_ladder(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
elifs: set[int] = set()
|
|
for node in ast.walk(tree):
|
|
if not isinstance(node, ast.If) or id(node) in elifs:
|
|
continue
|
|
keys, current = [], node
|
|
while isinstance(current, ast.If):
|
|
keys.append(_ladder_key(current.test))
|
|
nxt = current.orelse
|
|
if len(nxt) == 1 and isinstance(nxt[0], ast.If):
|
|
elifs.add(id(nxt[0]))
|
|
current = nxt[0]
|
|
else:
|
|
break
|
|
if len(keys) >= 4 and keys[0] is not None and len(set(keys)) == 1:
|
|
yield node.lineno
|
|
|
|
|
|
_COPY_CONTEXT = "contextvars.copy_context"
|
|
|
|
|
|
def _context_names(own: list[ast.AST]) -> set[str]:
|
|
"""Names this scope binds only to ``contextvars.copy_context()``."""
|
|
copied: set[str] = set()
|
|
targets: set[int] = set()
|
|
for node in own:
|
|
if isinstance(node, (ast.Assign, ast.AnnAssign)) and isinstance(node.value, ast.Call):
|
|
if _call_name(node.value) == _COPY_CONTEXT:
|
|
names = [t for t in (node.targets if isinstance(node, ast.Assign) else [node.target])
|
|
if isinstance(t, ast.Name)]
|
|
copied.update(t.id for t in names)
|
|
targets.update(id(t) for t in names)
|
|
rebound = {n.id for n in own if isinstance(n, ast.Name) and not isinstance(n.ctx, ast.Load)
|
|
and id(n) not in targets}
|
|
return copied - rebound
|
|
|
|
|
|
def _runs_in_copied_context(call: ast.Call, contexts: set[str]) -> bool:
|
|
"""``Thread(target=<ctx>.run, ...)`` with ``ctx`` a ``copy_context()``: the thread runs
|
|
in the caller's context, which is what ``spawn_context_thread`` does."""
|
|
target = next((kw.value for kw in call.keywords if kw.arg == "target"),
|
|
call.args[1] if len(call.args) > 1 else None)
|
|
if not (isinstance(target, ast.Attribute) and target.attr == "run"):
|
|
return False
|
|
ctx = target.value
|
|
if isinstance(ctx, ast.Call):
|
|
return _call_name(ctx) == _COPY_CONTEXT
|
|
return isinstance(ctx, ast.Name) and ctx.id in contexts
|
|
|
|
|
|
def raw_thread(tree: ast.Module, ctx: Ctx) -> Iterable[int]:
|
|
for _, _, own in _scopes(tree):
|
|
contexts = _context_names(own)
|
|
for node in own:
|
|
if (isinstance(node, ast.Call) and _call_name(node) == "threading.Thread"
|
|
and not _runs_in_copied_context(node, contexts)):
|
|
yield node.lineno
|
|
|
|
|
|
CHECKERS: dict[str, Callable[[ast.Module, Ctx], Iterable[int]]] = {
|
|
"HX001": hardcoded_home,
|
|
"HX002": new_env_var,
|
|
"HX003": argv_identity,
|
|
"HX004": unscoped_secret_fallback,
|
|
"HX005": import_time_capture,
|
|
"HX006": missing_timeout,
|
|
"HX007": sync_config_in_async,
|
|
"HX008": get_event_loop,
|
|
"HX009": gather_exception_check,
|
|
"HX010": bool_of_env,
|
|
"HX011": elif_ladder,
|
|
"HX012": raw_thread,
|
|
}
|