From 2be53ecd7b8d7b04bbc8112ce9467c65ef75e863 Mon Sep 17 00:00:00 2001 From: ethernet Date: Thu, 24 Sep 2026 11:43:41 -0400 Subject: [PATCH] fix(encoding): read kernel pseudo-files as plain utf-8 utf-8-sig exists to tolerate BOMs that Windows tooling adds to files users edit. /proc and /sys files are generated by the Linux kernel, never BOM'd and absent on Windows, so -sig there only muddies the read/write policy. Switch every literal /proc/ and /sys/ read to utf-8 and teach the footgun read rule that string literals starting with /proc/ or /sys/ are exempt (user-edited files keep utf-8-sig). --- agent/proxy_sources/iron_proxy.py | 2 +- gateway/cgroup_cleanup.py | 4 ++-- gateway/drain_control.py | 4 ++-- gateway/shutdown_forensics.py | 4 ++-- gateway/status.py | 4 ++-- hermes_cli/kanban_db_dispatch.py | 2 +- hermes_cli/local_runtime/hardware.py | 2 +- hermes_cli/main_desktop.py | 2 +- hermes_cli/mem_trim.py | 2 +- hermes_cli/security_audit_startup.py | 4 ++-- hermes_cli/web_server_lifecycle.py | 2 +- scripts/check-windows-footguns.py | 9 ++++++++- tests/scripts/test_footgun_encoding_direction.py | 3 +++ tests/tools/test_bot_desktop_install.py | 2 +- tools/bot_desktop/browser.py | 2 +- tools/browser_tool_install.py | 2 +- tools/browser_tool_session.py | 2 +- tools/process_registry.py | 2 +- 18 files changed, 32 insertions(+), 22 deletions(-) diff --git a/agent/proxy_sources/iron_proxy.py b/agent/proxy_sources/iron_proxy.py index ab7db30542..b9ad1261b4 100644 --- a/agent/proxy_sources/iron_proxy.py +++ b/agent/proxy_sources/iron_proxy.py @@ -584,7 +584,7 @@ def _read_pid() -> Optional[int]: def _pid_proc_starttime(pid: int) -> Optional[str]: """/proc//stat starttime (field 22) on Linux, else None — cheap PID-recycling detector.""" try: - text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8-sig") + text = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") except OSError: return None # comm may contain spaces/parens, so split after the LAST ")"; field 22 -> tail index 19. diff --git a/gateway/cgroup_cleanup.py b/gateway/cgroup_cleanup.py index 55f1c9e6d3..8e34fc5a89 100644 --- a/gateway/cgroup_cleanup.py +++ b/gateway/cgroup_cleanup.py @@ -21,7 +21,7 @@ from pathlib import Path def _own_cgroup_path() -> str | None: """Return the cgroup v2 path for the calling process, or None.""" try: - text = Path("/proc/self/cgroup").read_text(encoding="utf-8-sig") + text = Path("/proc/self/cgroup").read_text(encoding="utf-8") except OSError: return None match = re.search(r"^0::(.+)$", text, re.MULTILINE) @@ -30,7 +30,7 @@ def _own_cgroup_path() -> str | None: def _read_cgroup_pids(cgroup_path: str) -> list[int]: try: - raw = Path(f"/sys/fs/cgroup{cgroup_path}/cgroup.procs").read_text(encoding="utf-8-sig") + raw = Path(f"/sys/fs/cgroup{cgroup_path}/cgroup.procs").read_text(encoding="utf-8") except OSError: return [] pids: list[int] = [] diff --git a/gateway/drain_control.py b/gateway/drain_control.py index 29719876aa..3618498d39 100644 --- a/gateway/drain_control.py +++ b/gateway/drain_control.py @@ -48,11 +48,11 @@ def current_instantiation_epoch() -> str: """ boot_id = pid1_start = "" with contextlib.suppress(OSError): - boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8-sig").strip() + boot_id = Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip() with contextlib.suppress(OSError, IndexError): # " () ...": comm may contain spaces/parens, so split on the # LAST ')'. starttime is field 22 (1-indexed) = tail index 19. - pid1_start = Path("/proc/1/stat").read_text(encoding="utf-8-sig").rsplit(")", 1)[1].split()[19] + pid1_start = Path("/proc/1/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split()[19] return f"{boot_id}:{pid1_start}" if (boot_id or pid1_start) else "" diff --git a/gateway/shutdown_forensics.py b/gateway/shutdown_forensics.py index 058e7a2c92..2812c30340 100644 --- a/gateway/shutdown_forensics.py +++ b/gateway/shutdown_forensics.py @@ -41,7 +41,7 @@ def _signal_name(sig: Any) -> str: def _read_proc_field(pid: int, key: str) -> Optional[str]: """Read a single field from /proc//status. Linux only; None elsewhere.""" - with contextlib.suppress(OSError), open(f"/proc/{pid}/status", encoding="utf-8-sig") as fh: + with contextlib.suppress(OSError), open(f"/proc/{pid}/status", encoding="utf-8") as fh: for line in fh: if line.startswith(key + ":"): return line.split(":", 1)[1].strip() @@ -210,7 +210,7 @@ def check_systemd_timing_alignment( return None # Not running under systemd (or at least not directly) # /proc/self/cgroup: "0::/user.slice/.../hermes-gateway.service" unit_name: Optional[str] = None - with contextlib.suppress(OSError), open("/proc/self/cgroup", encoding="utf-8-sig") as fh: + with contextlib.suppress(OSError), open("/proc/self/cgroup", encoding="utf-8") as fh: for line in fh: parts = reversed(line.strip().split("/")) unit_name = next((p for p in parts if p.endswith(".service")), None) diff --git a/gateway/status.py b/gateway/status.py index 5e0d360e83..9a1a3bce2b 100644 --- a/gateway/status.py +++ b/gateway/status.py @@ -896,7 +896,7 @@ def _pid_exists(pid: int) -> bool: def _posix_is_zombie(pid: int) -> bool: """Zombie via ``/proc//stat`` field 3, or ``ps -o state=`` without /proc (macOS/BSD).""" try: - stat_fields = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8-sig").split() + stat_fields = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8").split() return len(stat_fields) > 2 and stat_fields[2] == "Z" except FileNotFoundError: with contextlib.suppress(Exception): @@ -1515,7 +1515,7 @@ def _scoped_lock_record_is_stale(existing: dict[str, Any], existing_pid: Optiona def _process_is_stopped(pid: int) -> bool: """True for a stopped / tracing-stop state (T/t) in ``/proc//status``.""" with contextlib.suppress(OSError): - for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8-sig").splitlines(): + for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8").splitlines(): if line.startswith("State:"): return line.split()[1] in {"T", "t"} return False diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py index 7d459526d9..7b71be5569 100644 --- a/hermes_cli/kanban_db_dispatch.py +++ b/hermes_cli/kanban_db_dispatch.py @@ -328,7 +328,7 @@ def _pid_alive(pid: Optional[int]) -> bool: return False if sys.platform == "linux": try: - with open(f"/proc/{int(pid)}/status", "r", encoding="utf-8-sig") as f: + with open(f"/proc/{int(pid)}/status", "r", encoding="utf-8") as f: for line in f: if line.startswith("State:"): # "State:\tZ (zombie)" → dead diff --git a/hermes_cli/local_runtime/hardware.py b/hermes_cli/local_runtime/hardware.py index e3df442232..9665eaf558 100644 --- a/hermes_cli/local_runtime/hardware.py +++ b/hermes_cli/local_runtime/hardware.py @@ -67,7 +67,7 @@ _LINUX_RAM_KEYS = frozenset({"MemTotal", "MemAvailable", "MemFree"}) def _linux_meminfo_text() -> str | None: """Raw /proc/meminfo, or None when procfs cannot be read.""" try: - return Path("/proc/meminfo").read_text(encoding="utf-8-sig") + return Path("/proc/meminfo").read_text(encoding="utf-8") except OSError: return None diff --git a/hermes_cli/main_desktop.py b/hermes_cli/main_desktop.py index d9a2cb9a50..8d24ffae9f 100644 --- a/hermes_cli/main_desktop.py +++ b/hermes_cli/main_desktop.py @@ -979,7 +979,7 @@ def _desktop_linux_needs_no_sandbox() -> bool: if hasattr(os, "geteuid") and os.geteuid() == 0: return False try: - with open("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", encoding="utf-8-sig") as f: + with open("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", encoding="utf-8") as f: return f.read().strip() == "1" except OSError: return False diff --git a/hermes_cli/mem_trim.py b/hermes_cli/mem_trim.py index 4c863322e9..9ed5856193 100644 --- a/hermes_cli/mem_trim.py +++ b/hermes_cli/mem_trim.py @@ -77,7 +77,7 @@ def _read_proc_status() -> str | None: if sys.platform != "linux": return None try: - return Path("/proc/self/status").read_text(encoding="utf-8-sig") + return Path("/proc/self/status").read_text(encoding="utf-8") except OSError: return None diff --git a/hermes_cli/security_audit_startup.py b/hermes_cli/security_audit_startup.py index 8935ef4a00..19b73b0958 100644 --- a/hermes_cli/security_audit_startup.py +++ b/hermes_cli/security_audit_startup.py @@ -74,7 +74,7 @@ def _in_container() -> bool: if os.environ.get("HERMES_DESKTOP_CHILD_PID"): return False # desktop child, not a server container try: - cgroup = Path("/proc/1/cgroup").read_text(encoding="utf-8-sig", errors="replace") + cgroup = Path("/proc/1/cgroup").read_text(encoding="utf-8", errors="replace") except Exception: return False return any(tok in cgroup for tok in ("docker", "containerd", "kubepods", "libpod")) @@ -89,7 +89,7 @@ def _path_is_mounted(path: Path) -> bool: except Exception: target = path try: - mounts = Path("/proc/mounts").read_text(encoding="utf-8-sig", errors="replace").splitlines() + mounts = Path("/proc/mounts").read_text(encoding="utf-8", errors="replace").splitlines() except Exception: return True # can't tell — fail safe (no warning) # (mountpoint, fstype) entries at or above target; the longest wins, first one on ties. diff --git a/hermes_cli/web_server_lifecycle.py b/hermes_cli/web_server_lifecycle.py index 48a712d6ed..89f59725e8 100644 --- a/hermes_cli/web_server_lifecycle.py +++ b/hermes_cli/web_server_lifecycle.py @@ -28,7 +28,7 @@ def _process_start_marker(pid: int) -> str: """ if sys.platform == "linux": try: - stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8-sig") + stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8") except FileNotFoundError as exc: raise ProcessLookupError(pid) from exc diff --git a/scripts/check-windows-footguns.py b/scripts/check-windows-footguns.py index adf12d4d56..5e08c4e069 100644 --- a/scripts/check-windows-footguns.py +++ b/scripts/check-windows-footguns.py @@ -464,7 +464,10 @@ FOOTGUNS: list[Footgun] = [ "path.read_text(encoding='utf-8-sig') / " "open(path, 'r', encoding='utf-8-sig')" ), - post_filter=lambda m, line: _is_read_shaped(line), + # Literal /proc/ and /sys/ paths are kernel pseudo-files: Linux + # generates them, no Windows tool can BOM them, and they do not + # exist on Windows at all, so plain utf-8 is the honest encoding. + post_filter=lambda m, line: _is_read_shaped(line) and not _KERNEL_PSEUDO_FILE.search(line), ), Footgun( name="write with encoding='utf-8-sig' (emits a BOM)", @@ -696,6 +699,10 @@ def _extract_mode(line: str) -> str | None: return None +# A string literal (plain or f-string) that starts with /proc/ or /sys/. +_KERNEL_PSEUDO_FILE = re.compile(r"""['"]/(?:proc|sys)/""") + + def _is_read_shaped(line: str) -> bool: """Heuristic: does this line READ a file (so utf-8-sig applies)? diff --git a/tests/scripts/test_footgun_encoding_direction.py b/tests/scripts/test_footgun_encoding_direction.py index 4d6c887130..821c2f998a 100644 --- a/tests/scripts/test_footgun_encoding_direction.py +++ b/tests/scripts/test_footgun_encoding_direction.py @@ -28,6 +28,9 @@ def linter(): ('data = path.read_text(encoding="utf_8")', [(1, READ)]), ("with open(path, mode='r', encoding='utf-8') as f:", [(1, READ)]), ('path.read_text(encoding="utf-8-sig")', []), + ('Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")', []), + ("with open('/sys/fs/cgroup/cgroup.procs', encoding='utf-8') as f:", []), + ('Path(proc_root, "stat").read_text(encoding="utf-8") # /proc/ in prose only', [(1, READ)]), ('path.write_text(data, encoding="utf-8")', []), ("open(path, 'w', encoding='utf-8')", []), ("open(path, 'a', encoding='utf-8')", []), diff --git a/tests/tools/test_bot_desktop_install.py b/tests/tools/test_bot_desktop_install.py index c370b51284..cdb66ac0ae 100644 --- a/tests/tools/test_bot_desktop_install.py +++ b/tests/tools/test_bot_desktop_install.py @@ -89,7 +89,7 @@ def test_timeout_kills_the_package_managers_whole_process_group(monkeypatch): def gone() -> bool: # /proc-based: a reparented orphan sits outside our subtree, where os.kill(pid, 0) is guarded try: - return "Z" in (Path(f"/proc/{child}/stat").read_text(encoding="utf-8-sig").rsplit(")", 1)[1].split() or ["Z"])[0] + return "Z" in (Path(f"/proc/{child}/stat").read_text(encoding="utf-8").rsplit(")", 1)[1].split() or ["Z"])[0] except OSError: return True diff --git a/tools/bot_desktop/browser.py b/tools/bot_desktop/browser.py index 9f41aa9b54..d95c8007d1 100644 --- a/tools/bot_desktop/browser.py +++ b/tools/bot_desktop/browser.py @@ -148,7 +148,7 @@ def _launched_by_session(chromium_pid: int) -> Optional[str]: for a human-started (dock) instance. Chromium itself gets a scrubbed environment, so the daemon's ``/proc//environ`` is the marker (Linux-only, same user).""" try: - with open(f"/proc/{chromium_pid}/status", encoding="utf-8-sig") as fh: + with open(f"/proc/{chromium_pid}/status", encoding="utf-8") as fh: ppid = next((int(line.split()[1]) for line in fh if line.startswith("PPid:")), 0) with open(f"/proc/{ppid}/environ", "rb") as fh: raw = fh.read() diff --git a/tools/browser_tool_install.py b/tools/browser_tool_install.py index dc1a4d20c8..b740b81132 100644 --- a/tools/browser_tool_install.py +++ b/tools/browser_tool_install.py @@ -133,7 +133,7 @@ def _running_in_docker() -> bool: if os.path.exists("/.dockerenv"): return True try: - with open("/proc/1/cgroup", "rt", encoding="utf-8-sig") as fp: + with open("/proc/1/cgroup", "rt", encoding="utf-8") as fp: return "docker" in fp.read() except OSError: return False diff --git a/tools/browser_tool_session.py b/tools/browser_tool_session.py index 16bb40c377..85ba45bfb3 100644 --- a/tools/browser_tool_session.py +++ b/tools/browser_tool_session.py @@ -41,7 +41,7 @@ def apparmor_restricts_unprivileged_userns() -> bool: """Ubuntu 23.10+ default: unprivileged user namespaces are denied, so a Chromium whose ``chrome_sandbox`` helper is not setuid (Playwright's bundle) dies with 'No usable sandbox'.""" try: - with open("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", encoding="utf-8-sig") as f: + with open("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", encoding="utf-8") as f: return f.read().strip() == "1" except OSError: return False diff --git a/tools/process_registry.py b/tools/process_registry.py index ed2781a194..f02463baf6 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -135,7 +135,7 @@ def _worker_memory_max_bytes() -> int: override, _MIN_WORKER_MEMORY_MAX_BYTES // (1024 * 1024)) candidates: List[int] = [] try: - for line in Path("/proc/self/cgroup").read_text(encoding="utf-8-sig").splitlines(): + for line in Path("/proc/self/cgroup").read_text(encoding="utf-8").splitlines(): if line.startswith("0::"): relative = line.partition("::")[2].lstrip("/") raw_limit = (