112 lines
4.1 KiB
Python
112 lines
4.1 KiB
Python
"""Periodic process memory usage logging for the gateway.
|
|
|
|
Emits one grep-friendly ``[MEMORY] ...`` line every N seconds (default 300)
|
|
from a daemon thread so slow leaks show up as an RSS time series in the logs.
|
|
A baseline snapshot is logged on start and a final one on stop. Uses stdlib
|
|
``resource`` first, ``psutil`` as fallback (Windows); if neither works the
|
|
monitor warns once and stays disabled.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import gc
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
import time
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_BYTES_TO_MB = 1024 * 1024
|
|
|
|
_monitor_thread: Optional[threading.Thread] = None
|
|
_stop_event: Optional[threading.Event] = None
|
|
_start_time: Optional[float] = None
|
|
_lock = threading.Lock()
|
|
|
|
|
|
def _get_rss_mb() -> Optional[int]:
|
|
"""Return process RSS high-water mark in MB, or None if unavailable."""
|
|
try:
|
|
import resource
|
|
|
|
# ru_maxrss is KB on Linux but bytes on macOS.
|
|
return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024))
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import psutil # type: ignore
|
|
|
|
return int(psutil.Process(os.getpid()).memory_info().rss / _BYTES_TO_MB)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def log_memory_usage(prefix: str = "") -> None:
|
|
"""Log ``[MEMORY] [<prefix> ]rss=... gc=... threads=... uptime=...``; safe from any thread."""
|
|
rss = _get_rss_mb()
|
|
logger.info(
|
|
"[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", f"{prefix} " if prefix else "",
|
|
"unavailable" if rss is None else f"{rss}MB", gc.get_count(), threading.active_count(),
|
|
int(time.monotonic() - _start_time) if _start_time else 0,
|
|
)
|
|
|
|
|
|
def _monitor_loop(stop_event: threading.Event, interval: float) -> None:
|
|
"""Background thread body — log every ``interval`` seconds until stopped."""
|
|
while not stop_event.wait(interval):
|
|
try:
|
|
log_memory_usage()
|
|
except Exception as e: # never let the monitor crash the gateway
|
|
logger.debug("Memory monitor iteration failed: %s", e)
|
|
|
|
|
|
def start_memory_monitoring(interval_seconds: float = 300.0) -> bool:
|
|
"""Start periodic logging in a daemon thread (baseline logged immediately). False if
|
|
already running or RSS introspection is unavailable (warned once)."""
|
|
global _monitor_thread, _stop_event, _start_time
|
|
|
|
with _lock:
|
|
if _monitor_thread is not None and _monitor_thread.is_alive():
|
|
return False
|
|
if _get_rss_mb() is None:
|
|
logger.warning(
|
|
"[MEMORY] Memory monitoring unavailable: neither resource.getrusage nor psutil could read process RSS "
|
|
"— skipping periodic logging.",
|
|
)
|
|
return False
|
|
_start_time = time.monotonic()
|
|
_stop_event = threading.Event()
|
|
log_memory_usage(prefix="baseline")
|
|
_monitor_thread = threading.Thread(
|
|
target=_monitor_loop, args=(_stop_event, float(interval_seconds)), name="gateway-memory-monitor", daemon=True
|
|
)
|
|
_monitor_thread.start()
|
|
logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(interval_seconds))
|
|
return True
|
|
|
|
|
|
def stop_memory_monitoring(timeout: float = 2.0) -> None:
|
|
"""Stop the monitor thread and log a final ``shutdown`` snapshot. No-op if never started."""
|
|
global _monitor_thread, _stop_event
|
|
|
|
with _lock:
|
|
if _stop_event is None or _monitor_thread is None:
|
|
return
|
|
with contextlib.suppress(Exception):
|
|
log_memory_usage(prefix="shutdown")
|
|
_stop_event.set()
|
|
thread, _monitor_thread, _stop_event = _monitor_thread, None, None
|
|
with contextlib.suppress(Exception): # join outside the lock so a stuck log call can't deadlock the stop path
|
|
thread.join(timeout=timeout)
|
|
logger.info("[MEMORY] Periodic memory monitoring stopped")
|
|
|
|
|
|
def is_running() -> bool:
|
|
"""True if the background monitor thread is alive."""
|
|
with _lock:
|
|
return _monitor_thread is not None and _monitor_thread.is_alive()
|