fix(gateway): persist startup watchdog stacks before stderr

A detached or service-managed gateway can have a blocked stderr. The watchdog exit escort then hard-exits after ten seconds before the file-based traceback is reached, leaving only the metadata record. Write the durable dump first and pin the ordering with a regression test.
This commit is contained in:
Ayush Nangia
2026-09-15 00:27:40 +05:30
committed by Teknium
parent cef69276ae
commit bcab2dd6eb
2 changed files with 24 additions and 6 deletions

View File

@@ -304,16 +304,16 @@ class StartupWatchdogHandle:
"exit_code": self.exit_code,
}
)
try:
faulthandler.dump_traceback(all_threads=True)
except Exception:
logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
# Also dump into the log file: detached/windowless runs (pythonw, some
# service managers) may have no stderr, and forensics are the point.
# Write the durable copy first. A detached service can have a blocked
# stderr, and the exit escort bounds the whole forensic path to 10s.
_append_dump(
lambda fh: faulthandler.dump_traceback(file=fh, all_threads=True),
"Startup watchdog file-based faulthandler dump failed",
)
try:
faulthandler.dump_traceback(all_threads=True)
except Exception:
logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
# Ledger write on a helper thread (it imports application code; the
# wedged main thread may hold the import lock). Bounded join, then exit
# regardless — NS-608 classification is best-effort; the respawn is not.

View File

@@ -483,6 +483,24 @@ class TestFire:
# absent on detached runs).
assert "Thread" in content or "Current thread" in content
def test_file_dump_precedes_stderr_dump(self, monkeypatch):
"""A blocked stderr must not hide the durable stack dump."""
writes = []
def _dump(*, file=None, all_threads=True):
writes.append("file" if file is not None else "stderr")
monkeypatch.setattr(sw.faulthandler, "dump_traceback", _dump)
monkeypatch.setattr(
StartupWatchdogHandle, "_exit", staticmethod(lambda code: None)
)
StartupWatchdogHandle(
timeout_s=60, exit_code=SERVICE_RESTART_EXIT_CODE
)._fire()
assert writes[:2] == ["file", "stderr"]
def test_fire_marks_lifecycle_exit(self, exit_capture, monkeypatch):
marked = {}