fix(gateway): persist startup watchdog stacks before stderr
A detached or service-managed gateway can have a blocked stderr. The watchdog exit escort then hard-exits after ten seconds before the file-based traceback is reached, leaving only the metadata record. Write the durable dump first and pin the ordering with a regression test.
This commit is contained in:
@@ -304,16 +304,16 @@ class StartupWatchdogHandle:
|
||||
"exit_code": self.exit_code,
|
||||
}
|
||||
)
|
||||
try:
|
||||
faulthandler.dump_traceback(all_threads=True)
|
||||
except Exception:
|
||||
logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
|
||||
# Also dump into the log file: detached/windowless runs (pythonw, some
|
||||
# service managers) may have no stderr, and forensics are the point.
|
||||
# Write the durable copy first. A detached service can have a blocked
|
||||
# stderr, and the exit escort bounds the whole forensic path to 10s.
|
||||
_append_dump(
|
||||
lambda fh: faulthandler.dump_traceback(file=fh, all_threads=True),
|
||||
"Startup watchdog file-based faulthandler dump failed",
|
||||
)
|
||||
try:
|
||||
faulthandler.dump_traceback(all_threads=True)
|
||||
except Exception:
|
||||
logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
|
||||
# Ledger write on a helper thread (it imports application code; the
|
||||
# wedged main thread may hold the import lock). Bounded join, then exit
|
||||
# regardless — NS-608 classification is best-effort; the respawn is not.
|
||||
|
||||
@@ -483,6 +483,24 @@ class TestFire:
|
||||
# absent on detached runs).
|
||||
assert "Thread" in content or "Current thread" in content
|
||||
|
||||
def test_file_dump_precedes_stderr_dump(self, monkeypatch):
|
||||
"""A blocked stderr must not hide the durable stack dump."""
|
||||
writes = []
|
||||
|
||||
def _dump(*, file=None, all_threads=True):
|
||||
writes.append("file" if file is not None else "stderr")
|
||||
|
||||
monkeypatch.setattr(sw.faulthandler, "dump_traceback", _dump)
|
||||
monkeypatch.setattr(
|
||||
StartupWatchdogHandle, "_exit", staticmethod(lambda code: None)
|
||||
)
|
||||
|
||||
StartupWatchdogHandle(
|
||||
timeout_s=60, exit_code=SERVICE_RESTART_EXIT_CODE
|
||||
)._fire()
|
||||
|
||||
assert writes[:2] == ["file", "stderr"]
|
||||
|
||||
def test_fire_marks_lifecycle_exit(self, exit_capture, monkeypatch):
|
||||
marked = {}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user