From bcab2dd6eb26f7234ee771a688af3d486c729705 Mon Sep 17 00:00:00 2001 From: Ayush Nangia Date: Tue, 15 Sep 2026 00:27:40 +0530 Subject: [PATCH] fix(gateway): persist startup watchdog stacks before stderr A detached or service-managed gateway can have a blocked stderr. The watchdog exit escort then hard-exits after ten seconds before the file-based traceback is reached, leaving only the metadata record. Write the durable dump first and pin the ordering with a regression test. --- hermes_startup_watchdog.py | 12 ++++++------ tests/gateway/test_startup_watchdog.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/hermes_startup_watchdog.py b/hermes_startup_watchdog.py index 3bfb34ebd5..88eae506af 100644 --- a/hermes_startup_watchdog.py +++ b/hermes_startup_watchdog.py @@ -304,16 +304,16 @@ class StartupWatchdogHandle: "exit_code": self.exit_code, } ) - try: - faulthandler.dump_traceback(all_threads=True) - except Exception: - logger.debug("Startup watchdog faulthandler dump failed", exc_info=True) - # Also dump into the log file: detached/windowless runs (pythonw, some - # service managers) may have no stderr, and forensics are the point. + # Write the durable copy first. A detached service can have a blocked + # stderr, and the exit escort bounds the whole forensic path to 10s. _append_dump( lambda fh: faulthandler.dump_traceback(file=fh, all_threads=True), "Startup watchdog file-based faulthandler dump failed", ) + try: + faulthandler.dump_traceback(all_threads=True) + except Exception: + logger.debug("Startup watchdog faulthandler dump failed", exc_info=True) # Ledger write on a helper thread (it imports application code; the # wedged main thread may hold the import lock). Bounded join, then exit # regardless — NS-608 classification is best-effort; the respawn is not. diff --git a/tests/gateway/test_startup_watchdog.py b/tests/gateway/test_startup_watchdog.py index 5aec6ccd0b..cd6e1e5bbe 100644 --- a/tests/gateway/test_startup_watchdog.py +++ b/tests/gateway/test_startup_watchdog.py @@ -483,6 +483,24 @@ class TestFire: # absent on detached runs). assert "Thread" in content or "Current thread" in content + def test_file_dump_precedes_stderr_dump(self, monkeypatch): + """A blocked stderr must not hide the durable stack dump.""" + writes = [] + + def _dump(*, file=None, all_threads=True): + writes.append("file" if file is not None else "stderr") + + monkeypatch.setattr(sw.faulthandler, "dump_traceback", _dump) + monkeypatch.setattr( + StartupWatchdogHandle, "_exit", staticmethod(lambda code: None) + ) + + StartupWatchdogHandle( + timeout_s=60, exit_code=SERVICE_RESTART_EXIT_CODE + )._fire() + + assert writes[:2] == ["file", "stderr"] + def test_fire_marks_lifecycle_exit(self, exit_capture, monkeypatch): marked = {}