From 02fc8f7de791105fe47162d59ceb2e8ce8847f9e Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:56:08 -0700 Subject: [PATCH] fix(tests): failure-writer ownership probe no longer races a real tirith download The probe scrubs the environment down to PATH/LANG/LC_ALL/TZ, which also dropped the harness's HERMES_DISABLE_LAZY_INSTALLS. In a fresh home with no tirith on PATH (CI), GatewayRunner() then starts tirith's startup install on a background thread: the PM runtime downloads and unpacks Python and uv from GitHub while the cases run. When that unpacking overlaps the archive case, its buffers land in the process-wide tracemalloc window and push the peak past the 8 MiB bound, so the probe reports 19/20. The loopback fence also missed it: hermes_bootstrap's happy-eyeballs create_connection dials with connect_ex, so the downloads reached GitHub with blocked_external_attempts empty. - set HERMES_DISABLE_LAZY_INSTALLS=1 in the probe environment - fence socket.connect_ex as well as connect - assert the receipt records no external network attempts --- evals/gateway_failure_ownership/README.md | 6 ++-- evals/gateway_failure_ownership/probe.py | 29 ++++++++++++++----- .../gateway/test_failure_writer_ownership.py | 3 ++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/evals/gateway_failure_ownership/README.md b/evals/gateway_failure_ownership/README.md index c08761fbc4..d0b81c108f 100644 --- a/evals/gateway_failure_ownership/README.md +++ b/evals/gateway_failure_ownership/README.md @@ -11,8 +11,10 @@ scripts/run_tests.sh -j 1 tests/gateway/test_failure_writer_ownership.py The first argument selects the production checkout, so the same fixture can A/B another checkout without editing it. The receipt records its temporary HERMES_HOME. -Inherited credentials are cleared, tools are disabled, and socket connections are -restricted to loopback. No live messaging account or paid provider is contacted. +Inherited credentials are cleared, tools are disabled, lazy installs are off (so +GatewayRunner does not start tirith's background download), and socket connections +(`connect` and `connect_ex`) are restricted to loopback. No live messaging account or +paid provider is contacted. ## Fault boundaries and controls diff --git a/evals/gateway_failure_ownership/probe.py b/evals/gateway_failure_ownership/probe.py index 1589c260fd..fc187279aa 100644 --- a/evals/gateway_failure_ownership/probe.py +++ b/evals/gateway_failure_ownership/probe.py @@ -1,4 +1,4 @@ -import os, sys, json, asyncio, threading, tempfile, sqlite3, socket, subprocess, tracemalloc +import os, sys, json, asyncio, threading, tempfile, sqlite3, socket, subprocess, tracemalloc, errno from pathlib import Path from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler from types import SimpleNamespace @@ -14,6 +14,11 @@ os.environ.update( HOME=str(HOME), HERMES_HOME=str(HOME), HERMES_DISABLE_PLUGINS="1", + # The scrub above drops the test harness's hermetic switch. Without it, GatewayRunner() + # in this fresh home starts tirith's startup install on a background thread (PM runtime, + # Python and uv downloaded and unpacked while the cases run) and its buffers land in the + # process-wide tracemalloc window the archive case measures. + HERMES_DISABLE_LAZY_INSTALLS="1", NO_PROXY="127.0.0.1,localhost", ) sys.path.insert(0, str(ROOT)) @@ -21,23 +26,33 @@ os.chdir(HOME) (HOME / "config.yaml").write_text( "model:\n provider: openai-compat\n default: fixture-model\n context_length: 131072\nagent:\n max_iterations: 2\ncompression:\n enabled: false\ndatabase:\n journal_mode: delete\n" ) -# Fence all network calls to loopback, including optional discovery/aux paths. +# Fence all network calls to loopback, including optional discovery/aux paths. Both entry +# points: hermes_bootstrap's happy-eyeballs socket.create_connection dials with connect_ex. orig_connect = socket.socket.connect +orig_connect_ex = socket.socket.connect_ex blocked = [] +def _external(address): + return isinstance(address, tuple) and address[0] not in ("127.0.0.1", "::1", "localhost") + + def local_connect(self, address): - if isinstance(address, tuple) and address[0] not in ( - "127.0.0.1", - "::1", - "localhost", - ): + if _external(address): blocked.append(str(address)) raise OSError("fixture forbids external network") return orig_connect(self, address) +def local_connect_ex(self, address): + if _external(address): + blocked.append(str(address)) + return errno.ENETUNREACH + return orig_connect_ex(self, address) + + socket.socket.connect = local_connect +socket.socket.connect_ex = local_connect_ex requests = [] diff --git a/tests/gateway/test_failure_writer_ownership.py b/tests/gateway/test_failure_writer_ownership.py index 44811f36f0..31696d9b22 100644 --- a/tests/gateway/test_failure_writer_ownership.py +++ b/tests/gateway/test_failure_writer_ownership.py @@ -26,6 +26,9 @@ def test_gateway_failure_writer_preserves_accepted_turn_identity(tmp_path): assert receipt.exists(), result.stdout + result.stderr data = json.loads(receipt.read_text()) assert all(row["reached"] for row in data["observations"]), data + # A startup lazy install (tirith → PM runtime) would download over the real network on a + # background thread and pollute the archive case's process-wide tracemalloc window. + assert data["blocked_external_attempts"] == [], data["blocked_external_attempts"] assert data["passed"] == data["total"], data["observations"] assert result.returncode == 0, result.stdout + result.stderr