diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da8b950cce..da242d55ab 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -197,6 +197,9 @@ jobs: # time, but resolves from the warm cache instead of re-downloading # and re-building wheels. enable-cache: true + # Own cache entry: the unit job's older uv would otherwise share and + # overwrite this key with a cache this uv version did not write. + cache-suffix: e2e cache-dependency-glob: | pyproject.toml uv.lock @@ -241,6 +244,10 @@ jobs: # Multi-process episodes (torture chamber, compaction kill -9, # gateway liveness) legitimately run past the 300 s default. HERMES_TEST_FILE_TIMEOUT: "900" + # No automatic re-run of a failed file: the torture chamber and the + # exactly-once/compaction suites are race detectors, and a rare + # corruption that passes on retry is still a corruption. + HERMES_TEST_FILE_RETRIES: "0" OPENROUTER_API_KEY: "" OPENAI_API_KEY: "" NOUS_API_KEY: "" diff --git a/tests/e2e/core/chaos/_gateway_harness.py b/tests/e2e/core/chaos/_gateway_harness.py index a67197b8be..c2bf6525ae 100644 --- a/tests/e2e/core/chaos/_gateway_harness.py +++ b/tests/e2e/core/chaos/_gateway_harness.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Optional from tests.e2e.core.chaos import _gateway_fake_platform as fake_platform -from tests.e2e.core.chaos._helpers import hermetic_env, python_exe, write_chaos_home +from tests.e2e.core.chaos._helpers import hermetic_env, kill_tagged, python_exe, write_chaos_home BOOT_DEADLINE_S = 180.0 SHUTDOWN_DEADLINE_S = 60.0 @@ -112,9 +112,11 @@ class GatewayProc: "HERMES_STATE_DB_GUARD_BYPASS": "1", }) log = open(self.log_path, "wb") + # Same process group as pytest (no start_new_session): when the runner kills a timed-out + # file's group, the gateway goes with it instead of outliving the run. self.proc = subprocess.Popen( [python_exe(), "-m", "gateway.run"], cwd=str(self.home), env=env, - stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT) log.close() deadline = time.monotonic() + BOOT_DEADLINE_S while True: @@ -169,8 +171,9 @@ class GatewayProc: self.shutdown_s = time.monotonic() - t0 except subprocess.TimeoutExpired: self.shutdown_s = None + kill_tagged(self.tag) with _suppress_oserror(): - os.killpg(self.proc.pid, signal.SIGKILL) + self.proc.kill() self.exit_code = self.proc.wait(timeout=30) finally: for sock in (self._conn, self._listener): diff --git a/tests/e2e/core/chaos/_helpers.py b/tests/e2e/core/chaos/_helpers.py index 9ab7f37980..9b10a91b0c 100644 --- a/tests/e2e/core/chaos/_helpers.py +++ b/tests/e2e/core/chaos/_helpers.py @@ -90,6 +90,10 @@ def chaos_config( "memory:\n" " memory_enabled: false\n" " user_profile_enabled: false\n" + # Offline: the passive update check does a GitHub round-trip and, on a partial clone + # whose objects lag upstream, spawns a git lazy fetch that outlives the gateway. + "updates:\n" + " check: false\n" + extra ) diff --git a/tests/e2e/core/chaos/test_agent_turn_liveness.py b/tests/e2e/core/chaos/test_agent_turn_liveness.py index ad03410008..ef722e64c3 100644 --- a/tests/e2e/core/chaos/test_agent_turn_liveness.py +++ b/tests/e2e/core/chaos/test_agent_turn_liveness.py @@ -393,11 +393,15 @@ class Run: log = hermes_home / "logs" / "agent.log" if rep["orphans"] and log.exists(): rep["agent_log_tail"] = log.read_text(errors="replace")[-6000:] + # Stale-kill timeline: tells a PROBE refused by the cross-turn breaker apart from a slow probe. + rep["stale_log"] = [ln[:110] for ln in (log.read_text(errors="replace").splitlines() if log.exists() else []) + if "stale" in ln.lower() and ("WARNING" in ln or "ERROR" in ln)][-12:] db = hermes_home / "state.db" rep["persisted"] = persisted_messages(db, self.session_id) if db.exists() else None rep["integrity"] = integrity_ok(db) if db.exists() else "missing" probes = self.probe_requests() rep["probe_request"] = probes[-1] if probes else None + rep["probe_count"] = len(probes) mains = [r["body"] for r in list(self.srv.requests) if r["kind"] == "main"] rep["last_request"] = mains[-1] if mains else None rep["max_request_bytes"] = max((len(json.dumps(b)) for b in mains), default=0) @@ -471,7 +475,9 @@ def test_agent_turn_liveness(scenario_id: str, runs: dict[str, Future]) -> None: assert rep["probe_request"] is None, "a tripped stale breaker still billed the provider" sent = rep["last_request"]["messages"] else: - assert f"alive {rep_nonce(rep)}" in rep["turn1"]["final"], f"PROBE not answered: {rep['turn1']}" + assert f"alive {rep_nonce(rep)}" in rep["turn1"]["final"], ( + f"PROBE not answered: {rep['turn1']}\nfault calls {rep['fault_calls']}, probe requests " + f"{rep['probe_count']}, stale log:\n" + "\n".join(rep.get("stale_log", []))) assert rep["probe_request"] is not None, "PROBE never reached the provider" sent = rep["probe_request"]["messages"] assert any(f"[[chaos:{scenario_id}]]" in _text(m.get("content")) for m in sent if m.get("role") == "user"), \ diff --git a/tests/e2e/core/chaos/test_tui_gateway_turn_liveness.py b/tests/e2e/core/chaos/test_tui_gateway_turn_liveness.py index 9a36f6895b..5a1850d76f 100644 --- a/tests/e2e/core/chaos/test_tui_gateway_turn_liveness.py +++ b/tests/e2e/core/chaos/test_tui_gateway_turn_liveness.py @@ -288,9 +288,17 @@ def _assert_heartbeat(scn: Scenario, hb: Heartbeat, stats: dict[str, Any], turn_ f"{scn.id}: only {len(hb.latencies)} heartbeats answered in a {turn_s:.1f}s turn") +class ToolOutlivedGateway(Exception): + """The in-flight tool outlived the gateway's exit: its process tree survives and/or its + tool_call was left with no result in state.db. Deliberately NOT an AssertionError: the + known-bug xfail matches only this, so an RPC failure, a crash or any other broken invariant + (heartbeat, exit deadline, integrity) still fails the test.""" + + def _exit_mid_turn(scn: Scenario, gw: TuiGatewayProcess, hb: Heartbeat, tag: str, state_db: Path, stored: str, stats: dict[str, Any]) -> dict[str, Any]: - """The client vanishes (stdin EOF) or the supervisor stops us (SIGTERM) mid-wedge.""" + """The client vanishes (stdin EOF) or the supervisor stops us (SIGTERM) mid-wedge. + Every other invariant is asserted first; the tool leftovers are checked together, last.""" _assert_heartbeat(scn, hb, stats, turn_s=LONG_TIMEOUT_S) t0 = time.monotonic() if scn.action == "sigterm": @@ -303,11 +311,14 @@ def _exit_mid_turn(scn: Scenario, gw: TuiGatewayProcess, hb: Heartbeat, tag: str rc = gw.close_stdin_and_wait(EXIT_TIMEOUT_S) assert rc is not None, f"{scn.id}: gateway still alive {EXIT_TIMEOUT_S}s after {scn.action} mid-turn{gw.tail()}" stats["exit_s"] = round(time.monotonic() - t0, 2) - survivors = wait_no_tagged(tag) - assert survivors == [], f"{scn.id}: orphans after {scn.action} mid-turn: {describe_pids(survivors)}" assert integrity_ok(state_db) == "ok" - assert unanswered_tool_calls(persisted_messages(state_db, stored)) == [], ( - f"{scn.id}: state.db keeps a tool_call with no result after {scn.action} (next resume sends it)") + leftovers = [] + if dangling := unanswered_tool_calls(persisted_messages(state_db, stored)): + leftovers.append(f"state.db keeps tool_call(s) {dangling} with no result (next resume sends them)") + if survivors := wait_no_tagged(tag): + leftovers.append(f"orphans: {describe_pids(survivors)}") + if leftovers: + raise ToolOutlivedGateway(f"{scn.id} after {scn.action} mid-turn: " + "; ".join(leftovers)) return stats @@ -448,10 +459,13 @@ def scenario_futures(request: pytest.FixtureRequest, tmp_path_factory: pytest.Te # Real production bug on base (reported, not fixed here): when the gateway leaves mid-tool — # client closes stdin or supervisor SIGTERMs — _shutdown_sessions() closes the agents but the # in-flight foreground terminal command (its own process group) is never killed, so the -# `bash -c ...` + `sleep 3600` tree survives, reparented to init. strict: flips red once fixed. +# `bash -c ...` + `sleep 3600` tree survives, reparented to init, and its tool_call is left with +# no result in state.db. strict: flips red once fixed. raises= names only the leftovers check's +# exception, so everything before it is asserted normally. _ORPHANED_FOREGROUND_TOOL = pytest.mark.xfail( - strict=True, raises=AssertionError, - reason="tui_gateway exit (EOF/SIGTERM) orphans the running foreground terminal tool's process tree") + strict=True, raises=ToolOutlivedGateway, + reason="tui_gateway exit (EOF/SIGTERM) orphans the running foreground terminal tool's process tree " + "and leaves its tool_call without a result") KNOWN_BUGS = {"stdin_eof_during_hung_tool": _ORPHANED_FOREGROUND_TOOL, "sigterm_during_hung_tool": _ORPHANED_FOREGROUND_TOOL} diff --git a/tests/e2e/core/compaction/_helpers.py b/tests/e2e/core/compaction/_helpers.py index e6b3326eb3..59f2d84fc5 100644 --- a/tests/e2e/core/compaction/_helpers.py +++ b/tests/e2e/core/compaction/_helpers.py @@ -142,6 +142,8 @@ def write_home(hermes_home: Path, base_url: str, extra_config: str) -> None: f" context_length: {CONTEXT_LENGTH}\n" "agent:\n" " api_max_retries: 1\n" + "updates:\n" + " check: false\n" # offline: no GitHub round-trip or git lazy fetch from a test surface + extra_config, encoding="utf-8", ) diff --git a/tests/e2e/core/history/_helpers.py b/tests/e2e/core/history/_helpers.py index 997963f38d..c33bd840dc 100644 --- a/tests/e2e/core/history/_helpers.py +++ b/tests/e2e/core/history/_helpers.py @@ -199,15 +199,16 @@ def _list_diff(a: list[tuple], b: list[tuple], an: str, bn: str) -> str: return "\n".join(lines) -def prefix_breaks(requests: list[dict[str, Any]]) -> list[tuple[int, str]]: +def prefix_breaks(requests: list[dict[str, Any]], *, tools: bool = True) -> list[tuple[int, str]]: """(C17) Indices i where request i is NOT a byte-identical extension of request i-1. - Byte-stability covers the tools array, the system prompt and every earlier message. + Byte-stability covers the tools array (unless ``tools=False``), the system prompt and every + earlier message. """ breaks = [] for i in range(1, len(requests)): prev, cur = requests[i - 1], requests[i] - if canon(prev.get("tools")) != canon(cur.get("tools")): + if tools and canon(prev.get("tools")) != canon(cur.get("tools")): breaks.append((i, "tools array changed: " + tools_diff(prev.get("tools"), cur.get("tools")))) continue pm, cm = prev["messages"], cur["messages"] @@ -222,6 +223,13 @@ def prefix_breaks(requests: list[dict[str, Any]]) -> list[tuple[int, str]]: return breaks +def tools_breaks(requests: list[dict[str, Any]]) -> list[tuple[int, str]]: + """Indices i where request i sends a different tools array than request i-1.""" + return [(i, tools_diff(requests[i - 1].get("tools"), requests[i].get("tools"))) + for i in range(1, len(requests)) + if canon(requests[i - 1].get("tools")) != canon(requests[i].get("tools"))] + + def tools_diff(a: Any, b: Any) -> str: an = {t["function"]["name"]: canon(t) for t in a or ()} bn = {t["function"]["name"]: canon(t) for t in b or ()} @@ -502,19 +510,32 @@ class TuiGateway: class Script: """Main-turn responder: pops scripted actions; an action may be a callable run AT request - arrival (so /steer and interrupt land while the request is genuinely in flight).""" + arrival (so /steer and interrupt land while the request is genuinely in flight). + + Callables run on the fake provider's HTTP handler thread, where a raised assertion only drops + the connection (the agent retries and the test moves on). ``errors`` keeps every such failure + so the test re-raises it on its own thread (``raise_errors``).""" def __init__(self) -> None: self.actions: list[Any] = [] self.n = 0 self.session: Any = None self.steered: list[str] = [] + self.errors: list[str] = [] def __call__(self, record: dict[str, Any]) -> Any: self.n += 1 if self.actions: act = self.actions.pop(0) - return act(record) if callable(act) else act + if not callable(act): + return act + try: + return act(record) + except BaseException: + import traceback + + self.errors.append(traceback.format_exc()) + raise # Unique text + varying usage per answer, so a duplicated row or a double-counted # request is always distinguishable. from tests.fakes.fake_llm_provider import Text @@ -522,6 +543,10 @@ class Script: return Text(f"answer #{self.n}", prompt_tokens=900 + 13 * self.n, completion_tokens=5 + self.n, cached_tokens=400 + self.n) + def raise_errors(self, where: str) -> None: + assert not self.errors, f"{where}: a scripted action failed on the provider thread:\n" + "\n".join( + self.errors) + def big(label: str, n: int = 12000) -> str: """A user message large enough that summarizing a few of them genuinely shrinks the context.""" diff --git a/tests/e2e/core/history/test_prefix_stability.py b/tests/e2e/core/history/test_prefix_stability.py index 9e3476ffa3..9f7b3c1776 100644 --- a/tests/e2e/core/history/test_prefix_stability.py +++ b/tests/e2e/core/history/test_prefix_stability.py @@ -38,6 +38,7 @@ from tests.e2e.core.history._helpers import ( prefix_breaks, row_counts, run_oneshot, + tools_breaks, views, ) from tests.fakes.fake_llm_provider import FakeLLMServer, Text, ToolCall, write_hermes_home @@ -91,6 +92,11 @@ KNOWN_BROKEN = { } +class ToolsArrayDrift(Exception): + """The tools array changed between requests of one session. Not an AssertionError: the + known-bug xfail matches only this, so every other invariant still fails the test.""" + + @pytest.fixture def world(tmp_path): home = tmp_path / "home" @@ -172,7 +178,7 @@ def run_journey(world: dict, hops: list[Hop]) -> tuple[str, list[tuple[int, str] @pytest.mark.parametrize("journey", [ - pytest.param(name, marks=pytest.mark.xfail(strict=True, reason=KNOWN_BROKEN[name])) + pytest.param(name, marks=pytest.mark.xfail(strict=True, raises=ToolsArrayDrift, reason=KNOWN_BROKEN[name])) if name in KNOWN_BROKEN else name for name in JOURNEYS ]) @@ -185,7 +191,9 @@ def test_request_prefix_is_byte_stable_across_processes(world, journey): def where(i: int) -> str: return f"request {i} (in {max((o for o in openings if o[0] <= i), default=(0, '?'))[1]})" - breaks = prefix_breaks(main) + # System prompt + messages first; the tools array is checked last, on its own, so a known + # tools-drift xfail cannot mask a message-prefix, usage or integrity regression. + breaks = prefix_breaks(main, tools=False) unexpected = [(i, why) for i, why in breaks if i != compaction_idx] assert not unexpected, "prompt-cache prefix broke outside the compaction boundary:\n" + "\n".join( f" {where(i)}: {why}" for i, why in unexpected) @@ -195,3 +203,8 @@ def test_request_prefix_is_byte_stable_across_processes(world, journey): assert_usage_matches(home, lineage(home, sid), srv.requests, f"after journey {journey}") integrity_ok(home) + + drift = [(i, why) for i, why in tools_breaks(main) if i != compaction_idx] + if drift: + raise ToolsArrayDrift("tools array changed within one session:\n" + "\n".join( + f" {where(i)}: {why}" for i, why in drift)) diff --git a/tests/e2e/core/history/test_transcript_ledger.py b/tests/e2e/core/history/test_transcript_ledger.py index c4d3797aa2..ca3ada2d5a 100644 --- a/tests/e2e/core/history/test_transcript_ledger.py +++ b/tests/e2e/core/history/test_transcript_ledger.py @@ -20,6 +20,7 @@ prefix at the declared compaction boundaries (C17). from __future__ import annotations import os +import time from pathlib import Path from typing import Any, Callable @@ -61,6 +62,11 @@ from tests.fakes.fake_llm_provider import ( Step = tuple[str, Any] +STEER_TEXT = "also mention the steer marker" +INTERRUPTED_TURN = "this request gets interrupted" +# The interrupted request hangs HANG_S and then drops; a working interrupt ends the turn long before. +INTERRUPT_HANG_S = 60.0 +INTERRUPT_DEADLINE_S = 30.0 def bulky(*labels: str) -> list[Step]: @@ -87,7 +93,7 @@ def _interrupt_during_request(script: Script) -> Callable[[dict], Any]: import threading threading.Thread(target=script.session.agent.interrupt, daemon=True).start() - return Hang(seconds=60) + return Hang(seconds=INTERRUPT_HANG_S) return act @@ -110,13 +116,13 @@ def scenario(name: str, script: Script) -> list[Step]: if name == "steer": return [("turn", "warm up"), ("script", [_steer_then(ToolCall("terminal", {"command": "echo steered-tool"}), - "also mention the steer marker", s), Text("steer seen")]), + STEER_TEXT, s), Text("steer seen")]), ("turn", "do a tool while I steer"), ("turn", "after the steer")] if name == "interrupt": return [("turn", "warm up"), ("script", [_interrupt_during_request(s)]), - ("turn", "this request gets interrupted"), + ("turn", INTERRUPTED_TURN), ("turn", "the follow-up after the interrupt"), ("turn", "one more")] if name == "stream_faults": @@ -206,9 +212,17 @@ def test_transcript_ledger(world, name): script.actions.extend(arg) elif kind == "turn": ledger.inputs.append(arg) + t0 = time.monotonic() result = session.turn(arg) + took = time.monotonic() - t0 + script.raise_errors(label) ledger.inputs += [x for x in script.steered if x not in ledger.inputs] - assert result.get("final_response") is not None or name == "interrupt", f"{label}: {result}" + if arg == INTERRUPTED_TURN: + assert result.get("interrupted") is True and took < INTERRUPT_DEADLINE_S, ( + f"{label}: interrupt did not end the hung request (interrupted=" + f"{result.get('interrupted')!r} after {took:.1f}s)") + else: + assert result.get("final_response") is not None, f"{label}: {result}" compaction = name == "micro_compaction" or ( name == "auto_compaction" and arg == "the turn that must compact first") ledger.step(session.sid, label, compaction=compaction) @@ -220,6 +234,9 @@ def test_transcript_ledger(world, name): f"{res.before_tokens}->{res.after_tokens} tokens") ledger.step(session.sid, label, compaction=True, turn=False) sid = session.sid + if name == "steer": + # The ledger's inputs check then requires the steer row shown exactly once. + assert script.steered == [STEER_TEXT], f"the steer never landed: {script.steered}" finally: session.close() assert not script.actions, f"scripted responses never consumed: {script.actions}" diff --git a/tests/e2e/core/parity/_drive_gateway.py b/tests/e2e/core/parity/_drive_gateway.py index 3f2862e78c..8b93568f58 100644 --- a/tests/e2e/core/parity/_drive_gateway.py +++ b/tests/e2e/core/parity/_drive_gateway.py @@ -204,30 +204,54 @@ def _http(method: str, url: str, *, key: str | None = None, body: dict | None = return exc.code, exc.read().decode("utf-8", "replace") +PORT_ATTEMPTS = 3 + + +def _await_own_api_server(proc: subprocess.Popen, base: str, key: str) -> bool: + """True once OUR child answers on ``base`` (authenticated ``/health/detailed`` reporting its + pid, so a stranger that grabbed the port is never mistaken for it); False when the child + reports the port taken. The port is picked free and then released before the child binds + it, so another process can win that race.""" + deadline = time.monotonic() + TURN_TIMEOUT + while True: + if "already in use" in _log_tail(proc, 20000): + return False + if proc.poll() is not None: + raise AssertionError(f"api server exited {proc.returncode} before ready\n{_log_tail(proc)}") + if time.monotonic() >= deadline: + raise AssertionError(f"api server never became ready on {base}\n{_log_tail(proc)}") + with contextlib.suppress(OSError, urllib.error.URLError, ValueError): + status, raw = _http("GET", f"{base}/health/detailed", key=key, timeout=2.0) + if status == 200 and json.loads(raw).get("pid") == proc.pid: + return True + time.sleep(0.2) + + def drive_api_server(ph: ParityHome, srv: FakeLLMServer, prompt: str) -> DriveResult: ph.pin_terminal_cwd() - port = _free_loopback_port() key = secrets.token_hex(32) - _append_env(ph, {"API_SERVER_ENABLED": "true", "API_SERVER_KEY": key, - "API_SERVER_HOST": "127.0.0.1", "API_SERVER_PORT": str(port)}) - proc = _spawn(ph, [sys.executable, "-m", "gateway.run"], "api_server.stderr.log") - base = f"http://127.0.0.1:{port}" + env_before = (ph.hermes_home / ".env").read_text(encoding="utf-8") if (ph.hermes_home / ".env").exists() else "" + for _attempt in range(PORT_ATTEMPTS): + port = _free_loopback_port() + (ph.hermes_home / ".env").write_text(env_before, encoding="utf-8") + _append_env(ph, {"API_SERVER_ENABLED": "true", "API_SERVER_KEY": key, + "API_SERVER_HOST": "127.0.0.1", "API_SERVER_PORT": str(port)}) + proc = _spawn(ph, [sys.executable, "-m", "gateway.run"], "api_server.stderr.log") + base = f"http://127.0.0.1:{port}" + try: + ready = _await_own_api_server(proc, base, key) + except BaseException: + _stop(ph, proc) + raise + if ready: + break + _stop(ph, proc) + else: + raise AssertionError(f"api server lost the port race {PORT_ATTEMPTS} times\n{_log_tail(proc)}") status: int | None = None payload: dict[str, Any] = {} graceful = False try: - deadline = time.monotonic() + TURN_TIMEOUT - while True: - if proc.poll() is not None: - raise AssertionError(f"api server exited {proc.returncode} before ready\n{_log_tail(proc)}") - if time.monotonic() >= deadline: - raise AssertionError(f"api server never became ready on {base}\n{_log_tail(proc)}") - try: - if _http("GET", f"{base}/health", timeout=2.0)[0] == 200: - break - except (OSError, urllib.error.URLError): - pass - time.sleep(0.2) status, raw = _http( "POST", f"{base}/v1/chat/completions", key=key, timeout=TURN_TIMEOUT, body={"model": "hermes-agent", "messages": [{"role": "user", "content": prompt}], diff --git a/tests/e2e/core/parity/_helpers.py b/tests/e2e/core/parity/_helpers.py index fb0c6dbc2b..db04772e2a 100644 --- a/tests/e2e/core/parity/_helpers.py +++ b/tests/e2e/core/parity/_helpers.py @@ -80,9 +80,15 @@ class ParityHome: """Hermetic env for a subprocess Hermes: fake HOME, no real credentials.""" import pwd # POSIX-only; the suite is Linux-gated + # Refuse only a home the real install would read as live state (its root or a profile). + # A tmp_path under ``~/.hermes/cache/scratch`` (TMPDIR when Hermes itself runs the suite) + # is fine: the child's HOME is the fixture home, so its ``~/.hermes`` never resolves there. real_root = Path(pwd.getpwuid(os.getuid()).pw_dir, ".hermes").resolve() - assert real_root not in (self.hermes_home.resolve(), *self.hermes_home.resolve().parents), ( - f"fixture HERMES_HOME {self.hermes_home} is inside the real {real_root}") + fixture = self.hermes_home.resolve() + assert fixture != real_root and fixture.parent != real_root / "profiles", ( + f"fixture HERMES_HOME {self.hermes_home} is the real install's live home") + assert fixture == (self.home / ".hermes").resolve(), ( + f"fixture HERMES_HOME {self.hermes_home} is not /.hermes") # Allowlist, not denylist: the runner may itself be a Hermes process whose # TERMINAL_CWD / HERMES_* / credential env would silently reroute the child. env = { @@ -168,6 +174,7 @@ def build_parity_home(root: Path, base_url: str, *, grandchild: bool = True, cfg["mcp_single_query_discovery_timeout"] = 120 # Keep turns hermetic and short: no title/aux model chatter decides anything here. cfg.setdefault("display", {})["compact"] = True + cfg["updates"] = {"check": False} # offline: no GitHub round-trip or git lazy fetch (hermes_home / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8") hooks_dir = hermes_home / "agent-hooks" diff --git a/tests/e2e/core/sqlite/_roles.py b/tests/e2e/core/sqlite/_roles.py index 7221bf2349..84e25fbf18 100644 --- a/tests/e2e/core/sqlite/_roles.py +++ b/tests/e2e/core/sqlite/_roles.py @@ -80,7 +80,10 @@ def _patient(a: dict, out: Out, op: str, fn, *, deadline: float = 90.0): try: return fn() except sqlite3.OperationalError as exc: - busy = any(m in str(exc).lower() for m in ("database is locked", "database is busy")) + # By result code, not text: SQLITE_BUSY also surfaces as "vtable constructor failed: + # messages_fts" when the FTS5 table's config read hits the lock during an open. + busy = getattr(exc, "sqlite_errorcode", None) in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED) or any( + m in str(exc).lower() for m in ("database is locked", "database is busy")) if not (busy and a.get("busy_ok")) or time.monotonic() > end: raise out.report(event="busy", op=op, error=repr(exc)) diff --git a/tests/e2e/core/sqlite/test_compaction_contention.py b/tests/e2e/core/sqlite/test_compaction_contention.py index 571f2c9543..4b98ce9862 100644 --- a/tests/e2e/core/sqlite/test_compaction_contention.py +++ b/tests/e2e/core/sqlite/test_compaction_contention.py @@ -28,6 +28,7 @@ turns. Invariants after every episode: from __future__ import annotations +import contextlib import json import random import re @@ -271,6 +272,11 @@ def test_compaction_episode(rig, fault): finally: sampler.stop.set() sampler.join(timeout=10) + # The rig is shared by every fault: a failed episode's live roles must not leak into the next. + for name, _proc in ch.live(): + if name.startswith(fault): + with contextlib.suppress(AssertionError): + ch.stop(name, deadline=30.0) problems: list[str] = [] problems += [f"{n}: {e.get('error')}\n{e.get('tb', '')}" for n, e in ch.errors() if n.startswith(fault)] diff --git a/tests/e2e/core/sqlite/test_torture_chamber.py b/tests/e2e/core/sqlite/test_torture_chamber.py index 7eece48851..7ad91a6bc9 100644 --- a/tests/e2e/core/sqlite/test_torture_chamber.py +++ b/tests/e2e/core/sqlite/test_torture_chamber.py @@ -31,6 +31,7 @@ Randomness (ack thresholds, kill points) is seeded per episode; the seed is in e from __future__ import annotations +import contextlib import os import random import sqlite3 @@ -346,10 +347,15 @@ def test_torture_episode(chamber, episode): before = counts(chamber.db) if chamber.db.exists() else {"__total__": 0} started = time.monotonic() - runs = EPISODES[episode](chamber, episode, rng) - - # Nothing but the long-lived reader may still be running on the file. - stragglers = [n for n, _p in chamber.live() if n != chamber.reader_name] + try: + runs = EPISODES[episode](chamber, episode, rng) + finally: + # Nothing but the long-lived reader may still be running on the file. Stop the rest either + # way: the chamber is shared, so a failed episode's writers must not fail every later one. + stragglers = [n for n, _p in chamber.live() if n != chamber.reader_name] + for name in stragglers: + with contextlib.suppress(AssertionError): + chamber.stop(name, deadline=30.0) assert not stragglers, f"{ctx} roles still running: {stragglers}" problems: list[str] = [] problems += [f"{n}: {e.get('error')}\n{e.get('tb', '')}" for n, e in chamber.errors()] diff --git a/tests/fakes/fake_llm_provider.py b/tests/fakes/fake_llm_provider.py index 50e5ad3deb..6c0065aa62 100644 --- a/tests/fakes/fake_llm_provider.py +++ b/tests/fakes/fake_llm_provider.py @@ -78,7 +78,7 @@ class Error: @dataclass class Hang: - """Accept the request and never answer within ``seconds``.""" + """Accept the request and never answer; the connection is dropped after ``seconds``.""" seconds: float = 3600.0 @@ -272,6 +272,9 @@ def _handler_for(server: FakeLLMServer) -> type[BaseHTTPRequestHandler]: return if isinstance(resp, Hang): server._stop.wait(resp.seconds) + # Drop the socket at the deadline: on a kept-alive HTTP/1.1 connection the client + # would otherwise wait for a response that never comes, far past ``seconds``. + self.close_connection = True return if isinstance(resp, Raw): body = resp.body.encode()