diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 589351c74e..7a1642fe29 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1098,6 +1098,13 @@ class _RequestClientRegistry: # timeout's run-budget cap is applied AFTER this floor (AIAgent._compute_non_stream_stale_timeout). HIGH_EFFORT_SILENCE_FLOOR_SECONDS = 300.0 +# First-progress budget for a lifecycle-only stream on an official-Codex large request: the +# stream opened but no substantive model event has arrived. Measured from the physical-attempt +# start (a reconnect restarts it; lifecycle frames do not), and applied regardless of reasoning +# effort. Equal to the high-effort floor today, but a separate knob so tuning one cannot silently +# retune the other. +CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS = 300.0 + def _high_effort_silence_floor(agent) -> float: """``HIGH_EFFORT_SILENCE_FLOOR_SECONDS`` when the wire reasoning config is enabled at ``high`` or any @@ -1124,6 +1131,7 @@ class _NonStreamWatchdogs: idle_enabled: bool idle_timeout: float idle_requires_progress: bool + progress_timeout: float = 0.0 def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs: @@ -1209,12 +1217,13 @@ def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs # default for unset AND unparseable values, so both count as implicit. idle_explicit = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", -1.0) != -1.0 idle_timeout = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", idle_default) + progress_gated = codex and openai_codex_backend and codex_floor > 0 and not idle_explicit return _NonStreamWatchdogs(stale_timeout=stale_timeout, codex=codex, est_tokens=est_tokens, ttfb_enabled=ttfb_enabled, ttfb_timeout=ttfb_timeout, idle_enabled=codex and idle_timeout > 0, - idle_timeout=idle_timeout, - idle_requires_progress=( - codex and openai_codex_backend and codex_floor > 0 and not idle_explicit - )) + idle_timeout=idle_timeout, idle_requires_progress=progress_gated, + # A lifecycle frame proves transport liveness, not model progress. Bound that phase + # from the physical-attempt start; events cannot restart the grace period. + progress_timeout=CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS if progress_gated else 0.0) def _codex_silent_hang_hint(agent, api_kwargs: dict) -> Optional[str]: diff --git a/agent/chat_completion_nonstream.py b/agent/chat_completion_nonstream.py index 8d12aff8db..62d5bd7af5 100644 --- a/agent/chat_completion_nonstream.py +++ b/agent/chat_completion_nonstream.py @@ -119,19 +119,31 @@ class _NonStreamRequest: return self.api_kwargs.get("model", "unknown") def _codex_watchdog_snapshot(self): + """``(last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts)`` under one lock. + + ``attempt_started_ts`` is the physical attempt's origin (the reconnect marker, else + ``call_start``); the notice and the kill loop must anchor to the same value.""" state = self.codex_watchdog_state if state is None: # non-codex request: no watchdog reads these - return (None, None, None) + return (None, None, None, self.call_start) with state.lock: - return state.last_event_ts, state.last_progress_ts, state.retry_started_ts + retry_started_ts = state.retry_started_ts + return (state.last_event_ts, state.last_progress_ts, retry_started_ts, + retry_started_ts if retry_started_ts is not None else self.call_start) + + def _pre_progress(self, last_event_ts, last_progress_ts) -> bool: + """Stream open on this attempt, but no substantive model progress yet.""" + return self.wd.progress_timeout > 0 and last_event_ts is not None and last_progress_ts is None def _emit_wait_notice(self, elapsed: float, *, heartbeat: bool = True) -> None: wd = self.wd try: - last_event_ts, last_progress_ts, retry_started_ts = self._codex_watchdog_snapshot() - activity_ts = retry_started_ts if retry_started_ts is not None else last_event_ts - # Only undo a notice this request owns, promptly rather than at the - # next heartbeat: reasoning callbacks do not reset the CLI spinner. + last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot() + pre_progress = self._pre_progress(last_event_ts, last_progress_ts) + activity_ts = attempt_started_ts if pre_progress else ( + retry_started_ts if retry_started_ts is not None else last_event_ts) + # Lifecycle chatter is not progress: once pre-progress begins it must not + # clear/reset this notice. Real progress changes phase and clears it. if (self.wait_notice_started_ts is not None and activity_ts is not None and activity_ts > self.wait_notice_started_ts): self.agent._emit_wait_notice("") @@ -147,7 +159,9 @@ class _NonStreamRequest: if retry_started_ts is not None else "waiting for provider response") return phase = "first_event" - if retry_started_ts is not None: + if pre_progress: + phase = "pre_progress" + elif retry_started_ts is not None: phase = "reconnect" elif last_event_ts is not None: phase = "post_event" @@ -156,7 +170,7 @@ class _NonStreamRequest: last_event_ts=last_event_ts, last_progress_ts=last_progress_ts, retry_started_ts=retry_started_ts, call_start=self.call_start, idle_enabled=wd.idle_enabled, idle_timeout=wd.idle_timeout, - idle_requires_progress=wd.idle_requires_progress, + idle_requires_progress=wd.idle_requires_progress, progress_timeout=wd.progress_timeout, elapsed=elapsed) # One neutral notice per silence; repeating it every heartbeat made # healthy long calls read as provider trouble (#92550). @@ -189,6 +203,21 @@ class _NonStreamRequest: f"(TTFB threshold: {int(wd.ttfb_timeout)}s)" + (f". {silent_hint}" if silent_hint else "")) + def _progress_kill(self, elapsed: float) -> None: + """The stream opened, but this physical attempt never made model progress.""" + agent, wd = self.agent, self.wd + h.logger.warning("Codex stream produced lifecycle events but no substantive model progress " + "for %.0fs (threshold %.0fs, model=%s, context=~%s tokens). Reconnecting.", + elapsed, wd.progress_timeout, self._model(), f"{wd.est_tokens:,}") + agent._buffer_diagnostic_status( + f"⚠️ Codex stream opened but made no model progress for {int(elapsed)}s " + f"(model: {self._model()}). Reconnecting.") + self._abort_request("codex_progress_kill") + agent._touch_activity(f"codex stream killed after {int(elapsed)}s without model progress") + self._await_worker_after_kill( + f"Codex stream produced no substantive model progress for {int(elapsed)}s " + f"(progress threshold: {int(wd.progress_timeout)}s)") + def _idle_kill(self, event_stale_elapsed: float) -> None: """SSE events stopped after the phase-specific idle arm point. @@ -226,7 +255,7 @@ class _NonStreamRequest: def _interrupt(self, elapsed: float) -> None: agent = self.agent - last_event_ts, _, _ = self._codex_watchdog_snapshot() + last_event_ts, _, _, _ = self._codex_watchdog_snapshot() h._record_interrupted_provider_wait(agent, elapsed, response_started=self.wd.codex and last_event_ts is not None ) @@ -262,14 +291,14 @@ class _NonStreamRequest: now = h.time.time() elapsed = now - self.call_start self._emit_wait_notice(elapsed, heartbeat=poll_count % 100 == 0) - last_event_ts, last_progress_ts, retry_started_ts = self._codex_watchdog_snapshot() - retry_ttfb_elapsed = now - retry_started_ts if retry_started_ts is not None else None - if wd.ttfb_enabled and retry_ttfb_elapsed is not None and retry_ttfb_elapsed > wd.ttfb_timeout: - self._ttfb_kill(retry_ttfb_elapsed) + last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot() + attempt_elapsed = now - attempt_started_ts + if (wd.ttfb_enabled and last_event_ts is None and attempt_elapsed > wd.ttfb_timeout): + self._ttfb_kill(attempt_elapsed) break - if (retry_started_ts is None and wd.ttfb_enabled - and elapsed > wd.ttfb_timeout and last_event_ts is None): - self._ttfb_kill(elapsed) + if (self._pre_progress(last_event_ts, last_progress_ts) + and attempt_elapsed > wd.progress_timeout): + self._progress_kill(attempt_elapsed) break idle_elapsed = now - last_event_ts if last_event_ts is not None else None if (retry_started_ts is None and wd.idle_enabled and idle_elapsed is not None diff --git a/agent/chat_completion_wait_notice.py b/agent/chat_completion_wait_notice.py index 20b47b1f29..5fc15959cc 100644 --- a/agent/chat_completion_wait_notice.py +++ b/agent/chat_completion_wait_notice.py @@ -22,6 +22,7 @@ _PHASE_TEXT = { # Codex Responses (non-stream request path) "first_event": "{n}s waiting for the first provider event", "reconnect": "{n}s waiting for the first provider event after reconnect", + "pre_progress": "provider stream open; {n}s without substantive model progress", "post_event": "provider stream active; {n}s without stream events", # Chat-completions streaming path "first_chunk": "{n}s waiting for the first stream chunk", @@ -43,18 +44,19 @@ def wait_notice_text(model: str, silence_secs: float, phase: str, def codex_watchdog_deadline(*, stale_timeout: float, ttfb_enabled: bool, ttfb_timeout: float, last_event_ts: Optional[float], last_progress_ts: Optional[float], retry_started_ts: Optional[float], call_start: float, idle_enabled: bool, - idle_timeout: float, idle_requires_progress: bool, elapsed: float) -> Optional[tuple[str, float]]: + idle_timeout: float, idle_requires_progress: bool, elapsed: float, + progress_timeout: float = 0.0) -> Optional[tuple[str, float]]: """Earliest enabled Codex watchdog as ``(label, seconds_until_it_fires)``; None when none applies (disabled/infinite, or its deadline already passed).""" deadlines: list[tuple[str, float]] = [] if math.isfinite(stale_timeout): deadlines.append(("wall-clock stale", stale_timeout)) - if retry_started_ts is not None: + attempt_offset = max(0.0, retry_started_ts - call_start) if retry_started_ts is not None else 0.0 + if last_event_ts is None: if ttfb_enabled and math.isfinite(ttfb_timeout): - deadlines.append(("TTFB", max(0.0, retry_started_ts - call_start) + ttfb_timeout)) - elif last_event_ts is None: - if ttfb_enabled and math.isfinite(ttfb_timeout): - deadlines.append(("TTFB", ttfb_timeout)) + deadlines.append(("TTFB", attempt_offset + ttfb_timeout)) + elif progress_timeout > 0 and last_progress_ts is None: + deadlines.append(("first progress", attempt_offset + progress_timeout)) elif (not idle_requires_progress or last_progress_ts is not None) and idle_enabled and math.isfinite(idle_timeout): deadlines.append(("stream idle", max(0.0, last_event_ts - call_start) + idle_timeout)) if not deadlines: diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index 14ba9d17c5..917212b5b1 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -1058,14 +1058,22 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta if getattr(agent, "_last_api_first_chunk_at", None) is None: agent._last_api_first_chunk_at = now has_progress = _codex_event_has_content(event) + first_event = first_progress = False if watchdog_state is not None: with watchdog_state.lock: - if watchdog_state.retry_started_ts is not None: - watchdog_state.retry_started_ts = None - watchdog_state.last_progress_ts = None + first_event = watchdog_state.last_event_ts is None watchdog_state.last_event_ts = now if has_progress: + first_progress = watchdog_state.last_progress_ts is None watchdog_state.last_progress_ts = now + if watchdog_state.phase_aware: + watchdog_state.retry_started_ts = None + if first_event: + logger.info("Codex stream first parsed event at %.3f (attempt=%s/%s, model=%s)", + now, attempt + 1, max_stream_retries + 1, model) + if first_progress: + logger.info("Codex stream first substantive progress at %.3f (attempt=%s/%s, model=%s)", + now, attempt + 1, max_stream_retries + 1, model) agent._touch_activity("receiving stream response") def _interrupt_or_superseded() -> bool: @@ -1105,6 +1113,8 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta # Claim the delta sink for THIS attempt; a newer attempt supersedes this token. writer_token["value"] = claim_stream_writer(agent) writer_token["raw_stream"] = _raw_stream + logger.debug("Codex stream opened (attempt=%s/%s, model=%s)", + attempt + 1, max_stream_retries + 1, model) def _drain_for_finalizer(event_stream: Any) -> None: # ``final`` is already assembled; draining only lets Relay run its finalizer. A transport error @@ -1166,10 +1176,14 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta if agent._interrupt_requested: raise InterruptedError("Agent interrupted before Codex stream retry") if attempt > 0 and watchdog_state is not None and watchdog_state.phase_aware: - # A physical reconnect has its own no-event TTFB phase. Its first parsed - # event clears this marker and starts a fresh model-progress phase. + # One origin for the whole physical attempt: lifecycle frames may change + # diagnostics, but cannot restart the first-progress budget. with watchdog_state.lock: watchdog_state.retry_started_ts = time.time() + watchdog_state.last_event_ts = None + watchdog_state.last_progress_ts = None + logger.info("Codex physical stream retry at %.3f (attempt=%s/%s, model=%s)", + watchdog_state.retry_started_ts, attempt + 1, max_stream_retries + 1, model) intercepted_events: list = [] writer_token["value"] = writer_token["raw_stream"] = event_stream = None writer_token["superseded_logged"] = False diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index fb79d6242f..442f1d8b8a 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -606,14 +606,18 @@ def _print_billing_or_entitlement_guidance( )) -def _bot_chat_prompt_stale(agent, stored_prompt: str) -> bool: +def _bot_chat_prompt_stale(agent, stored_prompt: str | None) -> bool: """Bot Chat capability epoch check for a stored prompt. The stored prompt embeds a capability fingerprint; a mismatch is a deliberate once-per-change rebuild. Unstamped prompts never match; probe failures fail closed to "reuse" so the cache is kept. Legacy upgrade: a Bot Chat prompt predating the epoch mechanism gets ONE title-gated migration rebuild; the stamped result cannot - re-fire.""" + re-fire. A NULL or empty stored prompt already rebuilds every turn, so this probe + is not a gate there and must not run. + """ + if not stored_prompt: + return False try: from tools.bot_mode_probe import ( BOT_CHAT_TITLE, @@ -702,6 +706,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history) ) if stored_prompt and _stored_prompt_matches_runtime(agent, stored_prompt): + # NULL/empty rows never reach this probe: they already rebuild below. if _bot_chat_prompt_stale(agent, stored_prompt): logger.info( "Bot Chat capability epoch changed for session %s; rebuilding system prompt to " diff --git a/hermes_cli/local_runtime/hardware.py b/hermes_cli/local_runtime/hardware.py index b343ea5268..9665eaf558 100644 --- a/hermes_cli/local_runtime/hardware.py +++ b/hermes_cli/local_runtime/hardware.py @@ -61,6 +61,52 @@ def _stdout(*argv: str) -> str: ).stdout +_LINUX_RAM_KEYS = frozenset({"MemTotal", "MemAvailable", "MemFree"}) + + +def _linux_meminfo_text() -> str | None: + """Raw /proc/meminfo, or None when procfs cannot be read.""" + try: + return Path("/proc/meminfo").read_text(encoding="utf-8") + except OSError: + return None + + +def _linux_ram_from_meminfo(text: str) -> tuple[int, int] | None: + """(total, available) from meminfo text, or None if it cannot be trusted. + + MemAvailable includes reclaimable page cache, which ``getconf _AVPHYS_PAGES`` + counts as used. Kernels that omit MemAvailable fall back to MemFree. Zero is + a real available value — a missing-field check must not treat it as absent. + """ + fields: dict[str, int] = {} + try: + for line in text.splitlines(): + name, separator, value = line.partition(":") + if not separator or name not in _LINUX_RAM_KEYS: + continue + parts = value.split() + if len(parts) != 2 or parts[1] != "kB": + continue + fields[name] = int(parts[0]) * 1024 + total = fields.get("MemTotal") + # Key presence, not truthiness: MemAvailable 0 must not fall through to MemFree. + if "MemAvailable" in fields: + available = fields["MemAvailable"] + else: + available = fields.get("MemFree") + if ( + total is None + or total <= 0 + or available is None + or not 0 <= available <= total + ): + return None + return total, available + except (AttributeError, TypeError, ValueError): + return None + + def _ram_bytes() -> tuple[int, int]: """(total, available) physical memory, cross-platform stdlib.""" try: @@ -98,6 +144,12 @@ def _ram_bytes() -> tuple[int, int]: if pages > 0: avail = pages * page return total, avail + if sys.platform.startswith("linux"): + meminfo = _linux_meminfo_text() + if meminfo is not None: + linux_ram = _linux_ram_from_meminfo(meminfo) + if linux_ram is not None: + return linux_ram # POSIX page = int(_stdout("getconf", "PAGE_SIZE") or 4096) total = int(_stdout("getconf", "_PHYS_PAGES") or 0) * page diff --git a/plugin-catalog/blinkenbar.yaml b/plugin-catalog/blinkenbar.yaml index ac9ce2139c..1e35b10d58 100644 --- a/plugin-catalog/blinkenbar.yaml +++ b/plugin-catalog/blinkenbar.yaml @@ -1,13 +1,13 @@ name: blinkenbar repo: https://github.com/cygnostik/Hermes-Plugin-Blinkenlights -sha: 144876ab41205c68e4dfe621bdad6cb4fcc5e1b0 +sha: 8bd20f8ca75716fb5816ca9cd4bc48db5ca4bcf6 description: Dense, animated supercomputer light banks driven by system telemetry and live agent activity in Hermes Desktop. maintainer: cygnostik tier: community category: desktop version: "0.9.0-pre.3" docs_url: https://github.com/cygnostik/Hermes-Plugin-Blinkenlights#readme -image: https://raw.githubusercontent.com/cygnostik/Hermes-Plugin-Blinkenlights/144876ab41205c68e4dfe621bdad6cb4fcc5e1b0/docs/media/blinkenbar-catalog.png +image: https://raw.githubusercontent.com/cygnostik/Hermes-Plugin-Blinkenlights/8bd20f8ca75716fb5816ca9cd4bc48db5ca4bcf6/docs/media/blinkenbar-prodyn-hero.png platforms: [windows, macos, linux] capabilities: provides_tools: [] diff --git a/plugin-catalog/remarkable.yaml b/plugin-catalog/remarkable.yaml index db654e8c16..b20afaaa47 100644 --- a/plugin-catalog/remarkable.yaml +++ b/plugin-catalog/remarkable.yaml @@ -1,6 +1,6 @@ name: remarkable repo: https://github.com/cygnostik/hermes-plugin-remarkable -sha: 98a0cade0fdc097ad2fca476245597f3bf1ee480 +sha: d582dd0b05cf99768887a09901350e26208c1c2a description: >- Unofficial reMarkable cloud transfers, library management, backups, typed text, and bounded native notebook rendering. Disclosure — talks to reMarkable cloud with a device token under ~/.hermes/credentials/remarkable/; remarkable_upload @@ -12,6 +12,7 @@ category: tools requires_hermes: ">=0.21.3" version: "0.2.0" docs_url: https://github.com/cygnostik/hermes-plugin-remarkable#readme +image: https://raw.githubusercontent.com/cygnostik/hermes-plugin-remarkable/d582dd0b05cf99768887a09901350e26208c1c2a/docs/media/remarkable-hermes-header.png platforms: [windows, linux, macos] capabilities: provides_tools: diff --git a/tests/agent/test_codex_ttfb_watchdog.py b/tests/agent/test_codex_ttfb_watchdog.py index 907b7838ce..7947a13f2f 100644 --- a/tests/agent/test_codex_ttfb_watchdog.py +++ b/tests/agent/test_codex_ttfb_watchdog.py @@ -10,10 +10,11 @@ emitting SSE events. Parsed-event activity is recorded on the request-local watchdog state; substantive model progress is recorded separately. For the implicit official -OpenAI Codex policy on large contexts, lifecycle frames satisfy TTFB without -arming the short post-progress idle budget. Small requests, explicit overrides, -and compatible backends retain their first-parsed-event semantics. Raw SSE -comments are outside this layer. +OpenAI Codex policy on large contexts, lifecycle frames prove transport liveness +but do not restart the attempt-local first-progress budget; substantive progress +moves the attempt into the normal event-idle phase. Small requests, explicit +overrides, and compatible backends retain their first-parsed-event semantics. +Raw SSE comments are outside this layer. """ from __future__ import annotations @@ -70,14 +71,18 @@ def _make_codex_agent( return agent -def _shorten_implicit_idle_watchdog(monkeypatch, helpers, timeout=2.0): - """Keep the resolver on its implicit branch while scaling time for tests.""" +def _shorten_implicit_idle_watchdog(monkeypatch, helpers, timeout=2.0, **overrides): + """Keep the resolver on its implicit branch while scaling time for tests. + + ``timeout`` shortens ``idle_timeout``; ``overrides`` set any other resolved field.""" monkeypatch.delenv("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", raising=False) original = helpers._resolve_nonstream_watchdogs def resolve(agent, api_kwargs): watchdogs = original(agent, api_kwargs) watchdogs.idle_timeout = timeout + for field, value in overrides.items(): + setattr(watchdogs, field, value) return watchdogs monkeypatch.setattr(helpers, "_resolve_nonstream_watchdogs", resolve) @@ -320,6 +325,45 @@ def test_idle_phase_policy_is_narrow_and_preserves_operator_overrides( assert watchdogs.est_tokens == input_chars // 4 assert watchdogs.idle_enabled is idle_enabled assert watchdogs.idle_requires_progress is requires_progress + assert (watchdogs.progress_timeout > 0) is requires_progress + + +def test_lifecycle_event_does_not_restart_first_progress_deadline(): + """The budget belongs to the physical attempt, not to the first lifecycle frame.""" + from agent import chat_completion_wait_notice as wn + + deadline = wn.codex_watchdog_deadline( + stale_timeout=900.0, ttfb_enabled=True, ttfb_timeout=300.0, + last_event_ts=280.0, last_progress_ts=None, retry_started_ts=None, + call_start=100.0, idle_enabled=True, idle_timeout=120.0, + idle_requires_progress=True, progress_timeout=300.0, elapsed=250.0, + ) + + assert deadline == ("first progress", 50.0) + + +def test_large_codex_lifecycle_only_stream_hits_attempt_progress_budget(tmp_path, monkeypatch): + """Lifecycle events may change phase diagnostics, but cannot buy another full grace period.""" + from agent import chat_completion_helpers as h + + agent = _make_codex_agent(tmp_path, monkeypatch) + _shorten_implicit_idle_watchdog(monkeypatch, h, ttfb_timeout=0.9, progress_timeout=0.9) + closes = [] + + def stream_attempt(): + time.sleep(0.7) + yield SimpleNamespace(type="response.created") + while getattr(agent, "_active_codex_stream_request_token", None) is not None: + time.sleep(0.02) + raise ConnectionError("retired lifecycle-only stream") + + _install_codex_event_stream(agent, monkeypatch, stream_attempt, closes) + started = time.monotonic() + with pytest.raises(TimeoutError, match="no substantive model progress"): + h.interruptible_api_call(agent, {"model": "gpt-5.6-sol", "input": "x" * 40_004}) + + assert time.monotonic() - started < 1.5 + assert "codex_progress_kill" in closes @pytest.mark.parametrize( diff --git a/tests/agent/test_nonstream_wait_notice.py b/tests/agent/test_nonstream_wait_notice.py index 2de95bdc54..0cc64859d8 100644 --- a/tests/agent/test_nonstream_wait_notice.py +++ b/tests/agent/test_nonstream_wait_notice.py @@ -23,7 +23,7 @@ def _request(): request.call_start = 1000.0 request.wd = SimpleNamespace( codex=True, stale_timeout=600.0, ttfb_enabled=True, ttfb_timeout=120.0, - idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False, + idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False, progress_timeout=0.0, ) request.codex_watchdog_state = SimpleNamespace( lock=threading.Lock(), last_event_ts=None, last_progress_ts=None, diff --git a/tests/agent/test_system_prompt_restore.py b/tests/agent/test_system_prompt_restore.py index 20605ba129..e6eef21371 100644 --- a/tests/agent/test_system_prompt_restore.py +++ b/tests/agent/test_system_prompt_restore.py @@ -608,5 +608,37 @@ class TestPerResponseSessionWritePath: assert "is null" not in caplog.text +def test_null_stored_prompt_does_not_take_the_stale_probe_path(tmp_path): + """A NULL system_prompt row already rebuilds. The capability probe must not gate it.""" + from hermes_state import SessionDB + from agent.conversation_loop import _bot_chat_prompt_stale + + agent = SimpleNamespace( + _bot_mode_protocol=True, + _session_title_hint="Bot Chat", + _session_db=None, + session_id="test-session-id", + ) + with patch( + "tools.bot_mode_probe.stored_prompt_capability_stale", return_value=False + ) as probe: + assert _bot_chat_prompt_stale(agent, None) is False + probe.assert_not_called() + + with SessionDB(db_path=tmp_path / "state.db") as db: + db.create_session("test-session-id", source="tui") + row = db.get_session("test-session-id") + assert row is not None and row["system_prompt"] is None + restoring = _make_agent(session_db=db, prebuilt_prompt="BUILT") + with patch( + "tools.bot_mode_probe.stored_prompt_capability_stale", return_value=False + ) as probe: + _restore_or_build_system_prompt( + restoring, None, [{"role": "user", "content": "hi"}] + ) + probe.assert_not_called() + restoring._build_system_prompt.assert_called_once() + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/agent/test_wait_notice_cadence.py b/tests/agent/test_wait_notice_cadence.py index f362190508..bf7356b1d2 100644 --- a/tests/agent/test_wait_notice_cadence.py +++ b/tests/agent/test_wait_notice_cadence.py @@ -22,7 +22,8 @@ def _nonstream_request(ttfb_timeout=300.0): request.api_kwargs = {"model": "test-model"} request.call_start = 1000.0 request.wd = SimpleNamespace(codex=True, stale_timeout=600.0, ttfb_enabled=True, ttfb_timeout=ttfb_timeout, - idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False) + idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False, + progress_timeout=0.0) request.codex_watchdog_state = SimpleNamespace(lock=threading.Lock(), last_event_ts=None, last_progress_ts=None, retry_started_ts=None) request.wait_notice_started_ts = None diff --git a/tests/hermes_cli/test_linux_ram_probe.py b/tests/hermes_cli/test_linux_ram_probe.py new file mode 100644 index 0000000000..eb4250fc5c --- /dev/null +++ b/tests/hermes_cli/test_linux_ram_probe.py @@ -0,0 +1,107 @@ +"""Linux RAM availability must include reclaimable page cache (#102252). + +``getconf _AVPHYS_PAGES`` counts only free pages, so the Desktop statusbar +treats page cache as used. ``MemAvailable`` is the reclaimable-aware figure. +A reported ``0`` is a real available value, not a missing field. +""" + +from __future__ import annotations + +import ctypes + +import hermes_cli.local_runtime.hardware as hw + +GIB = 1 << 30 + + +def _as_linux(monkeypatch) -> None: + """Reach the Linux branch on every host without editing the Windows probe.""" + monkeypatch.delattr(ctypes, "windll", raising=False) + monkeypatch.setattr(hw.sys, "platform", "linux") + + +def test_memavailable_is_used_when_present(): + text = ( + "MemTotal: 67108864 kB\n" + "MemFree: 12582912 kB\n" + "MemAvailable: 56623104 kB\n" + "Cached: 44040192 kB\n" + ) + + assert hw._linux_ram_from_meminfo(text) == (64 * GIB, 54 * GIB) + + +def test_zero_memavailable_is_a_real_value(): + text = ( + "MemTotal: 8388608 kB\n" + "MemFree: 2097152 kB\n" + "MemAvailable: 0 kB\n" + ) + + assert hw._linux_ram_from_meminfo(text) == (8 * GIB, 0) + + +def test_missing_memavailable_falls_back_to_memfree(): + text = "MemTotal: 8388608 kB\nMemFree: 2097152 kB\n" + + assert hw._linux_ram_from_meminfo(text) == (8 * GIB, 2 * GIB) + + +def test_unusable_meminfo_is_not_a_reading(): + assert hw._linux_ram_from_meminfo("MemAvailable: 1048576 kB\n") is None + assert hw._linux_ram_from_meminfo( + "MemTotal: 1048576 kB\nMemAvailable: 2097152 kB\n" + ) is None + + +def test_ram_bytes_uses_injected_meminfo_including_zero(monkeypatch): + _as_linux(monkeypatch) + monkeypatch.setattr( + hw, + "_linux_meminfo_text", + lambda: ( + "MemTotal: 8388608 kB\n" + "MemFree: 2097152 kB\n" + "MemAvailable: 0 kB\n" + ), + ) + + def _getconf_must_not_run(*_args, **_kwargs): + raise AssertionError("getconf must not run when MemAvailable is present") + + monkeypatch.setattr(hw, "_stdout", _getconf_must_not_run) + + assert hw._ram_bytes() == (8 * GIB, 0) + + +def test_ram_bytes_falls_back_when_memavailable_is_absent(monkeypatch): + _as_linux(monkeypatch) + monkeypatch.setattr( + hw, + "_linux_meminfo_text", + lambda: "MemTotal: 8388608 kB\nMemFree: 2097152 kB\n", + ) + + def _getconf_must_not_run(*_args, **_kwargs): + raise AssertionError("MemFree is a valid fallback; getconf must not run") + + monkeypatch.setattr(hw, "_stdout", _getconf_must_not_run) + + assert hw._ram_bytes() == (8 * GIB, 2 * GIB) + + +def test_ram_bytes_falls_back_to_getconf_when_meminfo_unusable(monkeypatch): + _as_linux(monkeypatch) + monkeypatch.setattr(hw, "_linux_meminfo_text", lambda: None) + values = { + "PAGE_SIZE": "4096\n", + "_PHYS_PAGES": "2097152\n", + "_AVPHYS_PAGES": "524288\n", + } + + def fake_stdout(*argv): + return values[argv[-1]] + + monkeypatch.setattr(hw, "_stdout", fake_stdout) + + assert hw._ram_bytes() == (8 * GIB, 2 * GIB) diff --git a/tests/tools/test_bot_mode_probe.py b/tests/tools/test_bot_mode_probe.py index 00e87e6c33..3f542520f0 100644 --- a/tests/tools/test_bot_mode_probe.py +++ b/tests/tools/test_bot_mode_probe.py @@ -209,6 +209,60 @@ def test_fingerprint_changes_on_each_capability_axis(tmp_path): assert bot_mode_probe.capability_fingerprint(home) != after_soul +def test_fingerprint_changes_when_model_vision_override_flips(tmp_path): + """A Bot Chat prompt must rebuild when model.supports_vision flips.""" + home = tmp_path / ".hermes" + home.mkdir() + _make_bot_profile(home, "researcher", managed=True) + config = home / "config.yaml" + + config.write_text("model:\n supports_vision: true\n", encoding="utf-8") + vision_enabled = bot_mode_probe.capability_fingerprint(home) + stamped = "system stuff\n\n" + bot_mode_probe.epoch_line(home) + assert vision_enabled == bot_mode_probe.capability_fingerprint(home) + assert not bot_mode_probe.stored_prompt_capability_stale(stamped, home) + + config.write_text("model:\n supports_vision: false\n", encoding="utf-8") + vision_disabled = bot_mode_probe.capability_fingerprint(home) + assert vision_disabled != vision_enabled + assert bot_mode_probe.stored_prompt_capability_stale(stamped, home) + + restamped = "system stuff\n\n" + bot_mode_probe.epoch_line(home) + assert not bot_mode_probe.stored_prompt_capability_stale(restamped, home) + config.write_text("model:\n supports_vision: true\n", encoding="utf-8") + assert bot_mode_probe.capability_fingerprint(home) != vision_disabled + + +def test_vision_override_spellings_share_one_fingerprint(tmp_path): + """YAML boolean tokens that image routing treats as the same override share an epoch.""" + home = tmp_path / ".hermes" + home.mkdir() + _make_bot_profile(home, "researcher", managed=True) + config = home / "config.yaml" + + config.write_text("model:\n supports_vision: true\n", encoding="utf-8") + enabled = bot_mode_probe.capability_fingerprint(home) + config.write_text("model:\n supports_vision: yes\n", encoding="utf-8") + assert bot_mode_probe.capability_fingerprint(home) == enabled + config.write_text("model:\n supports_vision: false\n", encoding="utf-8") + assert bot_mode_probe.capability_fingerprint(home) != enabled + + +def test_fingerprint_changes_when_model_context_length_override_changes(tmp_path): + """context_length truncates context files in the rebuilt prompt, so it is part of the epoch.""" + home = tmp_path / ".hermes" + home.mkdir() + _make_bot_profile(home, "researcher", managed=True) + config = home / "config.yaml" + + config.write_text("model:\n context_length: 32768\n", encoding="utf-8") + narrow = bot_mode_probe.capability_fingerprint(home) + config.write_text("model:\n context_length: 131072\n", encoding="utf-8") + assert bot_mode_probe.capability_fingerprint(home) != narrow + config.write_text("model:\n context_length: 32768\n", encoding="utf-8") + assert bot_mode_probe.capability_fingerprint(home) == narrow + + def test_stored_prompt_staleness(tmp_path): home = tmp_path / ".hermes" home.mkdir() diff --git a/tests/tools/test_fuzzy_match.py b/tests/tools/test_fuzzy_match.py index 6c52ea47c4..64f4adcdb6 100644 --- a/tests/tools/test_fuzzy_match.py +++ b/tests/tools/test_fuzzy_match.py @@ -229,6 +229,28 @@ class TestUnicodeNormalized: assert new == expected, f"Got {new!r}" + def test_equal_boundary_inside_expansion_keeps_region_text(self): + """An edit boundary falling inside a multi-char expansion (em-dash -> + '--') must snap to the expansion, not copy text from the region start. + + SequenceMatcher splits old/new so that the second equal block begins + at the expansion's second '-': a norm index with no direct original + position. The old fallback (position 0) spliced the whole region text + into the replacement, duplicating it after every edit. + """ + content = "value = x\u2014y\n" + new, count, strategy, err = fuzzy_find_and_replace( + content, "value = x--y", "value = x-@-y") + assert count == 1, f"Expected match, got err={err}" + assert strategy == "unicode_normalized" + assert new == "value = x\u2014@\u2014y\n", f"Got {new!r}" + + # Same boundary class for a 3-char expansion (ellipsis -> '...'). + new, count, strategy, err = fuzzy_find_and_replace("a\u2026b\n", "a...b", "a..X.b") + assert count == 1, f"Expected match, got err={err}" + assert strategy == "unicode_normalized" + assert new == "a\u2026X\u2026b\n", f"Got {new!r}" + class TestUnicodeSpaceAndMinusNormalized: """Space-separator family + Unicode minus normalization. diff --git a/tools/bot_mode_probe.py b/tools/bot_mode_probe.py index dd44c4d3c5..a964388c67 100644 --- a/tools/bot_mode_probe.py +++ b/tools/bot_mode_probe.py @@ -318,19 +318,50 @@ def get_bot_mode_protocol_section(home: str | os.PathLike | None = None, *, forc # ── capability epoch ───────────────────────────────────────────────────────── # Bot Chat sessions are effectively eternal, so "build the prompt once" would strand -# capability changes (skills, toolsets, MCP, SOUL, roster, peers) forever. The fingerprint -# hashes exactly that surface; the built prompt embeds it and agent/conversation_loop.py -# rebuilds only when the stored epoch differs from disk — once per change, never per-turn drift. +# capability changes (skills, toolsets, MCP, SOUL, roster, peers, model capability +# overrides that change the prompt) forever. The fingerprint hashes exactly that +# surface; the built prompt embeds it and agent/conversation_loop.py rebuilds only +# when the stored epoch differs from disk — once per change, never per-turn drift. _EPOCH_PREFIX = "Capability epoch: " _EPOCH_RE_TEXT = r"Capability epoch: ([0-9a-f]{12})" +def _model_prompt_capability_surface(model_cfg: object) -> dict: + """``model.*`` overrides whose flip changes a rebuilt prompt, coerced like their consumers. + + ``supports_vision`` uses image routing's strict bool so YAML ``yes`` and ``true`` share + one epoch. ``context_length`` is the cap ``build_system_prompt_parts`` uses to truncate + context files. Routing keys (provider, default, base_url) are identity lines, not this + surface — and no model id is special-cased. + """ + from agent.image_routing import _coerce_capability_bool + + if not isinstance(model_cfg, dict): + model_cfg = {} + raw_ctx = model_cfg.get("context_length") + ctx = None + # bool is an int subclass; ``context_length: true`` is not a window. + if isinstance(raw_ctx, bool): + ctx = None + elif isinstance(raw_ctx, int): + ctx = raw_ctx if raw_ctx > 0 else None + elif isinstance(raw_ctx, str) and raw_ctx.strip().isdigit(): + parsed = int(raw_ctx.strip()) + ctx = parsed if parsed > 0 else None + return { + "supports_vision": _coerce_capability_bool(model_cfg.get("supports_vision")), + "context_length": ctx, + } + + def capability_fingerprint(home: str | os.PathLike | None = None) -> str: """12-hex digest of the capability surface for ``home``'s profile: disabled skills + - enabled toolsets + MCP config, SOUL.md bytes, installed skill names, the Bot-Mode roster - (+ roles), peers and the relay roster. Deliberately NOT cached — the point is detecting - on-disk drift against a stored prompt's epoch. Never raises ("unavailable" on failure).""" + enabled toolsets + MCP config, model capability overrides that change the prompt + (``supports_vision``, ``context_length``), SOUL.md bytes, installed skill names, the + Bot-Mode roster (+ roles), peers and the relay roster. Deliberately NOT cached — the + point is detecting on-disk drift against a stored prompt's epoch. Never raises + ("unavailable" on failure).""" import hashlib import json @@ -350,6 +381,8 @@ def capability_fingerprint(home: str | os.PathLike | None = None) -> str: reset_hermes_home_override(token) skills_cfg = cfg.get("skills") if isinstance(cfg.get("skills"), dict) else {} tools_cfg = cfg.get("tools") if isinstance(cfg.get("tools"), dict) else {} + model_cfg = cfg.get("model") if isinstance(cfg.get("model"), dict) else {} + surface["model_capabilities"] = _model_prompt_capability_surface(model_cfg) surface["disabled_skills"] = sorted(str(s).lower() for s in (skills_cfg.get("disabled") or [])) surface["enabled_toolsets"] = sorted(str(t) for t in (tools_cfg.get("enabled_toolsets") or [])) mcp = cfg.get("mcp_servers") diff --git a/tools/fuzzy_match.py b/tools/fuzzy_match.py index 1ee3d9f6c5..c60d098405 100644 --- a/tools/fuzzy_match.py +++ b/tools/fuzzy_match.py @@ -9,6 +9,7 @@ still land on the intended region:: content, old_string, new_string, replace_all=False) """ +import bisect import re from difflib import SequenceMatcher from typing import Callable, Optional @@ -509,12 +510,12 @@ def _preserve_unicode_in_replacement(content: str, matches: list[Span], return new_string # strategy shouldn't have fired; fall back file_orig_to_norm = _build_orig_to_norm_map(file_region) - file_norm_to_orig = _invert_norm_map(file_orig_to_norm) result_parts: list[str] = [] for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes(): if tag == "equal": - orig_start = file_norm_to_orig.get(i1, 0) + # The original char owning norm index i1, even one inside a multi-char expansion (em-dash -> '--'). + orig_start = bisect.bisect_right(file_orig_to_norm, i1) - 1 orig_end = _norm_end_to_orig(file_orig_to_norm, orig_start, i2) result_parts.append(file_region[orig_start:orig_end]) elif tag != "delete":