Merge remote-tracking branch 'origin/main' into ethie/pm-clean
This commit is contained in:
@@ -1098,6 +1098,13 @@ class _RequestClientRegistry:
|
||||
# timeout's run-budget cap is applied AFTER this floor (AIAgent._compute_non_stream_stale_timeout).
|
||||
HIGH_EFFORT_SILENCE_FLOOR_SECONDS = 300.0
|
||||
|
||||
# First-progress budget for a lifecycle-only stream on an official-Codex large request: the
|
||||
# stream opened but no substantive model event has arrived. Measured from the physical-attempt
|
||||
# start (a reconnect restarts it; lifecycle frames do not), and applied regardless of reasoning
|
||||
# effort. Equal to the high-effort floor today, but a separate knob so tuning one cannot silently
|
||||
# retune the other.
|
||||
CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS = 300.0
|
||||
|
||||
|
||||
def _high_effort_silence_floor(agent) -> float:
|
||||
"""``HIGH_EFFORT_SILENCE_FLOOR_SECONDS`` when the wire reasoning config is enabled at ``high`` or any
|
||||
@@ -1124,6 +1131,7 @@ class _NonStreamWatchdogs:
|
||||
idle_enabled: bool
|
||||
idle_timeout: float
|
||||
idle_requires_progress: bool
|
||||
progress_timeout: float = 0.0
|
||||
|
||||
|
||||
def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs:
|
||||
@@ -1209,12 +1217,13 @@ def _resolve_nonstream_watchdogs(agent, api_kwargs: dict) -> _NonStreamWatchdogs
|
||||
# default for unset AND unparseable values, so both count as implicit.
|
||||
idle_explicit = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", -1.0) != -1.0
|
||||
idle_timeout = env_float("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", idle_default)
|
||||
progress_gated = codex and openai_codex_backend and codex_floor > 0 and not idle_explicit
|
||||
return _NonStreamWatchdogs(stale_timeout=stale_timeout, codex=codex, est_tokens=est_tokens,
|
||||
ttfb_enabled=ttfb_enabled, ttfb_timeout=ttfb_timeout, idle_enabled=codex and idle_timeout > 0,
|
||||
idle_timeout=idle_timeout,
|
||||
idle_requires_progress=(
|
||||
codex and openai_codex_backend and codex_floor > 0 and not idle_explicit
|
||||
))
|
||||
idle_timeout=idle_timeout, idle_requires_progress=progress_gated,
|
||||
# A lifecycle frame proves transport liveness, not model progress. Bound that phase
|
||||
# from the physical-attempt start; events cannot restart the grace period.
|
||||
progress_timeout=CODEX_FIRST_PROGRESS_TIMEOUT_SECONDS if progress_gated else 0.0)
|
||||
|
||||
|
||||
def _codex_silent_hang_hint(agent, api_kwargs: dict) -> Optional[str]:
|
||||
|
||||
@@ -119,19 +119,31 @@ class _NonStreamRequest:
|
||||
return self.api_kwargs.get("model", "unknown")
|
||||
|
||||
def _codex_watchdog_snapshot(self):
|
||||
"""``(last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts)`` under one lock.
|
||||
|
||||
``attempt_started_ts`` is the physical attempt's origin (the reconnect marker, else
|
||||
``call_start``); the notice and the kill loop must anchor to the same value."""
|
||||
state = self.codex_watchdog_state
|
||||
if state is None: # non-codex request: no watchdog reads these
|
||||
return (None, None, None)
|
||||
return (None, None, None, self.call_start)
|
||||
with state.lock:
|
||||
return state.last_event_ts, state.last_progress_ts, state.retry_started_ts
|
||||
retry_started_ts = state.retry_started_ts
|
||||
return (state.last_event_ts, state.last_progress_ts, retry_started_ts,
|
||||
retry_started_ts if retry_started_ts is not None else self.call_start)
|
||||
|
||||
def _pre_progress(self, last_event_ts, last_progress_ts) -> bool:
|
||||
"""Stream open on this attempt, but no substantive model progress yet."""
|
||||
return self.wd.progress_timeout > 0 and last_event_ts is not None and last_progress_ts is None
|
||||
|
||||
def _emit_wait_notice(self, elapsed: float, *, heartbeat: bool = True) -> None:
|
||||
wd = self.wd
|
||||
try:
|
||||
last_event_ts, last_progress_ts, retry_started_ts = self._codex_watchdog_snapshot()
|
||||
activity_ts = retry_started_ts if retry_started_ts is not None else last_event_ts
|
||||
# Only undo a notice this request owns, promptly rather than at the
|
||||
# next heartbeat: reasoning callbacks do not reset the CLI spinner.
|
||||
last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot()
|
||||
pre_progress = self._pre_progress(last_event_ts, last_progress_ts)
|
||||
activity_ts = attempt_started_ts if pre_progress else (
|
||||
retry_started_ts if retry_started_ts is not None else last_event_ts)
|
||||
# Lifecycle chatter is not progress: once pre-progress begins it must not
|
||||
# clear/reset this notice. Real progress changes phase and clears it.
|
||||
if (self.wait_notice_started_ts is not None and activity_ts is not None
|
||||
and activity_ts > self.wait_notice_started_ts):
|
||||
self.agent._emit_wait_notice("")
|
||||
@@ -147,7 +159,9 @@ class _NonStreamRequest:
|
||||
if retry_started_ts is not None else "waiting for provider response")
|
||||
return
|
||||
phase = "first_event"
|
||||
if retry_started_ts is not None:
|
||||
if pre_progress:
|
||||
phase = "pre_progress"
|
||||
elif retry_started_ts is not None:
|
||||
phase = "reconnect"
|
||||
elif last_event_ts is not None:
|
||||
phase = "post_event"
|
||||
@@ -156,7 +170,7 @@ class _NonStreamRequest:
|
||||
last_event_ts=last_event_ts, last_progress_ts=last_progress_ts,
|
||||
retry_started_ts=retry_started_ts,
|
||||
call_start=self.call_start, idle_enabled=wd.idle_enabled, idle_timeout=wd.idle_timeout,
|
||||
idle_requires_progress=wd.idle_requires_progress,
|
||||
idle_requires_progress=wd.idle_requires_progress, progress_timeout=wd.progress_timeout,
|
||||
elapsed=elapsed)
|
||||
# One neutral notice per silence; repeating it every heartbeat made
|
||||
# healthy long calls read as provider trouble (#92550).
|
||||
@@ -189,6 +203,21 @@ class _NonStreamRequest:
|
||||
f"(TTFB threshold: {int(wd.ttfb_timeout)}s)"
|
||||
+ (f". {silent_hint}" if silent_hint else ""))
|
||||
|
||||
def _progress_kill(self, elapsed: float) -> None:
|
||||
"""The stream opened, but this physical attempt never made model progress."""
|
||||
agent, wd = self.agent, self.wd
|
||||
h.logger.warning("Codex stream produced lifecycle events but no substantive model progress "
|
||||
"for %.0fs (threshold %.0fs, model=%s, context=~%s tokens). Reconnecting.",
|
||||
elapsed, wd.progress_timeout, self._model(), f"{wd.est_tokens:,}")
|
||||
agent._buffer_diagnostic_status(
|
||||
f"⚠️ Codex stream opened but made no model progress for {int(elapsed)}s "
|
||||
f"(model: {self._model()}). Reconnecting.")
|
||||
self._abort_request("codex_progress_kill")
|
||||
agent._touch_activity(f"codex stream killed after {int(elapsed)}s without model progress")
|
||||
self._await_worker_after_kill(
|
||||
f"Codex stream produced no substantive model progress for {int(elapsed)}s "
|
||||
f"(progress threshold: {int(wd.progress_timeout)}s)")
|
||||
|
||||
def _idle_kill(self, event_stale_elapsed: float) -> None:
|
||||
"""SSE events stopped after the phase-specific idle arm point.
|
||||
|
||||
@@ -226,7 +255,7 @@ class _NonStreamRequest:
|
||||
|
||||
def _interrupt(self, elapsed: float) -> None:
|
||||
agent = self.agent
|
||||
last_event_ts, _, _ = self._codex_watchdog_snapshot()
|
||||
last_event_ts, _, _, _ = self._codex_watchdog_snapshot()
|
||||
h._record_interrupted_provider_wait(agent, elapsed,
|
||||
response_started=self.wd.codex and last_event_ts is not None
|
||||
)
|
||||
@@ -262,14 +291,14 @@ class _NonStreamRequest:
|
||||
now = h.time.time()
|
||||
elapsed = now - self.call_start
|
||||
self._emit_wait_notice(elapsed, heartbeat=poll_count % 100 == 0)
|
||||
last_event_ts, last_progress_ts, retry_started_ts = self._codex_watchdog_snapshot()
|
||||
retry_ttfb_elapsed = now - retry_started_ts if retry_started_ts is not None else None
|
||||
if wd.ttfb_enabled and retry_ttfb_elapsed is not None and retry_ttfb_elapsed > wd.ttfb_timeout:
|
||||
self._ttfb_kill(retry_ttfb_elapsed)
|
||||
last_event_ts, last_progress_ts, retry_started_ts, attempt_started_ts = self._codex_watchdog_snapshot()
|
||||
attempt_elapsed = now - attempt_started_ts
|
||||
if (wd.ttfb_enabled and last_event_ts is None and attempt_elapsed > wd.ttfb_timeout):
|
||||
self._ttfb_kill(attempt_elapsed)
|
||||
break
|
||||
if (retry_started_ts is None and wd.ttfb_enabled
|
||||
and elapsed > wd.ttfb_timeout and last_event_ts is None):
|
||||
self._ttfb_kill(elapsed)
|
||||
if (self._pre_progress(last_event_ts, last_progress_ts)
|
||||
and attempt_elapsed > wd.progress_timeout):
|
||||
self._progress_kill(attempt_elapsed)
|
||||
break
|
||||
idle_elapsed = now - last_event_ts if last_event_ts is not None else None
|
||||
if (retry_started_ts is None and wd.idle_enabled and idle_elapsed is not None
|
||||
|
||||
@@ -22,6 +22,7 @@ _PHASE_TEXT = {
|
||||
# Codex Responses (non-stream request path)
|
||||
"first_event": "{n}s waiting for the first provider event",
|
||||
"reconnect": "{n}s waiting for the first provider event after reconnect",
|
||||
"pre_progress": "provider stream open; {n}s without substantive model progress",
|
||||
"post_event": "provider stream active; {n}s without stream events",
|
||||
# Chat-completions streaming path
|
||||
"first_chunk": "{n}s waiting for the first stream chunk",
|
||||
@@ -43,18 +44,19 @@ def wait_notice_text(model: str, silence_secs: float, phase: str,
|
||||
def codex_watchdog_deadline(*, stale_timeout: float, ttfb_enabled: bool, ttfb_timeout: float,
|
||||
last_event_ts: Optional[float], last_progress_ts: Optional[float],
|
||||
retry_started_ts: Optional[float], call_start: float, idle_enabled: bool,
|
||||
idle_timeout: float, idle_requires_progress: bool, elapsed: float) -> Optional[tuple[str, float]]:
|
||||
idle_timeout: float, idle_requires_progress: bool, elapsed: float,
|
||||
progress_timeout: float = 0.0) -> Optional[tuple[str, float]]:
|
||||
"""Earliest enabled Codex watchdog as ``(label, seconds_until_it_fires)``; None when
|
||||
none applies (disabled/infinite, or its deadline already passed)."""
|
||||
deadlines: list[tuple[str, float]] = []
|
||||
if math.isfinite(stale_timeout):
|
||||
deadlines.append(("wall-clock stale", stale_timeout))
|
||||
if retry_started_ts is not None:
|
||||
attempt_offset = max(0.0, retry_started_ts - call_start) if retry_started_ts is not None else 0.0
|
||||
if last_event_ts is None:
|
||||
if ttfb_enabled and math.isfinite(ttfb_timeout):
|
||||
deadlines.append(("TTFB", max(0.0, retry_started_ts - call_start) + ttfb_timeout))
|
||||
elif last_event_ts is None:
|
||||
if ttfb_enabled and math.isfinite(ttfb_timeout):
|
||||
deadlines.append(("TTFB", ttfb_timeout))
|
||||
deadlines.append(("TTFB", attempt_offset + ttfb_timeout))
|
||||
elif progress_timeout > 0 and last_progress_ts is None:
|
||||
deadlines.append(("first progress", attempt_offset + progress_timeout))
|
||||
elif (not idle_requires_progress or last_progress_ts is not None) and idle_enabled and math.isfinite(idle_timeout):
|
||||
deadlines.append(("stream idle", max(0.0, last_event_ts - call_start) + idle_timeout))
|
||||
if not deadlines:
|
||||
|
||||
@@ -1058,14 +1058,22 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta
|
||||
if getattr(agent, "_last_api_first_chunk_at", None) is None:
|
||||
agent._last_api_first_chunk_at = now
|
||||
has_progress = _codex_event_has_content(event)
|
||||
first_event = first_progress = False
|
||||
if watchdog_state is not None:
|
||||
with watchdog_state.lock:
|
||||
if watchdog_state.retry_started_ts is not None:
|
||||
watchdog_state.retry_started_ts = None
|
||||
watchdog_state.last_progress_ts = None
|
||||
first_event = watchdog_state.last_event_ts is None
|
||||
watchdog_state.last_event_ts = now
|
||||
if has_progress:
|
||||
first_progress = watchdog_state.last_progress_ts is None
|
||||
watchdog_state.last_progress_ts = now
|
||||
if watchdog_state.phase_aware:
|
||||
watchdog_state.retry_started_ts = None
|
||||
if first_event:
|
||||
logger.info("Codex stream first parsed event at %.3f (attempt=%s/%s, model=%s)",
|
||||
now, attempt + 1, max_stream_retries + 1, model)
|
||||
if first_progress:
|
||||
logger.info("Codex stream first substantive progress at %.3f (attempt=%s/%s, model=%s)",
|
||||
now, attempt + 1, max_stream_retries + 1, model)
|
||||
agent._touch_activity("receiving stream response")
|
||||
|
||||
def _interrupt_or_superseded() -> bool:
|
||||
@@ -1105,6 +1113,8 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta
|
||||
# Claim the delta sink for THIS attempt; a newer attempt supersedes this token.
|
||||
writer_token["value"] = claim_stream_writer(agent)
|
||||
writer_token["raw_stream"] = _raw_stream
|
||||
logger.debug("Codex stream opened (attempt=%s/%s, model=%s)",
|
||||
attempt + 1, max_stream_retries + 1, model)
|
||||
|
||||
def _drain_for_finalizer(event_stream: Any) -> None:
|
||||
# ``final`` is already assembled; draining only lets Relay run its finalizer. A transport error
|
||||
@@ -1166,10 +1176,14 @@ def run_codex_stream(agent, api_kwargs: dict, client: Any = None, on_first_delta
|
||||
if agent._interrupt_requested:
|
||||
raise InterruptedError("Agent interrupted before Codex stream retry")
|
||||
if attempt > 0 and watchdog_state is not None and watchdog_state.phase_aware:
|
||||
# A physical reconnect has its own no-event TTFB phase. Its first parsed
|
||||
# event clears this marker and starts a fresh model-progress phase.
|
||||
# One origin for the whole physical attempt: lifecycle frames may change
|
||||
# diagnostics, but cannot restart the first-progress budget.
|
||||
with watchdog_state.lock:
|
||||
watchdog_state.retry_started_ts = time.time()
|
||||
watchdog_state.last_event_ts = None
|
||||
watchdog_state.last_progress_ts = None
|
||||
logger.info("Codex physical stream retry at %.3f (attempt=%s/%s, model=%s)",
|
||||
watchdog_state.retry_started_ts, attempt + 1, max_stream_retries + 1, model)
|
||||
intercepted_events: list = []
|
||||
writer_token["value"] = writer_token["raw_stream"] = event_stream = None
|
||||
writer_token["superseded_logged"] = False
|
||||
|
||||
@@ -606,14 +606,18 @@ def _print_billing_or_entitlement_guidance(
|
||||
))
|
||||
|
||||
|
||||
def _bot_chat_prompt_stale(agent, stored_prompt: str) -> bool:
|
||||
def _bot_chat_prompt_stale(agent, stored_prompt: str | None) -> bool:
|
||||
"""Bot Chat capability epoch check for a stored prompt.
|
||||
|
||||
The stored prompt embeds a capability fingerprint; a mismatch is a deliberate
|
||||
once-per-change rebuild. Unstamped prompts never match; probe failures fail closed
|
||||
to "reuse" so the cache is kept. Legacy upgrade: a Bot Chat prompt predating the
|
||||
epoch mechanism gets ONE title-gated migration rebuild; the stamped result cannot
|
||||
re-fire."""
|
||||
re-fire. A NULL or empty stored prompt already rebuilds every turn, so this probe
|
||||
is not a gate there and must not run.
|
||||
"""
|
||||
if not stored_prompt:
|
||||
return False
|
||||
try:
|
||||
from tools.bot_mode_probe import (
|
||||
BOT_CHAT_TITLE,
|
||||
@@ -702,6 +706,7 @@ def _restore_or_build_system_prompt(agent, system_message, conversation_history)
|
||||
)
|
||||
|
||||
if stored_prompt and _stored_prompt_matches_runtime(agent, stored_prompt):
|
||||
# NULL/empty rows never reach this probe: they already rebuild below.
|
||||
if _bot_chat_prompt_stale(agent, stored_prompt):
|
||||
logger.info(
|
||||
"Bot Chat capability epoch changed for session %s; rebuilding system prompt to "
|
||||
|
||||
@@ -61,6 +61,52 @@ def _stdout(*argv: str) -> str:
|
||||
).stdout
|
||||
|
||||
|
||||
_LINUX_RAM_KEYS = frozenset({"MemTotal", "MemAvailable", "MemFree"})
|
||||
|
||||
|
||||
def _linux_meminfo_text() -> str | None:
|
||||
"""Raw /proc/meminfo, or None when procfs cannot be read."""
|
||||
try:
|
||||
return Path("/proc/meminfo").read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _linux_ram_from_meminfo(text: str) -> tuple[int, int] | None:
|
||||
"""(total, available) from meminfo text, or None if it cannot be trusted.
|
||||
|
||||
MemAvailable includes reclaimable page cache, which ``getconf _AVPHYS_PAGES``
|
||||
counts as used. Kernels that omit MemAvailable fall back to MemFree. Zero is
|
||||
a real available value — a missing-field check must not treat it as absent.
|
||||
"""
|
||||
fields: dict[str, int] = {}
|
||||
try:
|
||||
for line in text.splitlines():
|
||||
name, separator, value = line.partition(":")
|
||||
if not separator or name not in _LINUX_RAM_KEYS:
|
||||
continue
|
||||
parts = value.split()
|
||||
if len(parts) != 2 or parts[1] != "kB":
|
||||
continue
|
||||
fields[name] = int(parts[0]) * 1024
|
||||
total = fields.get("MemTotal")
|
||||
# Key presence, not truthiness: MemAvailable 0 must not fall through to MemFree.
|
||||
if "MemAvailable" in fields:
|
||||
available = fields["MemAvailable"]
|
||||
else:
|
||||
available = fields.get("MemFree")
|
||||
if (
|
||||
total is None
|
||||
or total <= 0
|
||||
or available is None
|
||||
or not 0 <= available <= total
|
||||
):
|
||||
return None
|
||||
return total, available
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _ram_bytes() -> tuple[int, int]:
|
||||
"""(total, available) physical memory, cross-platform stdlib."""
|
||||
try:
|
||||
@@ -98,6 +144,12 @@ def _ram_bytes() -> tuple[int, int]:
|
||||
if pages > 0:
|
||||
avail = pages * page
|
||||
return total, avail
|
||||
if sys.platform.startswith("linux"):
|
||||
meminfo = _linux_meminfo_text()
|
||||
if meminfo is not None:
|
||||
linux_ram = _linux_ram_from_meminfo(meminfo)
|
||||
if linux_ram is not None:
|
||||
return linux_ram
|
||||
# POSIX
|
||||
page = int(_stdout("getconf", "PAGE_SIZE") or 4096)
|
||||
total = int(_stdout("getconf", "_PHYS_PAGES") or 0) * page
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
name: blinkenbar
|
||||
repo: https://github.com/cygnostik/Hermes-Plugin-Blinkenlights
|
||||
sha: 144876ab41205c68e4dfe621bdad6cb4fcc5e1b0
|
||||
sha: 8bd20f8ca75716fb5816ca9cd4bc48db5ca4bcf6
|
||||
description: Dense, animated supercomputer light banks driven by system telemetry and live agent activity in Hermes Desktop.
|
||||
maintainer: cygnostik
|
||||
tier: community
|
||||
category: desktop
|
||||
version: "0.9.0-pre.3"
|
||||
docs_url: https://github.com/cygnostik/Hermes-Plugin-Blinkenlights#readme
|
||||
image: https://raw.githubusercontent.com/cygnostik/Hermes-Plugin-Blinkenlights/144876ab41205c68e4dfe621bdad6cb4fcc5e1b0/docs/media/blinkenbar-catalog.png
|
||||
image: https://raw.githubusercontent.com/cygnostik/Hermes-Plugin-Blinkenlights/8bd20f8ca75716fb5816ca9cd4bc48db5ca4bcf6/docs/media/blinkenbar-prodyn-hero.png
|
||||
platforms: [windows, macos, linux]
|
||||
capabilities:
|
||||
provides_tools: []
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
name: remarkable
|
||||
repo: https://github.com/cygnostik/hermes-plugin-remarkable
|
||||
sha: 98a0cade0fdc097ad2fca476245597f3bf1ee480
|
||||
sha: d582dd0b05cf99768887a09901350e26208c1c2a
|
||||
description: >-
|
||||
Unofficial reMarkable cloud transfers, library management, backups, typed text, and bounded native notebook rendering.
|
||||
Disclosure — talks to reMarkable cloud with a device token under ~/.hermes/credentials/remarkable/; remarkable_upload
|
||||
@@ -12,6 +12,7 @@ category: tools
|
||||
requires_hermes: ">=0.21.3"
|
||||
version: "0.2.0"
|
||||
docs_url: https://github.com/cygnostik/hermes-plugin-remarkable#readme
|
||||
image: https://raw.githubusercontent.com/cygnostik/hermes-plugin-remarkable/d582dd0b05cf99768887a09901350e26208c1c2a/docs/media/remarkable-hermes-header.png
|
||||
platforms: [windows, linux, macos]
|
||||
capabilities:
|
||||
provides_tools:
|
||||
|
||||
@@ -10,10 +10,11 @@ emitting SSE events.
|
||||
|
||||
Parsed-event activity is recorded on the request-local watchdog state;
|
||||
substantive model progress is recorded separately. For the implicit official
|
||||
OpenAI Codex policy on large contexts, lifecycle frames satisfy TTFB without
|
||||
arming the short post-progress idle budget. Small requests, explicit overrides,
|
||||
and compatible backends retain their first-parsed-event semantics. Raw SSE
|
||||
comments are outside this layer.
|
||||
OpenAI Codex policy on large contexts, lifecycle frames prove transport liveness
|
||||
but do not restart the attempt-local first-progress budget; substantive progress
|
||||
moves the attempt into the normal event-idle phase. Small requests, explicit
|
||||
overrides, and compatible backends retain their first-parsed-event semantics.
|
||||
Raw SSE comments are outside this layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -70,14 +71,18 @@ def _make_codex_agent(
|
||||
return agent
|
||||
|
||||
|
||||
def _shorten_implicit_idle_watchdog(monkeypatch, helpers, timeout=2.0):
|
||||
"""Keep the resolver on its implicit branch while scaling time for tests."""
|
||||
def _shorten_implicit_idle_watchdog(monkeypatch, helpers, timeout=2.0, **overrides):
|
||||
"""Keep the resolver on its implicit branch while scaling time for tests.
|
||||
|
||||
``timeout`` shortens ``idle_timeout``; ``overrides`` set any other resolved field."""
|
||||
monkeypatch.delenv("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", raising=False)
|
||||
original = helpers._resolve_nonstream_watchdogs
|
||||
|
||||
def resolve(agent, api_kwargs):
|
||||
watchdogs = original(agent, api_kwargs)
|
||||
watchdogs.idle_timeout = timeout
|
||||
for field, value in overrides.items():
|
||||
setattr(watchdogs, field, value)
|
||||
return watchdogs
|
||||
|
||||
monkeypatch.setattr(helpers, "_resolve_nonstream_watchdogs", resolve)
|
||||
@@ -320,6 +325,45 @@ def test_idle_phase_policy_is_narrow_and_preserves_operator_overrides(
|
||||
assert watchdogs.est_tokens == input_chars // 4
|
||||
assert watchdogs.idle_enabled is idle_enabled
|
||||
assert watchdogs.idle_requires_progress is requires_progress
|
||||
assert (watchdogs.progress_timeout > 0) is requires_progress
|
||||
|
||||
|
||||
def test_lifecycle_event_does_not_restart_first_progress_deadline():
|
||||
"""The budget belongs to the physical attempt, not to the first lifecycle frame."""
|
||||
from agent import chat_completion_wait_notice as wn
|
||||
|
||||
deadline = wn.codex_watchdog_deadline(
|
||||
stale_timeout=900.0, ttfb_enabled=True, ttfb_timeout=300.0,
|
||||
last_event_ts=280.0, last_progress_ts=None, retry_started_ts=None,
|
||||
call_start=100.0, idle_enabled=True, idle_timeout=120.0,
|
||||
idle_requires_progress=True, progress_timeout=300.0, elapsed=250.0,
|
||||
)
|
||||
|
||||
assert deadline == ("first progress", 50.0)
|
||||
|
||||
|
||||
def test_large_codex_lifecycle_only_stream_hits_attempt_progress_budget(tmp_path, monkeypatch):
|
||||
"""Lifecycle events may change phase diagnostics, but cannot buy another full grace period."""
|
||||
from agent import chat_completion_helpers as h
|
||||
|
||||
agent = _make_codex_agent(tmp_path, monkeypatch)
|
||||
_shorten_implicit_idle_watchdog(monkeypatch, h, ttfb_timeout=0.9, progress_timeout=0.9)
|
||||
closes = []
|
||||
|
||||
def stream_attempt():
|
||||
time.sleep(0.7)
|
||||
yield SimpleNamespace(type="response.created")
|
||||
while getattr(agent, "_active_codex_stream_request_token", None) is not None:
|
||||
time.sleep(0.02)
|
||||
raise ConnectionError("retired lifecycle-only stream")
|
||||
|
||||
_install_codex_event_stream(agent, monkeypatch, stream_attempt, closes)
|
||||
started = time.monotonic()
|
||||
with pytest.raises(TimeoutError, match="no substantive model progress"):
|
||||
h.interruptible_api_call(agent, {"model": "gpt-5.6-sol", "input": "x" * 40_004})
|
||||
|
||||
assert time.monotonic() - started < 1.5
|
||||
assert "codex_progress_kill" in closes
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -23,7 +23,7 @@ def _request():
|
||||
request.call_start = 1000.0
|
||||
request.wd = SimpleNamespace(
|
||||
codex=True, stale_timeout=600.0, ttfb_enabled=True, ttfb_timeout=120.0,
|
||||
idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False,
|
||||
idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False, progress_timeout=0.0,
|
||||
)
|
||||
request.codex_watchdog_state = SimpleNamespace(
|
||||
lock=threading.Lock(), last_event_ts=None, last_progress_ts=None,
|
||||
|
||||
@@ -608,5 +608,37 @@ class TestPerResponseSessionWritePath:
|
||||
assert "is null" not in caplog.text
|
||||
|
||||
|
||||
def test_null_stored_prompt_does_not_take_the_stale_probe_path(tmp_path):
|
||||
"""A NULL system_prompt row already rebuilds. The capability probe must not gate it."""
|
||||
from hermes_state import SessionDB
|
||||
from agent.conversation_loop import _bot_chat_prompt_stale
|
||||
|
||||
agent = SimpleNamespace(
|
||||
_bot_mode_protocol=True,
|
||||
_session_title_hint="Bot Chat",
|
||||
_session_db=None,
|
||||
session_id="test-session-id",
|
||||
)
|
||||
with patch(
|
||||
"tools.bot_mode_probe.stored_prompt_capability_stale", return_value=False
|
||||
) as probe:
|
||||
assert _bot_chat_prompt_stale(agent, None) is False
|
||||
probe.assert_not_called()
|
||||
|
||||
with SessionDB(db_path=tmp_path / "state.db") as db:
|
||||
db.create_session("test-session-id", source="tui")
|
||||
row = db.get_session("test-session-id")
|
||||
assert row is not None and row["system_prompt"] is None
|
||||
restoring = _make_agent(session_db=db, prebuilt_prompt="BUILT")
|
||||
with patch(
|
||||
"tools.bot_mode_probe.stored_prompt_capability_stale", return_value=False
|
||||
) as probe:
|
||||
_restore_or_build_system_prompt(
|
||||
restoring, None, [{"role": "user", "content": "hi"}]
|
||||
)
|
||||
probe.assert_not_called()
|
||||
restoring._build_system_prompt.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -22,7 +22,8 @@ def _nonstream_request(ttfb_timeout=300.0):
|
||||
request.api_kwargs = {"model": "test-model"}
|
||||
request.call_start = 1000.0
|
||||
request.wd = SimpleNamespace(codex=True, stale_timeout=600.0, ttfb_enabled=True, ttfb_timeout=ttfb_timeout,
|
||||
idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False)
|
||||
idle_enabled=True, idle_timeout=180.0, idle_requires_progress=False,
|
||||
progress_timeout=0.0)
|
||||
request.codex_watchdog_state = SimpleNamespace(lock=threading.Lock(), last_event_ts=None,
|
||||
last_progress_ts=None, retry_started_ts=None)
|
||||
request.wait_notice_started_ts = None
|
||||
|
||||
107
tests/hermes_cli/test_linux_ram_probe.py
Normal file
107
tests/hermes_cli/test_linux_ram_probe.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""Linux RAM availability must include reclaimable page cache (#102252).
|
||||
|
||||
``getconf _AVPHYS_PAGES`` counts only free pages, so the Desktop statusbar
|
||||
treats page cache as used. ``MemAvailable`` is the reclaimable-aware figure.
|
||||
A reported ``0`` is a real available value, not a missing field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
|
||||
import hermes_cli.local_runtime.hardware as hw
|
||||
|
||||
GIB = 1 << 30
|
||||
|
||||
|
||||
def _as_linux(monkeypatch) -> None:
|
||||
"""Reach the Linux branch on every host without editing the Windows probe."""
|
||||
monkeypatch.delattr(ctypes, "windll", raising=False)
|
||||
monkeypatch.setattr(hw.sys, "platform", "linux")
|
||||
|
||||
|
||||
def test_memavailable_is_used_when_present():
|
||||
text = (
|
||||
"MemTotal: 67108864 kB\n"
|
||||
"MemFree: 12582912 kB\n"
|
||||
"MemAvailable: 56623104 kB\n"
|
||||
"Cached: 44040192 kB\n"
|
||||
)
|
||||
|
||||
assert hw._linux_ram_from_meminfo(text) == (64 * GIB, 54 * GIB)
|
||||
|
||||
|
||||
def test_zero_memavailable_is_a_real_value():
|
||||
text = (
|
||||
"MemTotal: 8388608 kB\n"
|
||||
"MemFree: 2097152 kB\n"
|
||||
"MemAvailable: 0 kB\n"
|
||||
)
|
||||
|
||||
assert hw._linux_ram_from_meminfo(text) == (8 * GIB, 0)
|
||||
|
||||
|
||||
def test_missing_memavailable_falls_back_to_memfree():
|
||||
text = "MemTotal: 8388608 kB\nMemFree: 2097152 kB\n"
|
||||
|
||||
assert hw._linux_ram_from_meminfo(text) == (8 * GIB, 2 * GIB)
|
||||
|
||||
|
||||
def test_unusable_meminfo_is_not_a_reading():
|
||||
assert hw._linux_ram_from_meminfo("MemAvailable: 1048576 kB\n") is None
|
||||
assert hw._linux_ram_from_meminfo(
|
||||
"MemTotal: 1048576 kB\nMemAvailable: 2097152 kB\n"
|
||||
) is None
|
||||
|
||||
|
||||
def test_ram_bytes_uses_injected_meminfo_including_zero(monkeypatch):
|
||||
_as_linux(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
hw,
|
||||
"_linux_meminfo_text",
|
||||
lambda: (
|
||||
"MemTotal: 8388608 kB\n"
|
||||
"MemFree: 2097152 kB\n"
|
||||
"MemAvailable: 0 kB\n"
|
||||
),
|
||||
)
|
||||
|
||||
def _getconf_must_not_run(*_args, **_kwargs):
|
||||
raise AssertionError("getconf must not run when MemAvailable is present")
|
||||
|
||||
monkeypatch.setattr(hw, "_stdout", _getconf_must_not_run)
|
||||
|
||||
assert hw._ram_bytes() == (8 * GIB, 0)
|
||||
|
||||
|
||||
def test_ram_bytes_falls_back_when_memavailable_is_absent(monkeypatch):
|
||||
_as_linux(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
hw,
|
||||
"_linux_meminfo_text",
|
||||
lambda: "MemTotal: 8388608 kB\nMemFree: 2097152 kB\n",
|
||||
)
|
||||
|
||||
def _getconf_must_not_run(*_args, **_kwargs):
|
||||
raise AssertionError("MemFree is a valid fallback; getconf must not run")
|
||||
|
||||
monkeypatch.setattr(hw, "_stdout", _getconf_must_not_run)
|
||||
|
||||
assert hw._ram_bytes() == (8 * GIB, 2 * GIB)
|
||||
|
||||
|
||||
def test_ram_bytes_falls_back_to_getconf_when_meminfo_unusable(monkeypatch):
|
||||
_as_linux(monkeypatch)
|
||||
monkeypatch.setattr(hw, "_linux_meminfo_text", lambda: None)
|
||||
values = {
|
||||
"PAGE_SIZE": "4096\n",
|
||||
"_PHYS_PAGES": "2097152\n",
|
||||
"_AVPHYS_PAGES": "524288\n",
|
||||
}
|
||||
|
||||
def fake_stdout(*argv):
|
||||
return values[argv[-1]]
|
||||
|
||||
monkeypatch.setattr(hw, "_stdout", fake_stdout)
|
||||
|
||||
assert hw._ram_bytes() == (8 * GIB, 2 * GIB)
|
||||
@@ -209,6 +209,60 @@ def test_fingerprint_changes_on_each_capability_axis(tmp_path):
|
||||
assert bot_mode_probe.capability_fingerprint(home) != after_soul
|
||||
|
||||
|
||||
def test_fingerprint_changes_when_model_vision_override_flips(tmp_path):
|
||||
"""A Bot Chat prompt must rebuild when model.supports_vision flips."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
_make_bot_profile(home, "researcher", managed=True)
|
||||
config = home / "config.yaml"
|
||||
|
||||
config.write_text("model:\n supports_vision: true\n", encoding="utf-8")
|
||||
vision_enabled = bot_mode_probe.capability_fingerprint(home)
|
||||
stamped = "system stuff\n\n" + bot_mode_probe.epoch_line(home)
|
||||
assert vision_enabled == bot_mode_probe.capability_fingerprint(home)
|
||||
assert not bot_mode_probe.stored_prompt_capability_stale(stamped, home)
|
||||
|
||||
config.write_text("model:\n supports_vision: false\n", encoding="utf-8")
|
||||
vision_disabled = bot_mode_probe.capability_fingerprint(home)
|
||||
assert vision_disabled != vision_enabled
|
||||
assert bot_mode_probe.stored_prompt_capability_stale(stamped, home)
|
||||
|
||||
restamped = "system stuff\n\n" + bot_mode_probe.epoch_line(home)
|
||||
assert not bot_mode_probe.stored_prompt_capability_stale(restamped, home)
|
||||
config.write_text("model:\n supports_vision: true\n", encoding="utf-8")
|
||||
assert bot_mode_probe.capability_fingerprint(home) != vision_disabled
|
||||
|
||||
|
||||
def test_vision_override_spellings_share_one_fingerprint(tmp_path):
|
||||
"""YAML boolean tokens that image routing treats as the same override share an epoch."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
_make_bot_profile(home, "researcher", managed=True)
|
||||
config = home / "config.yaml"
|
||||
|
||||
config.write_text("model:\n supports_vision: true\n", encoding="utf-8")
|
||||
enabled = bot_mode_probe.capability_fingerprint(home)
|
||||
config.write_text("model:\n supports_vision: yes\n", encoding="utf-8")
|
||||
assert bot_mode_probe.capability_fingerprint(home) == enabled
|
||||
config.write_text("model:\n supports_vision: false\n", encoding="utf-8")
|
||||
assert bot_mode_probe.capability_fingerprint(home) != enabled
|
||||
|
||||
|
||||
def test_fingerprint_changes_when_model_context_length_override_changes(tmp_path):
|
||||
"""context_length truncates context files in the rebuilt prompt, so it is part of the epoch."""
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
_make_bot_profile(home, "researcher", managed=True)
|
||||
config = home / "config.yaml"
|
||||
|
||||
config.write_text("model:\n context_length: 32768\n", encoding="utf-8")
|
||||
narrow = bot_mode_probe.capability_fingerprint(home)
|
||||
config.write_text("model:\n context_length: 131072\n", encoding="utf-8")
|
||||
assert bot_mode_probe.capability_fingerprint(home) != narrow
|
||||
config.write_text("model:\n context_length: 32768\n", encoding="utf-8")
|
||||
assert bot_mode_probe.capability_fingerprint(home) == narrow
|
||||
|
||||
|
||||
def test_stored_prompt_staleness(tmp_path):
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
|
||||
@@ -229,6 +229,28 @@ class TestUnicodeNormalized:
|
||||
assert new == expected, f"Got {new!r}"
|
||||
|
||||
|
||||
def test_equal_boundary_inside_expansion_keeps_region_text(self):
|
||||
"""An edit boundary falling inside a multi-char expansion (em-dash ->
|
||||
'--') must snap to the expansion, not copy text from the region start.
|
||||
|
||||
SequenceMatcher splits old/new so that the second equal block begins
|
||||
at the expansion's second '-': a norm index with no direct original
|
||||
position. The old fallback (position 0) spliced the whole region text
|
||||
into the replacement, duplicating it after every edit.
|
||||
"""
|
||||
content = "value = x\u2014y\n"
|
||||
new, count, strategy, err = fuzzy_find_and_replace(
|
||||
content, "value = x--y", "value = x-@-y")
|
||||
assert count == 1, f"Expected match, got err={err}"
|
||||
assert strategy == "unicode_normalized"
|
||||
assert new == "value = x\u2014@\u2014y\n", f"Got {new!r}"
|
||||
|
||||
# Same boundary class for a 3-char expansion (ellipsis -> '...').
|
||||
new, count, strategy, err = fuzzy_find_and_replace("a\u2026b\n", "a...b", "a..X.b")
|
||||
assert count == 1, f"Expected match, got err={err}"
|
||||
assert strategy == "unicode_normalized"
|
||||
assert new == "a\u2026X\u2026b\n", f"Got {new!r}"
|
||||
|
||||
|
||||
class TestUnicodeSpaceAndMinusNormalized:
|
||||
"""Space-separator family + Unicode minus normalization.
|
||||
|
||||
@@ -318,19 +318,50 @@ def get_bot_mode_protocol_section(home: str | os.PathLike | None = None, *, forc
|
||||
|
||||
# ── capability epoch ─────────────────────────────────────────────────────────
|
||||
# Bot Chat sessions are effectively eternal, so "build the prompt once" would strand
|
||||
# capability changes (skills, toolsets, MCP, SOUL, roster, peers) forever. The fingerprint
|
||||
# hashes exactly that surface; the built prompt embeds it and agent/conversation_loop.py
|
||||
# rebuilds only when the stored epoch differs from disk — once per change, never per-turn drift.
|
||||
# capability changes (skills, toolsets, MCP, SOUL, roster, peers, model capability
|
||||
# overrides that change the prompt) forever. The fingerprint hashes exactly that
|
||||
# surface; the built prompt embeds it and agent/conversation_loop.py rebuilds only
|
||||
# when the stored epoch differs from disk — once per change, never per-turn drift.
|
||||
|
||||
_EPOCH_PREFIX = "Capability epoch: "
|
||||
_EPOCH_RE_TEXT = r"Capability epoch: ([0-9a-f]{12})"
|
||||
|
||||
|
||||
def _model_prompt_capability_surface(model_cfg: object) -> dict:
|
||||
"""``model.*`` overrides whose flip changes a rebuilt prompt, coerced like their consumers.
|
||||
|
||||
``supports_vision`` uses image routing's strict bool so YAML ``yes`` and ``true`` share
|
||||
one epoch. ``context_length`` is the cap ``build_system_prompt_parts`` uses to truncate
|
||||
context files. Routing keys (provider, default, base_url) are identity lines, not this
|
||||
surface — and no model id is special-cased.
|
||||
"""
|
||||
from agent.image_routing import _coerce_capability_bool
|
||||
|
||||
if not isinstance(model_cfg, dict):
|
||||
model_cfg = {}
|
||||
raw_ctx = model_cfg.get("context_length")
|
||||
ctx = None
|
||||
# bool is an int subclass; ``context_length: true`` is not a window.
|
||||
if isinstance(raw_ctx, bool):
|
||||
ctx = None
|
||||
elif isinstance(raw_ctx, int):
|
||||
ctx = raw_ctx if raw_ctx > 0 else None
|
||||
elif isinstance(raw_ctx, str) and raw_ctx.strip().isdigit():
|
||||
parsed = int(raw_ctx.strip())
|
||||
ctx = parsed if parsed > 0 else None
|
||||
return {
|
||||
"supports_vision": _coerce_capability_bool(model_cfg.get("supports_vision")),
|
||||
"context_length": ctx,
|
||||
}
|
||||
|
||||
|
||||
def capability_fingerprint(home: str | os.PathLike | None = None) -> str:
|
||||
"""12-hex digest of the capability surface for ``home``'s profile: disabled skills +
|
||||
enabled toolsets + MCP config, SOUL.md bytes, installed skill names, the Bot-Mode roster
|
||||
(+ roles), peers and the relay roster. Deliberately NOT cached — the point is detecting
|
||||
on-disk drift against a stored prompt's epoch. Never raises ("unavailable" on failure)."""
|
||||
enabled toolsets + MCP config, model capability overrides that change the prompt
|
||||
(``supports_vision``, ``context_length``), SOUL.md bytes, installed skill names, the
|
||||
Bot-Mode roster (+ roles), peers and the relay roster. Deliberately NOT cached — the
|
||||
point is detecting on-disk drift against a stored prompt's epoch. Never raises
|
||||
("unavailable" on failure)."""
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
@@ -350,6 +381,8 @@ def capability_fingerprint(home: str | os.PathLike | None = None) -> str:
|
||||
reset_hermes_home_override(token)
|
||||
skills_cfg = cfg.get("skills") if isinstance(cfg.get("skills"), dict) else {}
|
||||
tools_cfg = cfg.get("tools") if isinstance(cfg.get("tools"), dict) else {}
|
||||
model_cfg = cfg.get("model") if isinstance(cfg.get("model"), dict) else {}
|
||||
surface["model_capabilities"] = _model_prompt_capability_surface(model_cfg)
|
||||
surface["disabled_skills"] = sorted(str(s).lower() for s in (skills_cfg.get("disabled") or []))
|
||||
surface["enabled_toolsets"] = sorted(str(t) for t in (tools_cfg.get("enabled_toolsets") or []))
|
||||
mcp = cfg.get("mcp_servers")
|
||||
|
||||
@@ -9,6 +9,7 @@ still land on the intended region::
|
||||
content, old_string, new_string, replace_all=False)
|
||||
"""
|
||||
|
||||
import bisect
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from typing import Callable, Optional
|
||||
@@ -509,12 +510,12 @@ def _preserve_unicode_in_replacement(content: str, matches: list[Span],
|
||||
return new_string # strategy shouldn't have fired; fall back
|
||||
|
||||
file_orig_to_norm = _build_orig_to_norm_map(file_region)
|
||||
file_norm_to_orig = _invert_norm_map(file_orig_to_norm)
|
||||
|
||||
result_parts: list[str] = []
|
||||
for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes():
|
||||
if tag == "equal":
|
||||
orig_start = file_norm_to_orig.get(i1, 0)
|
||||
# The original char owning norm index i1, even one inside a multi-char expansion (em-dash -> '--').
|
||||
orig_start = bisect.bisect_right(file_orig_to_norm, i1) - 1
|
||||
orig_end = _norm_end_to_orig(file_orig_to_norm, orig_start, i2)
|
||||
result_parts.append(file_region[orig_start:orig_end])
|
||||
elif tag != "delete":
|
||||
|
||||
Reference in New Issue
Block a user